Update data_process/process/content_extract.py

data_process/process/content_extract.py

@@ -1,6 +1,7 @@
 import os
 import glob
 import json
+from typing import List, Dict
 
 valid_keys = [
     "Core_Fear_Source", "Pain_Threshold", "Time_Window_Pressure", "Helplessness_Index", 
@@ -77,6 +78,25 @@ def extract_json_data(json_files: list, threshold: int = 10) -> dict:
                     raise ValueError(f"Invalid key {key} in {json_file}")
     return filt_json_data(data, threshold)
 
+def load_data_from_dict(data_dict: List[dict]):
+    """
+    不进行阈值过滤，直接加载数据
+    """
+    data = {}
+    for idx, item in enumerate(data_dict):
+        data[idx] = {}
+        for key, value in item.items():
+            if key in valid_keys:
+                data[idx][key] = value.get("value", None)
+            elif key == "Follow_up_Priority":
+                continue
+            else:
+                match_key = try_match_error_key(key)
+                if match_key:
+                    data[idx][match_key] = value.get("value", None)
+                else:
+                    print(f"Warning: Invalid key {key} in data dict, skipped.")
+    return data
 
 if __name__=="__main__":
     deal_folder = "deal"