Update utils/json_to_spacy.py

utils/json_to_spacy.py

@@ -1,8 +1,15 @@
 import json
 import spacy
 from spacy.tokens import DocBin
+import os
 
 def read_in_chunks(file_path, chunk_size=1024):
+    """Read file in chunks to handle large files."""
+    print(f"Reading file: {file_path}")
+    if not os.path.exists(file_path):
+        print(f"Error: File not found at {file_path}")
+        return
+
     with open(file_path, 'r', encoding='utf-8') as file:
         while True:
             data = file.read(chunk_size)
@@ -10,58 +17,71 @@ def read_in_chunks(file_path, chunk_size=1024):
                 break
             yield data
 
-def convert_json_to_spacy(json_file_path, spacy_file_path):
-    # Read the file in chunks and combine the chunks
-    file_content = ""
-    for chunk in read_in_chunks(json_file_path):
-        file_content += chunk
+def extract_text_and_entities(item):
+    """Dynamically extract text and entities, handling multiple JSON formats."""
+    print(f"Processing item: {item}")
+    if isinstance(item, dict):
+        # Dictionary structure: {"text": ..., "entities": ...}
+        text = item.get("text", "")
+        entities = item.get("entities", [])
+    elif isinstance(item, list) and len(item) >= 2:
+        # List structure: ["text", {"entities": ...}]
+        text = item[0] if isinstance(item[0], str) else ""
+        entities = item[1].get("entities", []) if isinstance(item[1], dict) else []
+    else:
+        print(f"Unexpected item format: {item}")
+        return None, []  # Return empty text and entities
 
-    # Parse the JSON data
-    data = json.loads(file_content)
+    valid_entities = [
+        (start, end, label) for start, end, label in entities
+        if isinstance(start, int) and isinstance(end, int) and isinstance(label, str)
+    ]
+    return text, valid_entities
+
+def convert_json_to_spacy(json_file_path, spacy_file_path):
+    """Convert JSON data to spaCy format and save as .spacy file."""
+    try:
+        print(f"Reading JSON from: {json_file_path}")
+        file_content = "".join(chunk for chunk in read_in_chunks(json_file_path))
 
-    # Prepare the data for spaCy
-    spacy_format = []
+        data = json.loads(file_content)  # Parse JSON data
+        print(f"Successfully loaded JSON data. Found {len(data)} items.")
 
-    for item in data:
-        text = item[0]  # The first element in the list is the text
-        entities = item[1]['entities']  # The second element contains the dictionary with 'entities'
-        spacy_entities = [(start, end, label) for start, end, label in entities]
-        spacy_format.append({"text": text, "entities": spacy_entities})
+        spacy_format = []
+        for item in data:
+            text, entities = extract_text_and_entities(item)
+            if text:  # Skip if text is empty or invalid
+                spacy_format.append({"text": text, "entities": entities})
 
-    # Create a blank English model
-    nlp = spacy.blank("en")
+        # Create a blank spaCy model
+        nlp = spacy.blank("en")
+        doc_bin = DocBin()
 
-    # Initialize a DocBin object
-    doc_bin = DocBin()
+        for entry in spacy_format:
+            print(f"Creating spaCy Doc for text: {entry['text']}")
+            doc = nlp.make_doc(entry["text"])
+            entities = []
+            seen_positions = set()
 
-    # Convert the data to spaCy Doc objects and add to DocBin
-    for entry in spacy_format:
-        doc = nlp.make_doc(entry["text"])
-        # Convert entities
-        entities = []
-        seen_positions = set()  # To track positions and avoid overlap
-        for start, end, label in entry["entities"]:
-            # Ensure span is within the document's length
-            if start < 0 or end > len(doc.text) or start >= end:
-                print(f"Invalid span: start={start}, end={end}, label={label}")
-                continue
+            for start, end, label in entry["entities"]:
+                if start < 0 or end > len(doc.text) or start >= end:
+                    print(f"Invalid span: start={start}, end={end}, label={label}")
+                    continue
+                if not any(start < e_end and end > e_start for e_start, e_end, _ in seen_positions):
+                    span = doc.char_span(start, end, label=label)
+                    if span is not None:
+                        entities.append(span)
+                        seen_positions.add((start, end, label))
+                else:
+                    print(f"Overlapping span: start={start}, end={end}, label={label}")
 
-            # Check for overlaps and prioritize entities
-            if not any(start < e_end and end > e_start for e_start, e_end, _ in seen_positions):
-                span = doc.char_span(start, end, label=label)
-                if span is not None:
-                    entities.append(span)
-                    seen_positions.add((start, end, label))
-            else:
-                print(f"Overlapping span: start={start}, end={end}, label={label}")
-        
-        # Set entities
-        doc.ents = entities
-        
-        # Add to DocBin
-        doc_bin.add(doc)
+            doc.ents = entities
+            doc_bin.add(doc)
 
-    # Save the DocBin to a .spacy file
-    doc_bin.to_disk(spacy_file_path)
+        doc_bin.to_disk(spacy_file_path)
+        print(f"Data has been successfully saved to {spacy_file_path}!")
 
-    print(f"Data has been successfully saved to {spacy_file_path}!")
+    except json.JSONDecodeError as e:
+        print(f"Error decoding JSON: {e}")
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")