Update app.py

app.py

@@ -61,6 +61,139 @@ def max_over_ground_truths(metric_fn, prediction, ground_truths):
         scores.append(score)
     return max(scores) if scores else 0
 
+def load_cuad_dataset(hf_token=None):
+    """Try multiple methods to load CUAD dataset"""
+    print("Attempting to load CUAD dataset...")
+    
+    # Method 1: Try theatticusproject/cuad
+    try:
+        print("Trying theatticusproject/cuad...")
+        dataset = load_dataset("theatticusproject/cuad", token=hf_token)
+        if "test" in dataset:
+            test_data = dataset["test"]
+            print(f"✓ Loaded theatticusproject/cuad with {len(test_data)} test samples")
+            return test_data, "theatticusproject/cuad"
+        elif "validation" in dataset:
+            test_data = dataset["validation"]
+            print(f"✓ Loaded theatticusproject/cuad with {len(test_data)} validation samples")
+            return test_data, "theatticusproject/cuad"
+        else:
+            print("No test or validation split found in theatticusproject/cuad")
+    except Exception as e:
+        print(f"Failed to load theatticusproject/cuad: {e}")
+    
+    # Method 2: Try theatticusproject/cuad-qa
+    try:
+        print("Trying theatticusproject/cuad-qa...")
+        dataset = load_dataset("theatticusproject/cuad-qa", token=hf_token)
+        if "test" in dataset:
+            test_data = dataset["test"]
+            print(f"✓ Loaded theatticusproject/cuad-qa with {len(test_data)} test samples")
+            return test_data, "theatticusproject/cuad-qa"
+        elif "validation" in dataset:
+            test_data = dataset["validation"]
+            print(f"✓ Loaded theatticusproject/cuad-qa with {len(test_data)} validation samples")
+            return test_data, "theatticusproject/cuad-qa"
+    except Exception as e:
+        print(f"Failed to load theatticusproject/cuad-qa: {e}")
+    
+    # Method 3: Try the original cuad identifier
+    try:
+        print("Trying cuad...")
+        dataset = load_dataset("cuad", token=hf_token)
+        if "test" in dataset:
+            test_data = dataset["test"]
+            print(f"✓ Loaded cuad with {len(test_data)} test samples")
+            return test_data, "cuad"
+        elif "validation" in dataset:
+            test_data = dataset["validation"]
+            print(f"✓ Loaded cuad with {len(test_data)} validation samples")
+            return test_data, "cuad"
+    except Exception as e:
+        print(f"Failed to load cuad: {e}")
+    
+    # Method 4: Try with trust_remote_code=True
+    try:
+        print("Trying with trust_remote_code=True...")
+        dataset = load_dataset("theatticusproject/cuad", token=hf_token, trust_remote_code=True)
+        if "test" in dataset:
+            test_data = dataset["test"]
+            print(f"✓ Loaded with trust_remote_code, test samples: {len(test_data)}")
+            return test_data, "theatticusproject/cuad (trust_remote_code)"
+        elif "validation" in dataset:
+            test_data = dataset["validation"]
+            print(f"✓ Loaded with trust_remote_code, validation samples: {len(test_data)}")
+            return test_data, "theatticusproject/cuad (trust_remote_code)"
+    except Exception as e:
+        print(f"Failed with trust_remote_code: {e}")
+    
+    # Method 5: Create a synthetic CUAD-like dataset for testing
+    print("⚠️ Creating synthetic CUAD-like test data...")
+    synthetic_data = []
+    
+    # Create some contract-like questions and contexts
+    contract_samples = [
+        {
+            "context": "This Agreement shall commence on January 1, 2024 and shall continue for a period of twelve (12) months unless terminated earlier in accordance with the terms hereof. The initial term may be extended for additional periods of twelve (12) months each upon mutual written consent of both parties.",
+            "question": "What is the duration of the agreement?",
+            "answers": {"text": ["twelve (12) months", "12 months"], "answer_start": [85, 85]}
+        },
+        {
+            "context": "The Company shall pay the Consultant a fee of $50,000 per month for services rendered under this Agreement. Payment shall be made within thirty (30) days of the end of each calendar month.",
+            "question": "What is the monthly fee?",
+            "answers": {"text": ["$50,000 per month", "$50,000"], "answer_start": [45, 45]}
+        },
+        {
+            "context": "Either party may terminate this Agreement immediately upon written notice in the event of a material breach by the other party that remains uncured for thirty (30) days after written notice of such breach.",
+            "question": "What is the cure period for material breach?",
+            "answers": {"text": ["thirty (30) days", "30 days"], "answer_start": [125, 132]}
+        },
+        {
+            "context": "The Contractor shall maintain commercial general liability insurance with coverage of not less than $1,000,000 per occurrence and $2,000,000 in the aggregate.",
+            "question": "What is the minimum insurance coverage per occurrence?",
+            "answers": {"text": ["$1,000,000 per occurrence", "$1,000,000"], "answer_start": [85, 85]}
+        },
+        {
+            "context": "All intellectual property developed under this Agreement shall be owned by the Company. The Contractor hereby assigns all rights, title and interest in such intellectual property to the Company.",
+            "question": "Who owns the intellectual property?",
+            "answers": {"text": ["the Company", "Company"], "answer_start": [70, 74]}
+        }
+    ]
+    
+    # Duplicate samples to create a larger dataset
+    for i in range(100):  # Create 500 samples
+        sample = contract_samples[i % len(contract_samples)].copy()
+        sample["id"] = f"synthetic_{i}"
+        synthetic_data.append(sample)
+    
+    # Convert to dataset format
+    from datasets import Dataset
+    test_data = Dataset.from_list(synthetic_data)
+    
+    print(f"✓ Created synthetic CUAD-like dataset with {len(test_data)} samples")
+    return test_data, "synthetic_cuad"
+
+def inspect_dataset_structure(dataset, dataset_name="dataset"):
+    """Inspect dataset structure for debugging"""
+    print(f"\n=== {dataset_name} Dataset Structure ===")
+    print(f"Dataset type: {type(dataset)}")
+    print(f"Dataset length: {len(dataset)}")
+    
+    if len(dataset) > 0:
+        sample = dataset[0]
+        print(f"Sample keys: {list(sample.keys()) if isinstance(sample, dict) else 'Not a dict'}")
+        print(f"Sample structure:")
+        for key, value in sample.items():
+            if isinstance(value, dict):
+                print(f"  {key} (dict): {list(value.keys())}")
+                for sub_key, sub_value in value.items():
+                    print(f"    {sub_key}: {type(sub_value)} - {str(sub_value)[:50]}...")
+            else:
+                print(f"  {key}: {type(value)} - {str(value)[:100]}...")
+    print("=" * 50)
+    
+    return dataset
+
 def evaluate_model():
     # Authenticate with Hugging Face using the token
     hf_token = os.getenv("EVAL_TOKEN")
@@ -86,21 +219,6 @@ def evaluate_model():
         print(f"✗ Error loading model: {e}")
         return None, None
 
-def inspect_dataset_structure(dataset, num_samples=3):
-    """Inspect dataset structure for debugging"""
-    print(f"Dataset structure inspection:")
-    print(f"Dataset type: {type(dataset)}")
-    print(f"Dataset length: {len(dataset)}")
-    
-    if len(dataset) > 0:
-        sample = dataset[0]
-        print(f"Sample keys: {list(sample.keys()) if isinstance(sample, dict) else 'Not a dict'}")
-        print(f"Sample structure:")
-        for key, value in sample.items():
-            print(f"  {key}: {type(value)} - {str(value)[:100]}...")
-    
-    return dataset
-
 def run_evaluation(num_samples, progress=gr.Progress()):
     """Run evaluation and return results for Gradio interface"""
     
@@ -111,39 +229,19 @@ def run_evaluation(num_samples, progress=gr.Progress()):
     
     progress(0.1, desc="Loading CUAD dataset...")
     
-    # Load dataset - try multiple approaches
-    dataset = None
-    test_data = None
-    
-    try:
-        # Try cuad dataset directly
-        print("Attempting to load CUAD dataset...")
-        dataset = load_dataset("cuad", token=hf_token)
-        test_data = dataset["test"]
-        print(f"✓ Loaded CUAD dataset with {len(test_data)} samples")
-        
-        # Inspect structure
-        test_data = inspect_dataset_structure(test_data)
-        
-    except Exception as e:
-        print(f"Error loading CUAD dataset: {e}")
-        try:
-            # Try squad format as fallback
-            print("Trying SQuAD format...")
-            dataset = load_dataset("squad", split="validation", token=hf_token)
-            test_data = dataset.select(range(min(1000, len(dataset))))
-            print(f"✓ Loaded SQuAD dataset as fallback with {len(test_data)} samples")
-        except Exception as e2:
-            return f"❌ Error loading any dataset: {e2}", pd.DataFrame(), None
-    
+    # Load dataset
+    test_data, dataset_name = load_cuad_dataset(hf_token)
     if test_data is None:
-        return "❌ No test data available", pd.DataFrame(), None
+        return "❌ Failed to load any dataset", pd.DataFrame(), None
+    
+    # Inspect dataset structure
+    test_data = inspect_dataset_structure(test_data, dataset_name)
     
     # Limit samples
     num_samples = min(num_samples, len(test_data))
     test_subset = test_data.select(range(num_samples))
     
-    progress(0.2, desc=f"Starting evaluation on {num_samples} samples...")
+    progress(0.2, desc=f"Starting evaluation on {num_samples} samples from {dataset_name}...")
     
     # Initialize metrics
     exact_matches = []
@@ -179,11 +277,11 @@ def run_evaluation(num_samples, progress=gr.Progress()):
                 if isinstance(answers, dict):
                     if "text" in answers:
                         if isinstance(answers["text"], list):
-                            ground_truths = [ans for ans in answers["text"] if ans.strip()]
+                            ground_truths = [ans for ans in answers["text"] if ans and ans.strip()]
                         else:
-                            ground_truths = [answers["text"]] if answers["text"].strip() else []
+                            ground_truths = [answers["text"]] if answers["text"] and answers["text"].strip() else []
                 elif isinstance(answers, list):
-                    ground_truths = answers
+                    ground_truths = [ans for ans in answers if ans and ans.strip()]
             
             # Skip if no ground truth
             if not ground_truths:
@@ -238,9 +336,13 @@ def run_evaluation(num_samples, progress=gr.Progress()):
     # Create results summary
     results_summary = f"""
 # 📊 CUAD Model Evaluation Results
+
+## ⚠️ Dataset Information
+- **Dataset Used**: {dataset_name}
+- **Dataset Status**: {"✅ Authentic CUAD" if "cuad" in dataset_name.lower() and "synthetic" not in dataset_name else "⚠️ Fallback/Synthetic Data"}
+
 ## 🎯 Overall Performance
 - **Model**: AvocadoMuffin/roberta-cuad-qa-v3
-- **Dataset**: CUAD (Contract Understanding Atticus Dataset)
 - **Samples Evaluated**: {len(exact_matches)}
 - **Evaluation Date**: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
 
@@ -259,6 +361,8 @@ def run_evaluation(num_samples, progress=gr.Progress()):
 - **Samples with Multiple Ground Truths**: {len([p for p in predictions if p['Num_Ground_Truths'] > 1])}
 
 ## 🎯 Evaluation Quality
+{"✅ This evaluation uses the proper CUAD dataset for contract understanding tasks." if "cuad" in dataset_name.lower() and "synthetic" not in dataset_name else "⚠️ WARNING: This evaluation used fallback data. Results may not be representative of actual CUAD performance."}
+
 The evaluation accounts for multiple ground truth answers where available, using the maximum score across all valid answers for each question.
 """
     
@@ -271,12 +375,13 @@ The evaluation accounts for multiple ground truth answers where available, using
     
     detailed_results = {
         "model_name": "AvocadoMuffin/roberta-cuad-qa-v3",
-        "dataset": "cuad",
+        "dataset": dataset_name,
         "num_samples": len(exact_matches),
         "exact_match_score": avg_exact_match,
         "f1_score": avg_f1_score,
         "evaluation_date": datetime.now().isoformat(),
         "evaluation_methodology": "max_over_ground_truths",
+        "dataset_authentic": "cuad" in dataset_name.lower() and "synthetic" not in dataset_name,
         "predictions": predictions,
         "summary_stats": {
             "avg_confidence": float(np.mean([p['Confidence'] for p in predictions])),
@@ -306,6 +411,7 @@ def create_gradio_interface():
             <h1>🏛️ CUAD Model Evaluation Dashboard</h1>
             <p>Evaluate your CUAD (Contract Understanding Atticus Dataset) Question Answering model</p>
             <p><strong>Model:</strong> AvocadoMuffin/roberta-cuad-qa-v3</p>
+            <p><em>This tool will attempt to load the authentic CUAD dataset, with fallbacks if needed.</em></p>
         </div>
         """)
         
@@ -337,6 +443,7 @@ def create_gradio_interface():
                         <li><strong>Confidence</strong>: Model's confidence in its predictions</li>
                         <li><strong>Max-over-GT</strong>: Best score across multiple ground truth answers</li>
                     </ul>
+                    <p><strong>Note:</strong> This tool will try to load the authentic CUAD dataset. If that fails, it will use synthetic contract data for testing purposes.</p>
                 </div>
                 """)
             
@@ -386,6 +493,7 @@ def create_gradio_interface():
         <div style="text-align: center; margin-top: 30px; padding: 20px; color: #666;">
             <p>🤖 Powered by Hugging Face Transformers & Gradio</p>
             <p>📚 CUAD Dataset by The Atticus Project</p>
+            <p><small>⚠️ If authentic CUAD data cannot be loaded, synthetic contract data will be used for testing purposes.</small></p>
         </div>
         """)