Update app.py

app.py

@@ -19,7 +19,7 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 def load_qa_model():
-    """Load question-answering model with support for long input contexts."""
+    """Load question-answering model with long context support."""
     try:
         from transformers import AutoTokenizer, AutoModelForCausalLM
         
@@ -27,26 +27,26 @@ def load_qa_model():
         
         # Load tokenizer
         tokenizer = AutoTokenizer.from_pretrained(model_id, use_auth_token=os.getenv("HF_TOKEN"))
-        tokenizer.model_max_length = 8192  # Ensure the tokenizer can handle 8192 tokens
+        tokenizer.model_max_length = 8192  # Configure tokenizer for long inputs
         
-        # Load the model
+        # Load the model with simplified rope_scaling configuration
         model = AutoModelForCausalLM.from_pretrained(
             model_id,
             torch_dtype=torch.bfloat16,
             device_map="auto",
             rope_scaling={
-                "type": "dynamic",  # Ensure compatibility with long contexts
-                "factor": 8.0
+                "type": "dynamic",  # Simplified type as expected by the model
+                "factor": 8.0       # Scaling factor to support longer contexts
             },
             use_auth_token=os.getenv("HF_TOKEN")
         )
         
-        # Load the pipeline
+        # Initialize the pipeline
         qa_pipeline = pipeline(
             "text-generation",
             model=model,
             tokenizer=tokenizer,
-            max_new_tokens=4096,  # Adjust as needed for your use case
+            max_new_tokens=256,  # Limit generation as needed
         )
         
         return qa_pipeline
@@ -55,6 +55,7 @@ def load_qa_model():
         logger.error(f"Failed to load Q&A model: {str(e)}")
         return None
 
+        
 # def load_qa_model():
 #     """Load question-answering model"""
 #     try: