test 2

app.py

@@ -10,35 +10,41 @@ from llama_index.embeddings import HuggingFaceEmbedding
 from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 import torch
 from dotenv import load_dotenv
+from typing import Optional, Dict, Any
+import yt_dlp
+from transformers import AutoProcessor, AutoModelForVision2Seq
+from PIL import Image
+import re
 
 # (Keep Constants as is)
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
+
 # --- Basic Agent Definition ---
 # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
 class BasicAgent:
     def __init__(self):
         print("BasicAgent initialized.")
         load_dotenv()
-        
+
         # Use official Mistral model
         model_name = "mistralai/Mixtral-8x7B-Instruct-v0.2"  # or "mistralai/Mistral-7B-Instruct-v0.2"
-        
+
         # Configure quantization for efficient CPU usage
         quantization_config = BitsAndBytesConfig(
             load_in_8bit=True,  # 8-bit is more CPU friendly than 4-bit
-            bnb_4bit_compute_dtype=torch.float16
+            bnb_4bit_compute_dtype=torch.float16,
         )
-        
+
         self.tokenizer = AutoTokenizer.from_pretrained(model_name)
         self.model = AutoModelForCausalLM.from_pretrained(
             model_name,
             low_cpu_mem_usage=True,
             trust_remote_code=True,
-            quantization_config=quantization_config
+            quantization_config=quantization_config,
         )
-        
+
         # Create HuggingFaceLLM instance
         self.llm = HuggingFaceLLM(
             context_window=4096,
@@ -47,67 +53,135 @@ class BasicAgent:
             model=self.model,
             model_name=model_name,
             device_map="auto",
-            temperature=0.1
+            temperature=0.1,
         )
-        
-        print("Agent initialized.")
+
+        # Initialize vision model for image/video understanding
+        self.vision_processor = AutoProcessor.from_pretrained(
+            "microsoft/kosmos-2-patch14-224"
+        )
+        self.vision_model = AutoModelForVision2Seq.from_pretrained(
+            "microsoft/kosmos-2-patch14-224"
+        )
+
+        # Initialize tools
+        self.tools = {
+            "analyze_image": self.analyze_image,
+            "analyze_video": self.analyze_video,
+            "analyze_data": self.analyze_data,
+        }
+
+        print("Agent initialized with multimodal capabilities.")
+
+    def analyze_image(self, image_url: str) -> str:
+        try:
+            response = requests.get(image_url)
+            image = Image.open(BytesIO(response.content))
+            inputs = self.vision_processor(images=image, return_tensors="pt")
+            outputs = self.vision_model.generate(
+                pixel_values=inputs["pixel_values"], max_length=128, num_beams=5
+            )
+            return self.vision_processor.batch_decode(
+                outputs, skip_special_tokens=True
+            )[0]
+        except Exception as e:
+            print(f"Error analyzing image: {e}")
+            return "Error analyzing image"
+
+    def analyze_video(self, video_url: str) -> str:
+        try:
+            # Extract video info using yt-dlp
+            ydl_opts = {
+                "format": "worst",  # Lowest quality to save bandwidth
+                "extract_flat": True,
+                "quiet": True,
+            }
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                info = ydl.extract_info(video_url, download=False)
+
+            # Extract relevant information
+            title = info.get("title", "")
+            description = info.get("description", "")
+            duration = info.get("duration", 0)
+
+            # Create prompt for the LLM
+            video_context = f"""
+            Video Title: {title}
+            Duration: {duration} seconds
+            Description: {description}
+            """
+
+            return self.generate_response(video_context)
+        except Exception as e:
+            print(f"Error analyzing video: {e}")
+            return "Error analyzing video"
+
+    def analyze_data(self, df: pd.DataFrame, question: str) -> str:
+        try:
+            prompt = f"""Analyze this DataFrame:
+            Columns: {', '.join(df.columns)}
+            Sample: {df.head().to_string()}
+            Question: {question}
+            Provide only the numerical answer or specific value."""
+
+            return self.generate_response(prompt)
+        except Exception as e:
+            print(f"Error analyzing data: {e}")
+            return "Error analyzing data"
+
+    def generate_response(self, prompt: str) -> str:
+        try:
+            response = self.llm.complete(prompt)
+            return response.text.strip()
+        except Exception as e:
+            print(f"Error generating response: {e}")
+            return "Error generating response"
 
     def __call__(self, question: str) -> str:
         print(f"Agent received question: {question[:50]}...")
-        
+
         try:
-            # Extract data from the question if present
-            data_start = question.find('```python')
-            if data_start != -1:
-                data_end = question.find('```', data_start + 8)
-                data_code = question[data_start + 8:data_end].strip()
-                
-                # Execute the data code in a safe context to create DataFrame
+            # Check for data analysis task
+            if "```python" in question:
+                data_start = question.find("```python")
+                data_end = question.find("```", data_start + 8)
+                data_code = question[data_start + 8 : data_end].strip()
                 local_vars = {}
                 exec(data_code, {"pd": pd}, local_vars)
-                df = local_vars.get('df')
-                
-                # Extract the actual question (usually after the code block)
-                actual_question = question[data_end + 3:].strip()
-                
-                if df is not None:
-                    # Analyze DataFrame based on the question
-                    prompt = f"""You are a data analysis expert. Answer this question about the DataFrame:
-
-                    Question: {actual_question}
-
-                    The DataFrame has {len(df)} rows and these columns:
-                    {', '.join(df.columns)}
-
-                    First few rows of data:
-                    {df.head().to_string()}
-
-                    Provide only the numerical answer or specific value, without explanation."""
-                    
-                    # Get response from model
-                    response = self.llm.complete(prompt)
-                    return response.text.strip()
-                
-            # For non-DataFrame questions or if no DataFrame is found
-            prompt = f"""Answer this question clearly and concisely: {question}
-            Provide only the specific answer requested, no explanations."""
-            response = self.llm.complete(prompt)
-            return response.text.strip()
-            
+                df = local_vars.get("df")
+                actual_question = question[data_end + 3 :].strip()
+                return self.analyze_data(df, actual_question)
+
+            # Check for video analysis task
+            video_pattern = r"https?://(?:www\.)?youtube\.com/\S+"
+            video_match = re.search(video_pattern, question)
+            if video_match:
+                return self.analyze_video(video_match.group(0))
+
+            # Check for image analysis task
+            image_pattern = r"https?://\S+\.(?:jpg|jpeg|png|gif)"
+            image_match = re.search(image_pattern, question)
+            if image_match:
+                return self.analyze_image(image_match.group(0))
+
+            # General question
+            return self.generate_response(question)
+
         except Exception as e:
             print(f"Error processing question: {e}")
             return "Error occurred while processing the question"
 
-def run_and_submit_all( profile: gr.OAuthProfile | None):
+
+def run_and_submit_all(profile: gr.OAuthProfile | None):
     """
     Fetches all questions, runs the BasicAgent on them, submits all answers,
     and displays the results.
     """
     # --- Determine HF Space Runtime URL and Repo URL ---
-    space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
+    space_id = os.getenv("SPACE_ID")  # Get the SPACE_ID for sending link to the code
 
     if profile:
-        username= f"{profile.username}"
+        username = f"{profile.username}"
         print(f"User logged in: {username}")
     else:
         print("User not logged in.")
@@ -134,16 +208,16 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
         response.raise_for_status()
         questions_data = response.json()
         if not questions_data:
-             print("Fetched questions list is empty.")
-             return "Fetched questions list is empty or invalid format.", None
+            print("Fetched questions list is empty.")
+            return "Fetched questions list is empty or invalid format.", None
         print(f"Fetched {len(questions_data)} questions.")
     except requests.exceptions.RequestException as e:
         print(f"Error fetching questions: {e}")
         return f"Error fetching questions: {e}", None
     except requests.exceptions.JSONDecodeError as e:
-         print(f"Error decoding JSON response from questions endpoint: {e}")
-         print(f"Response text: {response.text[:500]}")
-         return f"Error decoding server response for questions: {e}", None
+        print(f"Error decoding JSON response from questions endpoint: {e}")
+        print(f"Response text: {response.text[:500]}")
+        return f"Error decoding server response for questions: {e}", None
     except Exception as e:
         print(f"An unexpected error occurred fetching questions: {e}")
         return f"An unexpected error occurred fetching questions: {e}", None
@@ -160,18 +234,36 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
             continue
         try:
             submitted_answer = agent(question_text)
-            answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
-            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
+            answers_payload.append(
+                {"task_id": task_id, "submitted_answer": submitted_answer}
+            )
+            results_log.append(
+                {
+                    "Task ID": task_id,
+                    "Question": question_text,
+                    "Submitted Answer": submitted_answer,
+                }
+            )
         except Exception as e:
-             print(f"Error running agent on task {task_id}: {e}")
-             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
+            print(f"Error running agent on task {task_id}: {e}")
+            results_log.append(
+                {
+                    "Task ID": task_id,
+                    "Question": question_text,
+                    "Submitted Answer": f"AGENT ERROR: {e}",
+                }
+            )
 
     if not answers_payload:
         print("Agent did not produce any answers to submit.")
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
 
-    # 4. Prepare Submission 
-    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
+    # 4. Prepare Submission
+    submission_data = {
+        "username": username.strip(),
+        "agent_code": agent_code,
+        "answers": answers_payload,
+    }
     status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
     print(status_update)
 
@@ -241,20 +333,19 @@ with gr.Blocks() as demo:
 
     run_button = gr.Button("Run Evaluation & Submit All Answers")
 
-    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
+    status_output = gr.Textbox(
+        label="Run Status / Submission Result", lines=5, interactive=False
+    )
     # Removed max_rows=10 from DataFrame constructor
     results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
 
-    run_button.click(
-        fn=run_and_submit_all,
-        outputs=[status_output, results_table]
-    )
+    run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])
 
 if __name__ == "__main__":
-    print("\n" + "-"*30 + " App Starting " + "-"*30)
+    print("\n" + "-" * 30 + " App Starting " + "-" * 30)
     # Check for SPACE_HOST and SPACE_ID at startup for information
     space_host_startup = os.getenv("SPACE_HOST")
-    space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
+    space_id_startup = os.getenv("SPACE_ID")  # Get SPACE_ID at startup
 
     if space_host_startup:
         print(f"✅ SPACE_HOST found: {space_host_startup}")
@@ -262,14 +353,18 @@ if __name__ == "__main__":
     else:
         print("ℹ️  SPACE_HOST environment variable not found (running locally?).")
 
-    if space_id_startup: # Print repo URLs if SPACE_ID is found
+    if space_id_startup:  # Print repo URLs if SPACE_ID is found
         print(f"✅ SPACE_ID found: {space_id_startup}")
         print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
-        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
+        print(
+            f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main"
+        )
     else:
-        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
+        print(
+            "ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined."
+        )
 
-    print("-"*(60 + len(" App Starting ")) + "\n")
+    print("-" * (60 + len(" App Starting ")) + "\n")
 
     print("Launching Gradio Interface for Basic Agent Evaluation...")
-    demo.launch(debug=True, share=False)
+    demo.launch(debug=True, share=False)

requirements.txt

@@ -7,4 +7,6 @@ torch
 sentence-transformers
 nltk
 accelerate
-bitsandbytes>=0.41.0
+bitsandbytes>=0.41.0
+yt-dlp
+Pillow