feat: image_generation and best model for task tools

app.py

@@ -1,5 +1,4 @@
-import datetime
-import requests
+from datetime import datetime
 import pytz
 import yaml
 import pycountry
@@ -7,69 +6,59 @@ import pycountry
 from tools.final_answer import FinalAnswerTool
 from tools.visit_webpage import VisitWebpageTool
 from tools.translation import TranslationTool
+from tools.best_model_for_task import HFModelDownloadsTool
 
 from transformers import pipeline
 from Gradio_UI import GradioUI
-from typing import Optional
 
 import os
 import base64
+from dotenv import load_dotenv
 
 from opentelemetry.sdk.trace import TracerProvider
 from openinference.instrumentation.smolagents import SmolagentsInstrumentor
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 from opentelemetry.sdk.trace.export import SimpleSpanProcessor
 
+from langchain_community.agent_toolkits.load_tools import load_tools
+from langchain.chains import LLMChain
+from langchain_community.utilities.dalle_image_generator import DallEAPIWrapper
+from langchain_core.prompts import PromptTemplate
+from langchain_openai import OpenAI
+
+from skimage import io
+from PIL import Image
+
 from smolagents import (
     CodeAgent,
     DuckDuckGoSearchTool,
     GoogleSearchTool,
     HfApiModel,
+    TransformersModel,
     load_tool,
-    tool
+    Tool,
+    tool,
+    ToolCollection
 )
 
+# load .env vars
+load_dotenv()
 
+
+# fast prototyping tools
 @tool
 def get_current_time_in_timezone(timezone: str) -> str:
-    """A tool that fetches the current local time in a specified timezone.
+    """A tool that fetches the current local time in a specified timezone formatted as '%m/%d/%y %H:%M:%S'
     Args:
         timezone (str): A string representing a valid timezone (e.g., 'America/New_York').
     """
     try:
-        # Create timezone object
         tz = pytz.timezone(timezone)
-        # Get current time in that timezone
-        local_time = datetime.datetime.now(tz).strftime("%Y-%m-%d %H:%M:%S")
+        local_time = datetime.now(tz).strftime('%m/%d/%y %H:%M:%S')
         return f"The current local time in {timezone} is: {local_time}"
     except Exception as e:
         return f"Error fetching time for timezone '{timezone}': {str(e)}"
 
-@tool
-def conversational_utterance(user_content: str, additional_context: Optional[str]="") -> str:
-    """
-    A tool that replies to a single casual query or message triggering any other tool is unfitted to reply.
-    
-    Args:
-        user_content: A string with the user's message or query (e.g., "Hi!", "How are you?", "Tell me a joke").
-        additional_context: An optional string with additional information (such as context, metadata, conversation history,
-            or instructions) to be passed as an 'assistant' turn (a thought) in the conversation. 
-    """
-    system_context_message = f"""
-        You are a highly intelligent, expert, and witty assistant who responds to user conversational messages.
-        You function as a tool activated by user intention via AI agents. In addition to your native LLM capabilities,
-        you have access to the following system tools that the user may leverage:
-        {tools}
-        You should mention these tools whenever relevant during the conversation.
-    """
-    
-    messages = [
-        {"role": "system", "content": [{"type": "text", "text": system_context_message}]},
-        {"role": "assistant", "content": [{"type": "text", "text": f"(additional_context: {additional_context})"}]},
-        {"role": "user", "content": [{"type": "text", "text": user_content}]}
-    ]
-    return model(messages).content
-
 
 @tool
 def language_detection(text:str)-> str:
@@ -100,6 +89,25 @@ def language_detection(text:str)-> str:
         return "None"
 
 
+@tool
+def advanced_image_generation(description:str)->Image.Image:
+    """Generates an image using a textual description.
+         Args:
+            description: the textual description provided by the user to prompt a text-to-image model
+        """
+    llm = OpenAI(temperature=0.9)
+    prompt = PromptTemplate(
+        input_variables=["image_desc"],
+        template="Generate a detailed but short prompt (must be less than 900 characters) to generate an image based on the following description: {image_desc}",
+    )
+    chain = LLMChain(llm=llm, prompt=prompt)
+    image_url = DallEAPIWrapper().run(chain.run(description))
+    image_array = io.imread(image_url)
+    pil_image = Image.fromarray(image_array)
+    return pil_image
+
+
+# telemetry
 def initialize_langfuse_opentelemetry_instrumentation():
     LANGFUSE_PUBLIC_KEY=os.environ.get("LANGFUSE_PUBLIC_KEY")
     LANGFUSE_SECRET_KEY=os.environ.get("LANGFUSE_SECRET_KEY")
@@ -115,50 +123,61 @@ def initialize_langfuse_opentelemetry_instrumentation():
 
 initialize_langfuse_opentelemetry_instrumentation()
 
-# tools from /tools/
+# load tools from /tools/
 final_answer = FinalAnswerTool()
 visit_webpage = VisitWebpageTool()
-translation_tool = TranslationTool()
-
-# tools from smoloagents library
-prefered_web_search = GoogleSearchTool()
-prefered_web_search.name = "preferred_web_search"
-alternative_web_search = DuckDuckGoSearchTool()
-alternative_web_search.name = "alternative_web_search"
+translation = TranslationTool()
+best_model_for_task = HFModelDownloadsTool()
+
+# load tools from smoloagents library
+google_web_search = GoogleSearchTool()
+google_web_search.name = "google_web_search"
+duckduckgo_web_search = DuckDuckGoSearchTool()
+duckduckgo_web_search.name = "duckduckgo_web_search"
+
+# load tools from hub and langchain
+# image_generation_tool = load_tool("agents-course/text-to-image", trust_remote_code=True)
+image_generation_tool = load_tool("m-ric/text-to-image", trust_remote_code=True)  # Tool.from_space("black-forest-labs/FLUX.1-schnell", name="image_generator", description="Generate an image from a prompt")
+advanced_search_tool = Tool.from_langchain(load_tools(["searchapi"], allow_dangerous_tools=True)[0])  # serpapi is not real time scrapping
+advanced_search_tool.name = "advanced_search_tool"
+
+image_generation_tool_fast = Tool.from_space(
+    "black-forest-labs/FLUX.1-schnell",
+    name="image_generator",
+    description="Generate an image from a prompt"
+)
 
-# tools from Hub
-image_generation_tool = load_tool("agents-course/text-to-image", trust_remote_code=True)
 
+# alternative hf inference endpoint
 model = HfApiModel(
 max_tokens=2096,
 temperature=0.5,
-model_id='Qwen/Qwen2.5-Coder-32B-Instruct',# it is possible that this model may be overloaded
+model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud',
 custom_role_conversions=None,
 )
 
-# If the agent does not answer, the model is overloaded, please use another model or the following Hugging Face Endpoint that also contains qwen2.5 coder:
-# model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud' 
-
 with open("prompts.yaml", 'r') as stream:
     prompt_templates = yaml.safe_load(stream)
 
 tools = [
-        final_answer, 
-        prefered_web_search, 
-        alternative_web_search,
+        final_answer,
+        best_model_for_task,
+        advanced_search_tool,
+        google_web_search,
+        duckduckgo_web_search,
         visit_webpage, 
-        get_current_time_in_timezone, 
-        conversational_utterance, 
+        get_current_time_in_timezone,
+        advanced_image_generation,
         image_generation_tool,
         language_detection,
-        translation_tool
+        translation
     ]
 
 agent = CodeAgent(
     model=model,
     tools=tools,
-    max_steps=7,
-    verbosity_level=2,
+    max_steps=10,
+    verbosity_level=1,
     grammar=None,
     planning_interval=None,
     name=None,

requirements.txt

@@ -4,9 +4,18 @@ requests
 duckduckgo_search
 pandas
 transformers
-smolagents[transformers]
+# transformers[agents]
 torch
+langchain
+openai
+# accelerate
+langchain-community
+google-search-results
+smolagents[transformers]
+scikit-image
 pycountry
 opentelemetry-sdk
 opentelemetry-exporter-otlp
-openinference-instrumentation-smolagents
+openinference-instrumentation-smolagents
+python-dotenv
+langchain-openai

tools/best_model_for_task.py

@@ -0,0 +1,22 @@
+from smolagents import Tool
+
+class HFModelDownloadsTool(Tool):
+    name = "model_download_counter"
+    description = """
+    This is a tool that returns the most downloaded model of a given task on the Hugging Face Hub.
+    It returns the name of the checkpoint."""
+    inputs = {
+        "task": {
+            "type": "string",
+            "description": "the task category (such as text-classification, depth-estimation, etc)",
+        }
+    }
+    output_type = "string"
+
+    def forward(self, task: str):
+        from huggingface_hub import list_models
+
+        model = next(iter(list_models(filter=task, sort="downloads", direction=-1)))
+        return model.id
+
+model_downloads_tool = HFModelDownloadsTool()