laverdes commited on
Commit
be9a1ee
·
1 Parent(s): f9fcb2a

feat: image_generation and best model for task tools

Browse files
Files changed (4) hide show
  1. .gitignore +0 -0
  2. app.py +74 -55
  3. requirements.txt +11 -2
  4. tools/best_model_for_task.py +22 -0
.gitignore ADDED
File without changes
app.py CHANGED
@@ -1,5 +1,4 @@
1
- import datetime
2
- import requests
3
  import pytz
4
  import yaml
5
  import pycountry
@@ -7,69 +6,59 @@ import pycountry
7
  from tools.final_answer import FinalAnswerTool
8
  from tools.visit_webpage import VisitWebpageTool
9
  from tools.translation import TranslationTool
 
10
 
11
  from transformers import pipeline
12
  from Gradio_UI import GradioUI
13
- from typing import Optional
14
 
15
  import os
16
  import base64
 
17
 
18
  from opentelemetry.sdk.trace import TracerProvider
19
  from openinference.instrumentation.smolagents import SmolagentsInstrumentor
20
  from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
21
  from opentelemetry.sdk.trace.export import SimpleSpanProcessor
22
 
 
 
 
 
 
 
 
 
 
23
  from smolagents import (
24
  CodeAgent,
25
  DuckDuckGoSearchTool,
26
  GoogleSearchTool,
27
  HfApiModel,
 
28
  load_tool,
29
- tool
 
 
30
  )
31
 
 
 
32
 
 
 
33
  @tool
34
  def get_current_time_in_timezone(timezone: str) -> str:
35
- """A tool that fetches the current local time in a specified timezone.
36
  Args:
37
  timezone (str): A string representing a valid timezone (e.g., 'America/New_York').
38
  """
39
  try:
40
- # Create timezone object
41
  tz = pytz.timezone(timezone)
42
- # Get current time in that timezone
43
- local_time = datetime.datetime.now(tz).strftime("%Y-%m-%d %H:%M:%S")
44
  return f"The current local time in {timezone} is: {local_time}"
45
  except Exception as e:
46
  return f"Error fetching time for timezone '{timezone}': {str(e)}"
47
 
48
- @tool
49
- def conversational_utterance(user_content: str, additional_context: Optional[str]="") -> str:
50
- """
51
- A tool that replies to a single casual query or message triggering any other tool is unfitted to reply.
52
-
53
- Args:
54
- user_content: A string with the user's message or query (e.g., "Hi!", "How are you?", "Tell me a joke").
55
- additional_context: An optional string with additional information (such as context, metadata, conversation history,
56
- or instructions) to be passed as an 'assistant' turn (a thought) in the conversation.
57
- """
58
- system_context_message = f"""
59
- You are a highly intelligent, expert, and witty assistant who responds to user conversational messages.
60
- You function as a tool activated by user intention via AI agents. In addition to your native LLM capabilities,
61
- you have access to the following system tools that the user may leverage:
62
- {tools}
63
- You should mention these tools whenever relevant during the conversation.
64
- """
65
-
66
- messages = [
67
- {"role": "system", "content": [{"type": "text", "text": system_context_message}]},
68
- {"role": "assistant", "content": [{"type": "text", "text": f"(additional_context: {additional_context})"}]},
69
- {"role": "user", "content": [{"type": "text", "text": user_content}]}
70
- ]
71
- return model(messages).content
72
-
73
 
74
  @tool
75
  def language_detection(text:str)-> str:
@@ -100,6 +89,25 @@ def language_detection(text:str)-> str:
100
  return "None"
101
 
102
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  def initialize_langfuse_opentelemetry_instrumentation():
104
  LANGFUSE_PUBLIC_KEY=os.environ.get("LANGFUSE_PUBLIC_KEY")
105
  LANGFUSE_SECRET_KEY=os.environ.get("LANGFUSE_SECRET_KEY")
@@ -115,50 +123,61 @@ def initialize_langfuse_opentelemetry_instrumentation():
115
 
116
  initialize_langfuse_opentelemetry_instrumentation()
117
 
118
- # tools from /tools/
119
  final_answer = FinalAnswerTool()
120
  visit_webpage = VisitWebpageTool()
121
- translation_tool = TranslationTool()
122
-
123
- # tools from smoloagents library
124
- prefered_web_search = GoogleSearchTool()
125
- prefered_web_search.name = "preferred_web_search"
126
- alternative_web_search = DuckDuckGoSearchTool()
127
- alternative_web_search.name = "alternative_web_search"
 
 
 
 
 
 
 
 
 
 
 
 
 
128
 
129
- # tools from Hub
130
- image_generation_tool = load_tool("agents-course/text-to-image", trust_remote_code=True)
131
 
 
132
  model = HfApiModel(
133
  max_tokens=2096,
134
  temperature=0.5,
135
- model_id='Qwen/Qwen2.5-Coder-32B-Instruct',# it is possible that this model may be overloaded
136
  custom_role_conversions=None,
137
  )
138
 
139
- # If the agent does not answer, the model is overloaded, please use another model or the following Hugging Face Endpoint that also contains qwen2.5 coder:
140
- # model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud'
141
-
142
  with open("prompts.yaml", 'r') as stream:
143
  prompt_templates = yaml.safe_load(stream)
144
 
145
  tools = [
146
- final_answer,
147
- prefered_web_search,
148
- alternative_web_search,
 
 
149
  visit_webpage,
150
- get_current_time_in_timezone,
151
- conversational_utterance,
152
  image_generation_tool,
153
  language_detection,
154
- translation_tool
155
  ]
156
 
157
  agent = CodeAgent(
158
  model=model,
159
  tools=tools,
160
- max_steps=7,
161
- verbosity_level=2,
162
  grammar=None,
163
  planning_interval=None,
164
  name=None,
 
1
+ from datetime import datetime
 
2
  import pytz
3
  import yaml
4
  import pycountry
 
6
  from tools.final_answer import FinalAnswerTool
7
  from tools.visit_webpage import VisitWebpageTool
8
  from tools.translation import TranslationTool
9
+ from tools.best_model_for_task import HFModelDownloadsTool
10
 
11
  from transformers import pipeline
12
  from Gradio_UI import GradioUI
 
13
 
14
  import os
15
  import base64
16
+ from dotenv import load_dotenv
17
 
18
  from opentelemetry.sdk.trace import TracerProvider
19
  from openinference.instrumentation.smolagents import SmolagentsInstrumentor
20
  from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
21
  from opentelemetry.sdk.trace.export import SimpleSpanProcessor
22
 
23
+ from langchain_community.agent_toolkits.load_tools import load_tools
24
+ from langchain.chains import LLMChain
25
+ from langchain_community.utilities.dalle_image_generator import DallEAPIWrapper
26
+ from langchain_core.prompts import PromptTemplate
27
+ from langchain_openai import OpenAI
28
+
29
+ from skimage import io
30
+ from PIL import Image
31
+
32
  from smolagents import (
33
  CodeAgent,
34
  DuckDuckGoSearchTool,
35
  GoogleSearchTool,
36
  HfApiModel,
37
+ TransformersModel,
38
  load_tool,
39
+ Tool,
40
+ tool,
41
+ ToolCollection
42
  )
43
 
44
+ # load .env vars
45
+ load_dotenv()
46
 
47
+
48
+ # fast prototyping tools
49
  @tool
50
  def get_current_time_in_timezone(timezone: str) -> str:
51
+ """A tool that fetches the current local time in a specified timezone formatted as '%m/%d/%y %H:%M:%S'
52
  Args:
53
  timezone (str): A string representing a valid timezone (e.g., 'America/New_York').
54
  """
55
  try:
 
56
  tz = pytz.timezone(timezone)
57
+ local_time = datetime.now(tz).strftime('%m/%d/%y %H:%M:%S')
 
58
  return f"The current local time in {timezone} is: {local_time}"
59
  except Exception as e:
60
  return f"Error fetching time for timezone '{timezone}': {str(e)}"
61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
 
63
  @tool
64
  def language_detection(text:str)-> str:
 
89
  return "None"
90
 
91
 
92
+ @tool
93
+ def advanced_image_generation(description:str)->Image.Image:
94
+ """Generates an image using a textual description.
95
+ Args:
96
+ description: the textual description provided by the user to prompt a text-to-image model
97
+ """
98
+ llm = OpenAI(temperature=0.9)
99
+ prompt = PromptTemplate(
100
+ input_variables=["image_desc"],
101
+ template="Generate a detailed but short prompt (must be less than 900 characters) to generate an image based on the following description: {image_desc}",
102
+ )
103
+ chain = LLMChain(llm=llm, prompt=prompt)
104
+ image_url = DallEAPIWrapper().run(chain.run(description))
105
+ image_array = io.imread(image_url)
106
+ pil_image = Image.fromarray(image_array)
107
+ return pil_image
108
+
109
+
110
+ # telemetry
111
  def initialize_langfuse_opentelemetry_instrumentation():
112
  LANGFUSE_PUBLIC_KEY=os.environ.get("LANGFUSE_PUBLIC_KEY")
113
  LANGFUSE_SECRET_KEY=os.environ.get("LANGFUSE_SECRET_KEY")
 
123
 
124
  initialize_langfuse_opentelemetry_instrumentation()
125
 
126
+ # load tools from /tools/
127
  final_answer = FinalAnswerTool()
128
  visit_webpage = VisitWebpageTool()
129
+ translation = TranslationTool()
130
+ best_model_for_task = HFModelDownloadsTool()
131
+
132
+ # load tools from smoloagents library
133
+ google_web_search = GoogleSearchTool()
134
+ google_web_search.name = "google_web_search"
135
+ duckduckgo_web_search = DuckDuckGoSearchTool()
136
+ duckduckgo_web_search.name = "duckduckgo_web_search"
137
+
138
+ # load tools from hub and langchain
139
+ # image_generation_tool = load_tool("agents-course/text-to-image", trust_remote_code=True)
140
+ image_generation_tool = load_tool("m-ric/text-to-image", trust_remote_code=True) # Tool.from_space("black-forest-labs/FLUX.1-schnell", name="image_generator", description="Generate an image from a prompt")
141
+ advanced_search_tool = Tool.from_langchain(load_tools(["searchapi"], allow_dangerous_tools=True)[0]) # serpapi is not real time scrapping
142
+ advanced_search_tool.name = "advanced_search_tool"
143
+
144
+ image_generation_tool_fast = Tool.from_space(
145
+ "black-forest-labs/FLUX.1-schnell",
146
+ name="image_generator",
147
+ description="Generate an image from a prompt"
148
+ )
149
 
 
 
150
 
151
+ # alternative hf inference endpoint
152
  model = HfApiModel(
153
  max_tokens=2096,
154
  temperature=0.5,
155
+ model_id='https://pflgm2locj2t89co.us-east-1.aws.endpoints.huggingface.cloud',
156
  custom_role_conversions=None,
157
  )
158
 
 
 
 
159
  with open("prompts.yaml", 'r') as stream:
160
  prompt_templates = yaml.safe_load(stream)
161
 
162
  tools = [
163
+ final_answer,
164
+ best_model_for_task,
165
+ advanced_search_tool,
166
+ google_web_search,
167
+ duckduckgo_web_search,
168
  visit_webpage,
169
+ get_current_time_in_timezone,
170
+ advanced_image_generation,
171
  image_generation_tool,
172
  language_detection,
173
+ translation
174
  ]
175
 
176
  agent = CodeAgent(
177
  model=model,
178
  tools=tools,
179
+ max_steps=10,
180
+ verbosity_level=1,
181
  grammar=None,
182
  planning_interval=None,
183
  name=None,
requirements.txt CHANGED
@@ -4,9 +4,18 @@ requests
4
  duckduckgo_search
5
  pandas
6
  transformers
7
- smolagents[transformers]
8
  torch
 
 
 
 
 
 
 
9
  pycountry
10
  opentelemetry-sdk
11
  opentelemetry-exporter-otlp
12
- openinference-instrumentation-smolagents
 
 
 
4
  duckduckgo_search
5
  pandas
6
  transformers
7
+ # transformers[agents]
8
  torch
9
+ langchain
10
+ openai
11
+ # accelerate
12
+ langchain-community
13
+ google-search-results
14
+ smolagents[transformers]
15
+ scikit-image
16
  pycountry
17
  opentelemetry-sdk
18
  opentelemetry-exporter-otlp
19
+ openinference-instrumentation-smolagents
20
+ python-dotenv
21
+ langchain-openai
tools/best_model_for_task.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from smolagents import Tool
2
+
3
+ class HFModelDownloadsTool(Tool):
4
+ name = "model_download_counter"
5
+ description = """
6
+ This is a tool that returns the most downloaded model of a given task on the Hugging Face Hub.
7
+ It returns the name of the checkpoint."""
8
+ inputs = {
9
+ "task": {
10
+ "type": "string",
11
+ "description": "the task category (such as text-classification, depth-estimation, etc)",
12
+ }
13
+ }
14
+ output_type = "string"
15
+
16
+ def forward(self, task: str):
17
+ from huggingface_hub import list_models
18
+
19
+ model = next(iter(list_models(filter=task, sort="downloads", direction=-1)))
20
+ return model.id
21
+
22
+ model_downloads_tool = HFModelDownloadsTool()