added SVG for Groq model model providers (#1470)

#1432 #1447
This PR adds support for the GROQ LLM (Large Language Model).

Groq is an AI solutions company delivering ultra-low latency inference
with the first-ever LPU™ Inference Engine. The Groq API enables
developers to integrate state-of-the-art LLMs, such as Llama-2 and
llama3-70b-8192, into low latency applications with the request limits
specified below. Learn more at [groq.com](https://groq.com/).
Supported Models


| ID | Requests per Minute | Requests per Day | Tokens per Minute |

|----------------------|---------------------|------------------|-------------------|
| gemma-7b-it | 30 | 14,400 | 15,000 |
| gemma2-9b-it | 30 | 14,400 | 15,000 |
| llama3-70b-8192 | 30 | 14,400 | 6,000 |
| llama3-8b-8192 | 30 | 14,400 | 30,000 |
| mixtral-8x7b-32768 | 30 | 14,400 | 5,000 |

---------

Co-authored-by: paresh0628 <[email protected]>
Co-authored-by: Kevin Hu <[email protected]>

api/db/init_data.py

@@ -180,6 +180,12 @@ factory_infos = [{
     "logo": "",
     "tags": "LLM,TEXT EMBEDDING,IMAGE2TEXT",
     "status": "1",
+},
+{
+    "name": "Groq",
+    "logo": "",
+    "tags": "LLM",
+    "status": "1",
 }
     # {
     #     "name": "文心一言",
@@ -933,6 +939,47 @@ def init_llm_factory():
             "tags": "TEXT EMBEDDING",
             "max_tokens": 2048,
             "model_type": LLMType.EMBEDDING.value
+        },
+        # ------------------------ Groq -----------------------
+        {
+            "fid": factory_infos[18]["name"],
+            "llm_name": "gemma-7b-it",
+            "tags": "LLM,CHAT,15k",
+            "max_tokens": 8192,
+
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[18]["name"],
+            "llm_name": "gemma2-9b-it",
+            "tags": "LLM,CHAT,15k",
+            "max_tokens": 8192,
+
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[18]["name"],
+            "llm_name": "llama3-70b-8192",
+            "tags": "LLM,CHAT,6k",
+            "max_tokens": 8192,
+
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[18]["name"],
+            "llm_name": "llama3-8b-8192",
+            "tags": "LLM,CHAT,30k",
+            "max_tokens": 8192,
+
+            "model_type": LLMType.CHAT.value
+        },
+        {
+            "fid": factory_infos[18]["name"],
+            "llm_name": "mixtral-8x7b-32768",
+            "tags": "LLM,CHAT,5k",
+            "max_tokens": 32768,
+
+            "model_type": LLMType.CHAT.value
         }
     ]
     for info in factory_infos:

rag/llm/__init__.py

@@ -32,7 +32,8 @@ EmbeddingModel = {
     "Jina": JinaEmbed,
     "BAAI": DefaultEmbedding,
     "Mistral": MistralEmbed,
-    "Bedrock": BedrockEmbed
+    "Bedrock": BedrockEmbed,
+    "Groq": GroqChat
 }
 
 

rag/llm/chat_model.py

@@ -23,6 +23,7 @@ from ollama import Client
 from volcengine.maas.v2 import MaasService
 from rag.nlp import is_english
 from rag.utils import num_tokens_from_string
+from groq import Groq
 
 
 class Base(ABC):
@@ -681,4 +682,63 @@ class GeminiChat(Base):
         except Exception as e:
             yield ans + "\n**ERROR**: " + str(e)
 
-        yield  response._chunks[-1].usage_metadata.total_token_count
+        yield  response._chunks[-1].usage_metadata.total_token_count
+
+
+
+class GroqChat:
+    def __init__(self, key, model_name,base_url=''):
+        self.client = Groq(api_key=key)
+        self.model_name = model_name
+
+    def chat(self, system, history, gen_conf):
+        if system:
+            history.insert(0, {"role": "system", "content": system})
+        for k in list(gen_conf.keys()):
+            if k not in ["temperature", "top_p", "max_tokens"]:
+                del gen_conf[k]
+
+        ans = ""
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=history,
+                **gen_conf
+            )
+            ans = response.choices[0].message.content
+            if response.choices[0].finish_reason == "length":
+                ans += "...\nFor the content length reason, it stopped, continue?" if self.is_english(
+                    [ans]) else "······\n由于长度的原因，回答被截断了，要继续吗？"
+            return ans, response.usage.total_tokens
+        except Exception as e:
+            return ans + "\n**ERROR**: " + str(e), 0
+
+    def chat_streamly(self, system, history, gen_conf):
+        if system:
+            history.insert(0, {"role": "system", "content": system})
+        for k in list(gen_conf.keys()):
+            if k not in ["temperature", "top_p", "max_tokens"]:
+                del gen_conf[k]
+        ans = ""
+        total_tokens = 0
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=history,
+                stream=True,
+                **gen_conf
+            )
+            for resp in response:
+                if not resp.choices or not resp.choices[0].delta.content:
+                    continue
+                ans += resp.choices[0].delta.content
+                total_tokens += 1
+                if resp.choices[0].finish_reason == "length":
+                    ans += "...\nFor the content length reason, it stopped, continue?" if self.is_english(
+                        [ans]) else "······\n由于长度的原因，回答被截断了，要继续吗？"
+                yield ans
+
+        except Exception as e:
+            yield ans + "\n**ERROR**: " + str(e)
+
+        yield total_tokens

requirements.txt

@@ -147,4 +147,5 @@ markdown==3.6
 mistralai==0.4.2
 boto3==1.34.140
 duckduckgo_search==6.1.9
-google-generativeai==0.7.2
+google-generativeai==0.7.2
+groq==0.9.0

requirements_arm.txt

@@ -148,4 +148,5 @@ markdown==3.6
 mistralai==0.4.2
 boto3==1.34.140
 duckduckgo_search==6.1.9
-google-generativeai==0.7.2
+google-generativeai==0.7.2
+groq==0.9.0

requirements_dev.txt

@@ -133,4 +133,5 @@ markdown==3.6
 mistralai==0.4.2
 boto3==1.34.140
 duckduckgo_search==6.1.9
-google-generativeai==0.7.2
+google-generativeai==0.7.2
+groq==0.9.0

web/src/pages/user-setting/setting-model/index.tsx

@@ -62,6 +62,7 @@ const IconMap = {
   'Azure-OpenAI': 'azure',
   Bedrock: 'bedrock',
   Gemini:'gemini',
+  Groq: 'Groq',
 };
 
 const LlmIcon = ({ name }: { name: string }) => {