add supprot for lepton (#1866)

### What problem does this PR solve?

add supprot for lepton
#1853

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

Co-authored-by: Zhedong Cen <[email protected]>

conf/llm_factories.json

@@ -2326,6 +2326,104 @@
                     "model_type": "rerank"
                 }
             ]
+        },
+        {
+            "name": "Lepton",
+            "logo": "",
+            "tags": "LLM",
+            "status": "1",
+            "llm": [
+                {
+                    "llm_name": "dolphin-mixtral-8x7b",
+                    "tags": "LLM,CHAT,32k",
+                    "max_tokens": 32768,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "gemma-7b",
+                    "tags": "LLM,CHAT,8k",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "llama3-1-8b",
+                    "tags": "LLM,CHAT,4k",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "llama3-8b",
+                    "tags": "LLM,CHAT,8K",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "llama2-13b",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "llama3-1-70b",
+                    "tags": "LLM,CHAT,8k",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "llama3-70b",
+                    "tags": "LLM,CHAT,8k",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "llama3-1-405b",
+                    "tags": "LLM,CHAT,8k",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "mistral-7b",
+                    "tags": "LLM,CHAT,8K",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "mistral-8x7b",
+                    "tags": "LLM,CHAT,8K",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "nous-hermes-llama2",
+                    "tags": "LLM,CHAT,4k",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "openchat-3-5",
+                    "tags": "LLM,CHAT,4k",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "toppy-m-7b",
+                    "tags": "LLM,CHAT,4k",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "wizardlm-2-7b",
+                    "tags": "LLM,CHAT,32k",
+                    "max_tokens": 32768,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "wizardlm-2-8x22b",
+                    "tags": "LLM,CHAT,64K",
+                    "max_tokens": 65536,
+                    "model_type": "chat"
+                }
+            ]
         }
     ]
 }

rag/llm/__init__.py

@@ -83,7 +83,8 @@ ChatModel = {
     "NVIDIA": NvidiaChat,
     "LM-Studio": LmStudioChat,
     "OpenAI-API-Compatible": OpenAI_APIChat,
-    "cohere": CoHereChat
+    "cohere": CoHereChat,
+    "LeptonAI": LeptonAIChat
 }
 
 

rag/llm/chat_model.py

@@ -71,7 +71,7 @@ class Base(ABC):
                         total_tokens
                         + num_tokens_from_string(resp.choices[0].delta.content)
                     )
-                    if not hasattr(resp, "usage")
+                    if not hasattr(resp, "usage") or not resp.usage
                     else resp.usage["total_tokens"]
                 )
                 if resp.choices[0].finish_reason == "length":
@@ -981,3 +981,10 @@ class CoHereChat(Base):
             yield ans + "\n**ERROR**: " + str(e)
 
         yield total_tokens
+
+
+class LeptonAIChat(Base):
+    def __init__(self, key, model_name, base_url=None):
+        if not base_url:
+            base_url = os.path.join("https://"+model_name+".lepton.run","api","v1")
+        super().__init__(key, model_name, base_url)

web/src/pages/user-setting/setting-model/constant.ts

@@ -23,7 +23,8 @@ export const IconMap = {
   NVIDIA:'nvidia',
   'LM-Studio':'lm-studio',
   'OpenAI-API-Compatible':'openai-api',
-  'cohere':'cohere'
+  'cohere':'cohere',
+  'LeptonAI':'lepton'
 };
 
 export const BedrockRegionList = [