We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
BUG: chatglm hang


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `xinference/model/llm/chatglm.py`
Content:
```
1 # Copyright 2022-2023 XProbe Inc.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #      http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import logging
16 import time
17 import uuid
18 from pathlib import Path
19 from typing import TYPE_CHECKING, Iterator, List, Optional, TypedDict, Union
20 
21 from ...types import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
22 from .core import Model
23 
24 if TYPE_CHECKING:
25     from .. import ModelSpec
26 
27 logger = logging.getLogger(__name__)
28 
29 
30 class ChatglmCppModelConfig(TypedDict, total=False):
31     pass
32 
33 
34 class ChatglmCppGenerateConfig(TypedDict, total=False):
35     max_tokens: int
36     top_p: float
37     temperature: float
38     stream: bool
39 
40 
41 class ChatglmCppChatModel(Model):
42     def __init__(
43         self,
44         model_uid: str,
45         model_spec: "ModelSpec",
46         model_path: str,
47         model_config: Optional[ChatglmCppModelConfig] = None,
48     ):
49         super().__init__(model_uid, model_spec)
50         self._llm = None
51         self._model_path = model_path
52         if model_spec.model_name == "chatglm":
53             self.max_context_length = 2048
54         elif model_spec.model_name == "chatglm2":
55             self.max_context_length = 8192
56         else:
57             raise ValueError(
58                 f"Invalid model name '{model_spec.model_name}'. Expected chatglm or chatglm2."
59             )
60 
61         # just a placeholder for now as the chatglm_cpp repo doesn't support model config.
62         self._model_config = model_config
63 
64     @classmethod
65     def _sanitize_generate_config(
66         cls,
67         chatglmcpp_generate_config: Optional[ChatglmCppGenerateConfig],
68     ) -> ChatglmCppGenerateConfig:
69         if chatglmcpp_generate_config is None:
70             chatglmcpp_generate_config = ChatglmCppGenerateConfig()
71         chatglmcpp_generate_config.setdefault("max_tokens", 256)
72         chatglmcpp_generate_config.setdefault("temperature", 0.95)
73         chatglmcpp_generate_config.setdefault("top_p", 0.8)
74         chatglmcpp_generate_config.setdefault("stream", False)
75         return chatglmcpp_generate_config
76 
77     def load(self):
78         try:
79             import chatglm_cpp
80         except ImportError:
81             error_message = "Failed to import module 'chatglm_cpp'"
82             installation_guide = [
83                 "Please make sure 'chatglm_cpp' is installed. ",
84                 "You can install it by running the following command in the terminal:\n",
85                 "pip install git+https://github.com/li-plus/chatglm.cpp.git@main\n\n",
86                 "Or visit the original git repo if the above command fails:\n",
87                 "https://github.com/li-plus/chatglm.cpp",
88             ]
89 
90             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
91 
92         self._llm = chatglm_cpp.Pipeline(Path(self._model_path))
93 
94     @staticmethod
95     def _convert_raw_text_chunks_to_chat(
96         tokens: Iterator[str], model_name: str
97     ) -> Iterator[ChatCompletionChunk]:
98         yield {
99             "id": "chat" + f"cmpl-{str(uuid.uuid4())}",
100             "model": model_name,
101             "object": "chat.completion.chunk",
102             "created": int(time.time()),
103             "choices": [
104                 {
105                     "index": 0,
106                     "delta": {
107                         "role": "assistant",
108                     },
109                     "finish_reason": None,
110                 }
111             ],
112         }
113         for token in enumerate(tokens):
114             yield {
115                 "id": "chat" + f"cmpl-{str(uuid.uuid4())}",
116                 "model": model_name,
117                 "object": "chat.completion.chunk",
118                 "created": int(time.time()),
119                 "choices": [
120                     {
121                         "index": 0,
122                         "delta": {
123                             "content": token[1],
124                         },
125                         "finish_reason": None,
126                     }
127                 ],
128             }
129 
130     @staticmethod
131     def _convert_raw_text_completion_to_chat(
132         text: str, model_name: str
133     ) -> ChatCompletion:
134         return {
135             "id": "chat" + f"cmpl-{str(uuid.uuid4())}",
136             "model": model_name,
137             "object": "chat.completion",
138             "created": int(time.time()),
139             "choices": [
140                 {
141                     "index": 0,
142                     "message": {
143                         "role": "assistant",
144                         "content": text,
145                     },
146                     "finish_reason": None,
147                 }
148             ],
149             "usage": {
150                 "prompt_tokens": -1,
151                 "completion_tokens": -1,
152                 "total_tokens": -1,
153             },
154         }
155 
156     def chat(
157         self,
158         prompt: str,
159         chat_history: Optional[List[ChatCompletionMessage]] = None,
160         generate_config: Optional[ChatglmCppGenerateConfig] = None,
161     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
162         if chat_history is not None:
163             chat_history_list = [message["content"] for message in chat_history]
164         else:
165             chat_history_list = []
166 
167         chat_history_list.append(prompt)
168         logger.debug("Full conversation history:\n%s", str(chat_history_list))
169 
170         generate_config = self._sanitize_generate_config(generate_config)
171 
172         assert self._llm is not None
173 
174         if generate_config.get("stream", False):
175             it = self._llm.stream_chat(
176                 chat_history_list,
177                 max_context_length=self.max_context_length,
178                 max_length=generate_config["max_tokens"],
179                 temperature=generate_config["temperature"],
180                 top_p=generate_config["top_p"],
181             )
182             assert not isinstance(it, str)
183             return self._convert_raw_text_chunks_to_chat(it, self.model_uid)
184         else:
185             c = self._llm.chat(
186                 chat_history_list,
187                 max_context_length=self.max_context_length,
188                 max_length=generate_config["max_tokens"],
189                 temperature=generate_config["temperature"],
190                 top_p=generate_config["top_p"],
191             )
192             assert not isinstance(c, Iterator)
193             return self._convert_raw_text_completion_to_chat(c, self.model_uid)
194
```

Path: `examples/chat.py`
Content:
```
1 from typing import List
2 
3 from xinference.client import Client
4 from xinference.types import ChatCompletionMessage
5 
6 if __name__ == '__main__':
7     import argparse
8 
9     parser = argparse.ArgumentParser()
10 
11     parser.add_argument("--endpoint", type=str, help="Xinference endpoint, required")
12     parser.add_argument("--model_name", type=str, help="Name of the model, required")
13     parser.add_argument(
14         "--model_size_in_billions", type=int, required=False, help="Size of the model in billions", )
15     parser.add_argument("--model_format", type=str, required=False, help="Format of the model", )
16     parser.add_argument("--quantization", type=str, required=False, help="Quantization")
17 
18     args = parser.parse_args()
19 
20     endpoint = args.endpoint
21     model_name = args.model_name
22     model_size_in_billions = args.model_size_in_billions
23     model_format = args.model_format
24     quantization = args.quantization
25 
26     print(f"Xinference endpoint: {endpoint}")
27     print(f"Model Name: {model_name}")
28     print(f"Model Size (in billions): {model_size_in_billions}")
29     print(f"Model Format: {model_format}")
30     print(f"Quantization: {quantization}")
31 
32     client = Client(endpoint)
33     model_uid = client.launch_model(
34         model_name=model_name,
35         model_size_in_billions=model_size_in_billions,
36         model_format=model_format,
37         quantization=quantization,
38         n_ctx=2048
39     )
40     model = client.get_model(model_uid)
41 
42     chat_history: List["ChatCompletionMessage"] = []
43     while True:
44         prompt = input("you: ")
45         completion = model.chat(
46             prompt,
47             chat_history,
48             generate_config={"max_tokens": 1024}
49         )
50         content = completion["choices"][0]["message"]["content"]
51         print(f"{model_name}: {content}")
52         chat_history.append(
53             ChatCompletionMessage(role="user", content=prompt)
54         )
55         chat_history.append(
56             ChatCompletionMessage(role="assistant", content=content)
57         )
58
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/chat.py b/examples/chat.py
--- a/examples/chat.py
+++ b/examples/chat.py
@@ -30,13 +30,7 @@
     print(f"Quantization: {quantization}")
 
     client = Client(endpoint)
-    model_uid = client.launch_model(
-        model_name=model_name,
-        model_size_in_billions=model_size_in_billions,
-        model_format=model_format,
-        quantization=quantization,
-        n_ctx=2048
-    )
+    model_uid = client.launch_model(model_name, n_ctx=2048)
     model = client.get_model(model_uid)
 
     chat_history: List["ChatCompletionMessage"] = []
diff --git a/xinference/model/llm/chatglm.py b/xinference/model/llm/chatglm.py
--- a/xinference/model/llm/chatglm.py
+++ b/xinference/model/llm/chatglm.py
@@ -22,6 +22,8 @@
 from .core import Model
 
 if TYPE_CHECKING:
+    from chatglm_cpp import Pipeline
+
     from .. import ModelSpec
 
 logger = logging.getLogger(__name__)
@@ -47,16 +49,8 @@
         model_config: Optional[ChatglmCppModelConfig] = None,
     ):
         super().__init__(model_uid, model_spec)
-        self._llm = None
+        self._llm: Optional["Pipeline"] = None
         self._model_path = model_path
-        if model_spec.model_name == "chatglm":
-            self.max_context_length = 2048
-        elif model_spec.model_name == "chatglm2":
-            self.max_context_length = 8192
-        else:
-            raise ValueError(
-                f"Invalid model name '{model_spec.model_name}'. Expected chatglm or chatglm2."
-            )
 
         # just a placeholder for now as the chatglm_cpp repo doesn't support model config.
         self._model_config = model_config
@@ -68,9 +62,6 @@
     ) -> ChatglmCppGenerateConfig:
         if chatglmcpp_generate_config is None:
             chatglmcpp_generate_config = ChatglmCppGenerateConfig()
-        chatglmcpp_generate_config.setdefault("max_tokens", 256)
-        chatglmcpp_generate_config.setdefault("temperature", 0.95)
-        chatglmcpp_generate_config.setdefault("top_p", 0.8)
         chatglmcpp_generate_config.setdefault("stream", False)
         return chatglmcpp_generate_config
 
@@ -174,20 +165,12 @@
         if generate_config.get("stream", False):
             it = self._llm.stream_chat(
                 chat_history_list,
-                max_context_length=self.max_context_length,
-                max_length=generate_config["max_tokens"],
-                temperature=generate_config["temperature"],
-                top_p=generate_config["top_p"],
             )
             assert not isinstance(it, str)
             return self._convert_raw_text_chunks_to_chat(it, self.model_uid)
         else:
             c = self._llm.chat(
                 chat_history_list,
-                max_context_length=self.max_context_length,
-                max_length=generate_config["max_tokens"],
-                temperature=generate_config["temperature"],
-                top_p=generate_config["top_p"],
             )
             assert not isinstance(c, Iterator)
             return self._convert_raw_text_completion_to_chat(c, self.model_uid)

{"golden_diff": "diff --git a/examples/chat.py b/examples/chat.py\n--- a/examples/chat.py\n+++ b/examples/chat.py\n@@ -30,13 +30,7 @@\n     print(f\"Quantization: {quantization}\")\n \n     client = Client(endpoint)\n-    model_uid = client.launch_model(\n-        model_name=model_name,\n-        model_size_in_billions=model_size_in_billions,\n-        model_format=model_format,\n-        quantization=quantization,\n-        n_ctx=2048\n-    )\n+    model_uid = client.launch_model(model_name, n_ctx=2048)\n     model = client.get_model(model_uid)\n \n     chat_history: List[\"ChatCompletionMessage\"] = []\ndiff --git a/xinference/model/llm/chatglm.py b/xinference/model/llm/chatglm.py\n--- a/xinference/model/llm/chatglm.py\n+++ b/xinference/model/llm/chatglm.py\n@@ -22,6 +22,8 @@\n from .core import Model\n \n if TYPE_CHECKING:\n+    from chatglm_cpp import Pipeline\n+\n     from .. import ModelSpec\n \n logger = logging.getLogger(__name__)\n@@ -47,16 +49,8 @@\n         model_config: Optional[ChatglmCppModelConfig] = None,\n     ):\n         super().__init__(model_uid, model_spec)\n-        self._llm = None\n+        self._llm: Optional[\"Pipeline\"] = None\n         self._model_path = model_path\n-        if model_spec.model_name == \"chatglm\":\n-            self.max_context_length = 2048\n-        elif model_spec.model_name == \"chatglm2\":\n-            self.max_context_length = 8192\n-        else:\n-            raise ValueError(\n-                f\"Invalid model name '{model_spec.model_name}'. Expected chatglm or chatglm2.\"\n-            )\n \n         # just a placeholder for now as the chatglm_cpp repo doesn't support model config.\n         self._model_config = model_config\n@@ -68,9 +62,6 @@\n     ) -> ChatglmCppGenerateConfig:\n         if chatglmcpp_generate_config is None:\n             chatglmcpp_generate_config = ChatglmCppGenerateConfig()\n-        chatglmcpp_generate_config.setdefault(\"max_tokens\", 256)\n-        chatglmcpp_generate_config.setdefault(\"temperature\", 0.95)\n-        chatglmcpp_generate_config.setdefault(\"top_p\", 0.8)\n         chatglmcpp_generate_config.setdefault(\"stream\", False)\n         return chatglmcpp_generate_config\n \n@@ -174,20 +165,12 @@\n         if generate_config.get(\"stream\", False):\n             it = self._llm.stream_chat(\n                 chat_history_list,\n-                max_context_length=self.max_context_length,\n-                max_length=generate_config[\"max_tokens\"],\n-                temperature=generate_config[\"temperature\"],\n-                top_p=generate_config[\"top_p\"],\n             )\n             assert not isinstance(it, str)\n             return self._convert_raw_text_chunks_to_chat(it, self.model_uid)\n         else:\n             c = self._llm.chat(\n                 chat_history_list,\n-                max_context_length=self.max_context_length,\n-                max_length=generate_config[\"max_tokens\"],\n-                temperature=generate_config[\"temperature\"],\n-                top_p=generate_config[\"top_p\"],\n             )\n             assert not isinstance(c, Iterator)\n             return self._convert_raw_text_completion_to_chat(c, self.model_uid)\n", "issue": "BUG: chatglm hang\n\n", "before_files": [{"content": "# Copyright 2022-2023 XProbe Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport time\nimport uuid\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Iterator, List, Optional, TypedDict, Union\n\nfrom ...types import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage\nfrom .core import Model\n\nif TYPE_CHECKING:\n    from .. import ModelSpec\n\nlogger = logging.getLogger(__name__)\n\n\nclass ChatglmCppModelConfig(TypedDict, total=False):\n    pass\n\n\nclass ChatglmCppGenerateConfig(TypedDict, total=False):\n    max_tokens: int\n    top_p: float\n    temperature: float\n    stream: bool\n\n\nclass ChatglmCppChatModel(Model):\n    def __init__(\n        self,\n        model_uid: str,\n        model_spec: \"ModelSpec\",\n        model_path: str,\n        model_config: Optional[ChatglmCppModelConfig] = None,\n    ):\n        super().__init__(model_uid, model_spec)\n        self._llm = None\n        self._model_path = model_path\n        if model_spec.model_name == \"chatglm\":\n            self.max_context_length = 2048\n        elif model_spec.model_name == \"chatglm2\":\n            self.max_context_length = 8192\n        else:\n            raise ValueError(\n                f\"Invalid model name '{model_spec.model_name}'. Expected chatglm or chatglm2.\"\n            )\n\n        # just a placeholder for now as the chatglm_cpp repo doesn't support model config.\n        self._model_config = model_config\n\n    @classmethod\n    def _sanitize_generate_config(\n        cls,\n        chatglmcpp_generate_config: Optional[ChatglmCppGenerateConfig],\n    ) -> ChatglmCppGenerateConfig:\n        if chatglmcpp_generate_config is None:\n            chatglmcpp_generate_config = ChatglmCppGenerateConfig()\n        chatglmcpp_generate_config.setdefault(\"max_tokens\", 256)\n        chatglmcpp_generate_config.setdefault(\"temperature\", 0.95)\n        chatglmcpp_generate_config.setdefault(\"top_p\", 0.8)\n        chatglmcpp_generate_config.setdefault(\"stream\", False)\n        return chatglmcpp_generate_config\n\n    def load(self):\n        try:\n            import chatglm_cpp\n        except ImportError:\n            error_message = \"Failed to import module 'chatglm_cpp'\"\n            installation_guide = [\n                \"Please make sure 'chatglm_cpp' is installed. \",\n                \"You can install it by running the following command in the terminal:\\n\",\n                \"pip install git+https://github.com/li-plus/chatglm.cpp.git@main\\n\\n\",\n                \"Or visit the original git repo if the above command fails:\\n\",\n                \"https://github.com/li-plus/chatglm.cpp\",\n            ]\n\n            raise ImportError(f\"{error_message}\\n\\n{''.join(installation_guide)}\")\n\n        self._llm = chatglm_cpp.Pipeline(Path(self._model_path))\n\n    @staticmethod\n    def _convert_raw_text_chunks_to_chat(\n        tokens: Iterator[str], model_name: str\n    ) -> Iterator[ChatCompletionChunk]:\n        yield {\n            \"id\": \"chat\" + f\"cmpl-{str(uuid.uuid4())}\",\n            \"model\": model_name,\n            \"object\": \"chat.completion.chunk\",\n            \"created\": int(time.time()),\n            \"choices\": [\n                {\n                    \"index\": 0,\n                    \"delta\": {\n                        \"role\": \"assistant\",\n                    },\n                    \"finish_reason\": None,\n                }\n            ],\n        }\n        for token in enumerate(tokens):\n            yield {\n                \"id\": \"chat\" + f\"cmpl-{str(uuid.uuid4())}\",\n                \"model\": model_name,\n                \"object\": \"chat.completion.chunk\",\n                \"created\": int(time.time()),\n                \"choices\": [\n                    {\n                        \"index\": 0,\n                        \"delta\": {\n                            \"content\": token[1],\n                        },\n                        \"finish_reason\": None,\n                    }\n                ],\n            }\n\n    @staticmethod\n    def _convert_raw_text_completion_to_chat(\n        text: str, model_name: str\n    ) -> ChatCompletion:\n        return {\n            \"id\": \"chat\" + f\"cmpl-{str(uuid.uuid4())}\",\n            \"model\": model_name,\n            \"object\": \"chat.completion\",\n            \"created\": int(time.time()),\n            \"choices\": [\n                {\n                    \"index\": 0,\n                    \"message\": {\n                        \"role\": \"assistant\",\n                        \"content\": text,\n                    },\n                    \"finish_reason\": None,\n                }\n            ],\n            \"usage\": {\n                \"prompt_tokens\": -1,\n                \"completion_tokens\": -1,\n                \"total_tokens\": -1,\n            },\n        }\n\n    def chat(\n        self,\n        prompt: str,\n        chat_history: Optional[List[ChatCompletionMessage]] = None,\n        generate_config: Optional[ChatglmCppGenerateConfig] = None,\n    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:\n        if chat_history is not None:\n            chat_history_list = [message[\"content\"] for message in chat_history]\n        else:\n            chat_history_list = []\n\n        chat_history_list.append(prompt)\n        logger.debug(\"Full conversation history:\\n%s\", str(chat_history_list))\n\n        generate_config = self._sanitize_generate_config(generate_config)\n\n        assert self._llm is not None\n\n        if generate_config.get(\"stream\", False):\n            it = self._llm.stream_chat(\n                chat_history_list,\n                max_context_length=self.max_context_length,\n                max_length=generate_config[\"max_tokens\"],\n                temperature=generate_config[\"temperature\"],\n                top_p=generate_config[\"top_p\"],\n            )\n            assert not isinstance(it, str)\n            return self._convert_raw_text_chunks_to_chat(it, self.model_uid)\n        else:\n            c = self._llm.chat(\n                chat_history_list,\n                max_context_length=self.max_context_length,\n                max_length=generate_config[\"max_tokens\"],\n                temperature=generate_config[\"temperature\"],\n                top_p=generate_config[\"top_p\"],\n            )\n            assert not isinstance(c, Iterator)\n            return self._convert_raw_text_completion_to_chat(c, self.model_uid)\n", "path": "xinference/model/llm/chatglm.py"}, {"content": "from typing import List\n\nfrom xinference.client import Client\nfrom xinference.types import ChatCompletionMessage\n\nif __name__ == '__main__':\n    import argparse\n\n    parser = argparse.ArgumentParser()\n\n    parser.add_argument(\"--endpoint\", type=str, help=\"Xinference endpoint, required\")\n    parser.add_argument(\"--model_name\", type=str, help=\"Name of the model, required\")\n    parser.add_argument(\n        \"--model_size_in_billions\", type=int, required=False, help=\"Size of the model in billions\", )\n    parser.add_argument(\"--model_format\", type=str, required=False, help=\"Format of the model\", )\n    parser.add_argument(\"--quantization\", type=str, required=False, help=\"Quantization\")\n\n    args = parser.parse_args()\n\n    endpoint = args.endpoint\n    model_name = args.model_name\n    model_size_in_billions = args.model_size_in_billions\n    model_format = args.model_format\n    quantization = args.quantization\n\n    print(f\"Xinference endpoint: {endpoint}\")\n    print(f\"Model Name: {model_name}\")\n    print(f\"Model Size (in billions): {model_size_in_billions}\")\n    print(f\"Model Format: {model_format}\")\n    print(f\"Quantization: {quantization}\")\n\n    client = Client(endpoint)\n    model_uid = client.launch_model(\n        model_name=model_name,\n        model_size_in_billions=model_size_in_billions,\n        model_format=model_format,\n        quantization=quantization,\n        n_ctx=2048\n    )\n    model = client.get_model(model_uid)\n\n    chat_history: List[\"ChatCompletionMessage\"] = []\n    while True:\n        prompt = input(\"you: \")\n        completion = model.chat(\n            prompt,\n            chat_history,\n            generate_config={\"max_tokens\": 1024}\n        )\n        content = completion[\"choices\"][0][\"message\"][\"content\"]\n        print(f\"{model_name}: {content}\")\n        chat_history.append(\n            ChatCompletionMessage(role=\"user\", content=prompt)\n        )\n        chat_history.append(\n            ChatCompletionMessage(role=\"assistant\", content=content)\n        )\n", "path": "examples/chat.py"}], "after_files": [{"content": "# Copyright 2022-2023 XProbe Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport time\nimport uuid\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Iterator, List, Optional, TypedDict, Union\n\nfrom ...types import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage\nfrom .core import Model\n\nif TYPE_CHECKING:\n    from chatglm_cpp import Pipeline\n\n    from .. import ModelSpec\n\nlogger = logging.getLogger(__name__)\n\n\nclass ChatglmCppModelConfig(TypedDict, total=False):\n    pass\n\n\nclass ChatglmCppGenerateConfig(TypedDict, total=False):\n    max_tokens: int\n    top_p: float\n    temperature: float\n    stream: bool\n\n\nclass ChatglmCppChatModel(Model):\n    def __init__(\n        self,\n        model_uid: str,\n        model_spec: \"ModelSpec\",\n        model_path: str,\n        model_config: Optional[ChatglmCppModelConfig] = None,\n    ):\n        super().__init__(model_uid, model_spec)\n        self._llm: Optional[\"Pipeline\"] = None\n        self._model_path = model_path\n\n        # just a placeholder for now as the chatglm_cpp repo doesn't support model config.\n        self._model_config = model_config\n\n    @classmethod\n    def _sanitize_generate_config(\n        cls,\n        chatglmcpp_generate_config: Optional[ChatglmCppGenerateConfig],\n    ) -> ChatglmCppGenerateConfig:\n        if chatglmcpp_generate_config is None:\n            chatglmcpp_generate_config = ChatglmCppGenerateConfig()\n        chatglmcpp_generate_config.setdefault(\"stream\", False)\n        return chatglmcpp_generate_config\n\n    def load(self):\n        try:\n            import chatglm_cpp\n        except ImportError:\n            error_message = \"Failed to import module 'chatglm_cpp'\"\n            installation_guide = [\n                \"Please make sure 'chatglm_cpp' is installed. \",\n                \"You can install it by running the following command in the terminal:\\n\",\n                \"pip install git+https://github.com/li-plus/chatglm.cpp.git@main\\n\\n\",\n                \"Or visit the original git repo if the above command fails:\\n\",\n                \"https://github.com/li-plus/chatglm.cpp\",\n            ]\n\n            raise ImportError(f\"{error_message}\\n\\n{''.join(installation_guide)}\")\n\n        self._llm = chatglm_cpp.Pipeline(Path(self._model_path))\n\n    @staticmethod\n    def _convert_raw_text_chunks_to_chat(\n        tokens: Iterator[str], model_name: str\n    ) -> Iterator[ChatCompletionChunk]:\n        yield {\n            \"id\": \"chat\" + f\"cmpl-{str(uuid.uuid4())}\",\n            \"model\": model_name,\n            \"object\": \"chat.completion.chunk\",\n            \"created\": int(time.time()),\n            \"choices\": [\n                {\n                    \"index\": 0,\n                    \"delta\": {\n                        \"role\": \"assistant\",\n                    },\n                    \"finish_reason\": None,\n                }\n            ],\n        }\n        for token in enumerate(tokens):\n            yield {\n                \"id\": \"chat\" + f\"cmpl-{str(uuid.uuid4())}\",\n                \"model\": model_name,\n                \"object\": \"chat.completion.chunk\",\n                \"created\": int(time.time()),\n                \"choices\": [\n                    {\n                        \"index\": 0,\n                        \"delta\": {\n                            \"content\": token[1],\n                        },\n                        \"finish_reason\": None,\n                    }\n                ],\n            }\n\n    @staticmethod\n    def _convert_raw_text_completion_to_chat(\n        text: str, model_name: str\n    ) -> ChatCompletion:\n        return {\n            \"id\": \"chat\" + f\"cmpl-{str(uuid.uuid4())}\",\n            \"model\": model_name,\n            \"object\": \"chat.completion\",\n            \"created\": int(time.time()),\n            \"choices\": [\n                {\n                    \"index\": 0,\n                    \"message\": {\n                        \"role\": \"assistant\",\n                        \"content\": text,\n                    },\n                    \"finish_reason\": None,\n                }\n            ],\n            \"usage\": {\n                \"prompt_tokens\": -1,\n                \"completion_tokens\": -1,\n                \"total_tokens\": -1,\n            },\n        }\n\n    def chat(\n        self,\n        prompt: str,\n        chat_history: Optional[List[ChatCompletionMessage]] = None,\n        generate_config: Optional[ChatglmCppGenerateConfig] = None,\n    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:\n        if chat_history is not None:\n            chat_history_list = [message[\"content\"] for message in chat_history]\n        else:\n            chat_history_list = []\n\n        chat_history_list.append(prompt)\n        logger.debug(\"Full conversation history:\\n%s\", str(chat_history_list))\n\n        generate_config = self._sanitize_generate_config(generate_config)\n\n        assert self._llm is not None\n\n        if generate_config.get(\"stream\", False):\n            it = self._llm.stream_chat(\n                chat_history_list,\n            )\n            assert not isinstance(it, str)\n            return self._convert_raw_text_chunks_to_chat(it, self.model_uid)\n        else:\n            c = self._llm.chat(\n                chat_history_list,\n            )\n            assert not isinstance(c, Iterator)\n            return self._convert_raw_text_completion_to_chat(c, self.model_uid)\n", "path": "xinference/model/llm/chatglm.py"}, {"content": "from typing import List\n\nfrom xinference.client import Client\nfrom xinference.types import ChatCompletionMessage\n\nif __name__ == '__main__':\n    import argparse\n\n    parser = argparse.ArgumentParser()\n\n    parser.add_argument(\"--endpoint\", type=str, help=\"Xinference endpoint, required\")\n    parser.add_argument(\"--model_name\", type=str, help=\"Name of the model, required\")\n    parser.add_argument(\n        \"--model_size_in_billions\", type=int, required=False, help=\"Size of the model in billions\", )\n    parser.add_argument(\"--model_format\", type=str, required=False, help=\"Format of the model\", )\n    parser.add_argument(\"--quantization\", type=str, required=False, help=\"Quantization\")\n\n    args = parser.parse_args()\n\n    endpoint = args.endpoint\n    model_name = args.model_name\n    model_size_in_billions = args.model_size_in_billions\n    model_format = args.model_format\n    quantization = args.quantization\n\n    print(f\"Xinference endpoint: {endpoint}\")\n    print(f\"Model Name: {model_name}\")\n    print(f\"Model Size (in billions): {model_size_in_billions}\")\n    print(f\"Model Format: {model_format}\")\n    print(f\"Quantization: {quantization}\")\n\n    client = Client(endpoint)\n    model_uid = client.launch_model(model_name, n_ctx=2048)\n    model = client.get_model(model_uid)\n\n    chat_history: List[\"ChatCompletionMessage\"] = []\n    while True:\n        prompt = input(\"you: \")\n        completion = model.chat(\n            prompt,\n            chat_history,\n            generate_config={\"max_tokens\": 1024}\n        )\n        content = completion[\"choices\"][0][\"message\"][\"content\"]\n        print(f\"{model_name}: {content}\")\n        chat_history.append(\n            ChatCompletionMessage(role=\"user\", content=prompt)\n        )\n        chat_history.append(\n            ChatCompletionMessage(role=\"assistant\", content=content)\n        )\n", "path": "examples/chat.py"}]}