We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
n_gpu_layers参数希望可以开放
跑GUFF的时候n_gpu_layers参数挺重要的，现在看到B站一个伙计抓包后又请求的，希望能直接开放页面能修改此参数。

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `xinference/model/llm/ggml/llamacpp.py`
Content:
```
1 # Copyright 2022-2023 XProbe Inc.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #      http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 import datetime
15 import logging
16 import os
17 from typing import Iterable, Iterator, List, Optional, Union
18 
19 from ....types import (
20     ChatCompletion,
21     ChatCompletionChunk,
22     ChatCompletionMessage,
23     Completion,
24     CompletionChunk,
25     CreateCompletionLlamaCpp,
26     Embedding,
27     LlamaCppGenerateConfig,
28     LlamaCppModelConfig,
29 )
30 from ..core import LLM
31 from ..llm_family import LLMFamilyV1, LLMSpecV1
32 from ..utils import ChatModelMixin
33 from .ctransformers import CTRANSFORMERS_SUPPORTED_MODEL
34 
35 logger = logging.getLogger(__name__)
36 
37 
38 SIZE_TO_GPU_LAYERS = {
39     3: 26,
40     7: 32,
41     13: 40,
42     30: 60,
43     65: 80,
44 }
45 
46 
47 class LlamaCppModel(LLM):
48     def __init__(
49         self,
50         model_uid: str,
51         model_family: "LLMFamilyV1",
52         model_spec: "LLMSpecV1",
53         quantization: str,
54         model_path: str,
55         llamacpp_model_config: Optional[LlamaCppModelConfig] = None,
56     ):
57         super().__init__(model_uid, model_family, model_spec, quantization, model_path)
58 
59         closest_size = min(
60             SIZE_TO_GPU_LAYERS.keys(),
61             key=lambda x: abs(
62                 x - self.handle_model_size(model_spec.model_size_in_billions)
63             ),
64         )
65         self._gpu_layers = SIZE_TO_GPU_LAYERS[closest_size]
66         self._llamacpp_model_config: LlamaCppModelConfig = self._sanitize_model_config(
67             llamacpp_model_config
68         )
69         self._llm = None
70 
71     def _can_apply_metal(self):
72         return self.quantization.lower() in ["q4_0", "q4_1", "q4_k_s", "q4_k_m"]
73 
74     def _can_apply_cublas(self):
75         # TODO: figure out the quantizations supported.
76         return True
77 
78     def _sanitize_model_config(
79         self, llamacpp_model_config: Optional[LlamaCppModelConfig]
80     ) -> LlamaCppModelConfig:
81         if llamacpp_model_config is None:
82             llamacpp_model_config = LlamaCppModelConfig()
83 
84         if self.model_family.context_length:
85             llamacpp_model_config.setdefault("n_ctx", self.model_family.context_length)
86         llamacpp_model_config.setdefault("embedding", True)
87         llamacpp_model_config.setdefault("use_mmap", False)
88         llamacpp_model_config.setdefault("use_mlock", True)
89 
90         if (
91             "llama-2" in self.model_family.model_name
92             and self.model_spec.model_size_in_billions == 70
93         ):
94             llamacpp_model_config["use_mlock"] = False
95             llamacpp_model_config["n_gqa"] = 8
96 
97         if self._is_darwin_and_apple_silicon() and self._can_apply_metal():
98             # TODO: platform.processor() is not safe, need to be replaced to other method.
99             llamacpp_model_config.setdefault("n_gpu_layers", 1)
100         elif self._is_linux() and self._can_apply_cublas():
101             llamacpp_model_config.setdefault("n_gpu_layers", self._gpu_layers)
102 
103         return llamacpp_model_config
104 
105     def _sanitize_generate_config(
106         self, generate_config: Optional[LlamaCppGenerateConfig]
107     ) -> LlamaCppGenerateConfig:
108         if generate_config is None:
109             generate_config = LlamaCppGenerateConfig(
110                 **CreateCompletionLlamaCpp().dict()
111             )
112         else:
113             from llama_cpp import LlamaGrammar
114 
115             grammar = generate_config.get("grammar")
116             if grammar is not None and not isinstance(grammar, LlamaGrammar):
117                 generate_config["grammar"] = LlamaGrammar.from_string(
118                     generate_config["grammar"]
119                 )
120             # Validate generate_config and fill default values to the generate config.
121             generate_config = LlamaCppGenerateConfig(
122                 **CreateCompletionLlamaCpp(**generate_config).dict()
123             )
124         return generate_config
125 
126     def _convert_ggml_to_gguf(self, model_path: str) -> str:
127         from .tools import convert
128 
129         root_dir = os.path.dirname(os.path.dirname(model_path))
130         gguf_dir = os.path.join(
131             root_dir,
132             "{}-ggufv2-{}b".format(
133                 self.model_family.model_name, self.model_spec.model_size_in_billions
134             ),
135         )
136         os.makedirs(gguf_dir, exist_ok=True)
137         gguf_path = os.path.join(
138             gguf_dir,
139             "{}.{}.ggufv2".format(self.model_family.model_name, self.quantization),
140         )
141         # trick for validation, use a mark file to make sure the gguf file is converted
142         mark_file = os.path.join(gguf_dir, f"__valid_{self.quantization}")
143         if os.path.exists(mark_file):
144             return gguf_path
145         else:
146             logger.warning(
147                 "You are using a model with ggmlv3, "
148                 "and it will take some time to convert to ggufv2"
149             )
150             convert(model_path, gguf_path)
151             with open(mark_file, "w") as f:
152                 f.write(str(datetime.datetime.now()))
153             return gguf_path
154 
155     def load(self):
156         try:
157             import llama_cpp
158             from llama_cpp import Llama
159 
160             if llama_cpp.__version__ < "0.2.0":
161                 raise ValueError(
162                     "The llama_cpp version must be greater than 0.2.0. "
163                     "Please upgrade your version via `pip install -U llama_cpp` or refer to "
164                     "https://github.com/abetlen/llama-cpp-python#installation-with-openblas--cublas--clblast--metal."
165                 )
166         except ImportError:
167             error_message = "Failed to import module 'llama_cpp'"
168             installation_guide = [
169                 "Please make sure 'llama_cpp' is installed. ",
170                 "You can install it by visiting the installation section of the git repo:\n",
171                 "https://github.com/abetlen/llama-cpp-python#installation-with-openblas--cublas--clblast--metal",
172             ]
173 
174             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
175 
176         # handle legacy cache.
177         model_path = os.path.join(
178             self.model_path,
179             self.model_spec.model_file_name_template.format(
180                 quantization=self.quantization
181             ),
182         )
183         legacy_model_file_path = os.path.join(self.model_path, "model.bin")
184         if os.path.exists(legacy_model_file_path):
185             model_path = legacy_model_file_path
186 
187         if self.model_spec.model_format == "ggmlv3":
188             model_path = self._convert_ggml_to_gguf(model_path)
189 
190         try:
191             self._llm = Llama(
192                 model_path=model_path,
193                 verbose=True,
194                 **self._llamacpp_model_config,
195             )
196         except AssertionError:
197             raise RuntimeError(f"Load model {self.model_family.model_name} failed")
198 
199     @classmethod
200     def match(
201         cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str
202     ) -> bool:
203         if llm_spec.model_format not in ["ggmlv3", "ggufv2"]:
204             return False
205         if (
206             "chatglm" in llm_family.model_name
207             or "qwen" in llm_family.model_name
208             or llm_family.model_name in CTRANSFORMERS_SUPPORTED_MODEL
209         ):
210             return False
211         if "generate" not in llm_family.model_ability:
212             return False
213         return True
214 
215     def generate(
216         self, prompt: str, generate_config: Optional[LlamaCppGenerateConfig] = None
217     ) -> Union[Completion, Iterator[CompletionChunk]]:
218         def generator_wrapper(
219             _prompt: str,
220             _generate_config: LlamaCppGenerateConfig,
221         ) -> Iterator[CompletionChunk]:
222             assert self._llm is not None
223             for _completion_chunk in self._llm(prompt=_prompt, **_generate_config):
224                 yield _completion_chunk
225 
226         logger.debug(
227             "Enter generate, prompt: %s, generate config: %s", prompt, generate_config
228         )
229 
230         generate_config = self._sanitize_generate_config(generate_config)
231 
232         stream = generate_config.get("stream", False)
233 
234         if not stream:
235             assert self._llm is not None
236             completion = self._llm(prompt=prompt, **generate_config)
237 
238             return completion
239         else:
240             return generator_wrapper(prompt, generate_config)
241 
242     def create_embedding(self, input: Union[str, List[str]]) -> Embedding:
243         assert self._llm is not None
244         embedding = self._llm.create_embedding(input)
245         return embedding
246 
247 
248 class LlamaCppChatModel(LlamaCppModel, ChatModelMixin):
249     def __init__(
250         self,
251         model_uid: str,
252         model_family: "LLMFamilyV1",
253         model_spec: "LLMSpecV1",
254         quantization: str,
255         model_path: str,
256         llamacpp_model_config: Optional[LlamaCppModelConfig] = None,
257     ):
258         super().__init__(
259             model_uid,
260             model_family,
261             model_spec,
262             quantization,
263             model_path,
264             llamacpp_model_config,
265         )
266 
267     @classmethod
268     def match(
269         cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str
270     ) -> bool:
271         if llm_spec.model_format not in ["ggmlv3", "ggufv2"]:
272             return False
273         if (
274             "chatglm" in llm_family.model_name
275             or llm_family.model_name in CTRANSFORMERS_SUPPORTED_MODEL
276         ):
277             return False
278         if "chat" not in llm_family.model_ability:
279             return False
280         return True
281 
282     def _sanitize_generate_config(
283         self, generate_config: Optional[LlamaCppGenerateConfig]
284     ) -> LlamaCppGenerateConfig:
285         generate_config = super()._sanitize_generate_config(generate_config)
286         if self.model_family.prompt_style and self.model_family.prompt_style.stop:
287             generate_config["stop"] = self.model_family.prompt_style.stop
288         return generate_config
289 
290     def chat(
291         self,
292         prompt: str,
293         system_prompt: Optional[str] = None,
294         chat_history: Optional[List[ChatCompletionMessage]] = None,
295         generate_config: Optional[LlamaCppGenerateConfig] = None,
296     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
297         assert self.model_family.prompt_style is not None
298         prompt_style = self.model_family.prompt_style.copy()
299         if system_prompt:
300             prompt_style.system_prompt = system_prompt
301 
302         chat_history = chat_history or []
303         assert prompt_style is not None
304         tools = generate_config.pop("tools", []) if generate_config else None
305         full_prompt = self.get_prompt(prompt, chat_history, prompt_style, tools=tools)
306 
307         generate_config = self._sanitize_generate_config(generate_config)
308         # TODO(codingl2k1): qwen hacky to set stop for function call.
309         model_family = self.model_family.model_family or self.model_family.model_name
310         if tools and model_family in ["qwen-chat", "qwen1.5-chat"]:
311             stop = generate_config.get("stop")
312             if isinstance(stop, str):
313                 generate_config["stop"] = [stop, "Observation:"]
314             elif isinstance(stop, Iterable):
315                 assert not isinstance(stop, str)
316                 generate_config["stop"] = stop + ["Observation:"]
317             else:
318                 generate_config["stop"] = "Observation:"
319 
320         stream = generate_config.get("stream", False)
321         if stream:
322             it = self.generate(full_prompt, generate_config)
323             assert isinstance(it, Iterator)
324             return self._to_chat_completion_chunks(it)
325         else:
326             c = self.generate(full_prompt, generate_config)
327             assert not isinstance(c, Iterator)
328             if tools:
329                 return self._tool_calls_completion(
330                     self.model_family, self.model_uid, c, tools
331                 )
332             return self._to_chat_completion(c)
333
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/xinference/model/llm/ggml/llamacpp.py b/xinference/model/llm/ggml/llamacpp.py
--- a/xinference/model/llm/ggml/llamacpp.py
+++ b/xinference/model/llm/ggml/llamacpp.py
@@ -35,15 +35,6 @@
 logger = logging.getLogger(__name__)
 
 
-SIZE_TO_GPU_LAYERS = {
-    3: 26,
-    7: 32,
-    13: 40,
-    30: 60,
-    65: 80,
-}
-
-
 class LlamaCppModel(LLM):
     def __init__(
         self,
@@ -56,13 +47,6 @@
     ):
         super().__init__(model_uid, model_family, model_spec, quantization, model_path)
 
-        closest_size = min(
-            SIZE_TO_GPU_LAYERS.keys(),
-            key=lambda x: abs(
-                x - self.handle_model_size(model_spec.model_size_in_billions)
-            ),
-        )
-        self._gpu_layers = SIZE_TO_GPU_LAYERS[closest_size]
         self._llamacpp_model_config: LlamaCppModelConfig = self._sanitize_model_config(
             llamacpp_model_config
         )
@@ -96,9 +80,9 @@
 
         if self._is_darwin_and_apple_silicon() and self._can_apply_metal():
             # TODO: platform.processor() is not safe, need to be replaced to other method.
-            llamacpp_model_config.setdefault("n_gpu_layers", 1)
+            llamacpp_model_config.setdefault("n_gpu_layers", -1)
         elif self._is_linux() and self._can_apply_cublas():
-            llamacpp_model_config.setdefault("n_gpu_layers", self._gpu_layers)
+            llamacpp_model_config.setdefault("n_gpu_layers", -1)
 
         return llamacpp_model_config
 
@@ -313,7 +297,7 @@
                 generate_config["stop"] = [stop, "Observation:"]
             elif isinstance(stop, Iterable):
                 assert not isinstance(stop, str)
-                generate_config["stop"] = stop + ["Observation:"]
+                generate_config["stop"] = stop + ["Observation:"]  # type: ignore
             else:
                 generate_config["stop"] = "Observation:"

{"golden_diff": "diff --git a/xinference/model/llm/ggml/llamacpp.py b/xinference/model/llm/ggml/llamacpp.py\n--- a/xinference/model/llm/ggml/llamacpp.py\n+++ b/xinference/model/llm/ggml/llamacpp.py\n@@ -35,15 +35,6 @@\n logger = logging.getLogger(__name__)\n \n \n-SIZE_TO_GPU_LAYERS = {\n-    3: 26,\n-    7: 32,\n-    13: 40,\n-    30: 60,\n-    65: 80,\n-}\n-\n-\n class LlamaCppModel(LLM):\n     def __init__(\n         self,\n@@ -56,13 +47,6 @@\n     ):\n         super().__init__(model_uid, model_family, model_spec, quantization, model_path)\n \n-        closest_size = min(\n-            SIZE_TO_GPU_LAYERS.keys(),\n-            key=lambda x: abs(\n-                x - self.handle_model_size(model_spec.model_size_in_billions)\n-            ),\n-        )\n-        self._gpu_layers = SIZE_TO_GPU_LAYERS[closest_size]\n         self._llamacpp_model_config: LlamaCppModelConfig = self._sanitize_model_config(\n             llamacpp_model_config\n         )\n@@ -96,9 +80,9 @@\n \n         if self._is_darwin_and_apple_silicon() and self._can_apply_metal():\n             # TODO: platform.processor() is not safe, need to be replaced to other method.\n-            llamacpp_model_config.setdefault(\"n_gpu_layers\", 1)\n+            llamacpp_model_config.setdefault(\"n_gpu_layers\", -1)\n         elif self._is_linux() and self._can_apply_cublas():\n-            llamacpp_model_config.setdefault(\"n_gpu_layers\", self._gpu_layers)\n+            llamacpp_model_config.setdefault(\"n_gpu_layers\", -1)\n \n         return llamacpp_model_config\n \n@@ -313,7 +297,7 @@\n                 generate_config[\"stop\"] = [stop, \"Observation:\"]\n             elif isinstance(stop, Iterable):\n                 assert not isinstance(stop, str)\n-                generate_config[\"stop\"] = stop + [\"Observation:\"]\n+                generate_config[\"stop\"] = stop + [\"Observation:\"]  # type: ignore\n             else:\n                 generate_config[\"stop\"] = \"Observation:\"\n", "issue": "n_gpu_layers\u53c2\u6570\u5e0c\u671b\u53ef\u4ee5\u5f00\u653e\n\u8dd1GUFF\u7684\u65f6\u5019n_gpu_layers\u53c2\u6570\u633a\u91cd\u8981\u7684\uff0c\u73b0\u5728\u770b\u5230B\u7ad9\u4e00\u4e2a\u4f19\u8ba1\u6293\u5305\u540e\u53c8\u8bf7\u6c42\u7684\uff0c\u5e0c\u671b\u80fd\u76f4\u63a5\u5f00\u653e\u9875\u9762\u80fd\u4fee\u6539\u6b64\u53c2\u6570\u3002\n", "before_files": [{"content": "# Copyright 2022-2023 XProbe Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport datetime\nimport logging\nimport os\nfrom typing import Iterable, Iterator, List, Optional, Union\n\nfrom ....types import (\n    ChatCompletion,\n    ChatCompletionChunk,\n    ChatCompletionMessage,\n    Completion,\n    CompletionChunk,\n    CreateCompletionLlamaCpp,\n    Embedding,\n    LlamaCppGenerateConfig,\n    LlamaCppModelConfig,\n)\nfrom ..core import LLM\nfrom ..llm_family import LLMFamilyV1, LLMSpecV1\nfrom ..utils import ChatModelMixin\nfrom .ctransformers import CTRANSFORMERS_SUPPORTED_MODEL\n\nlogger = logging.getLogger(__name__)\n\n\nSIZE_TO_GPU_LAYERS = {\n    3: 26,\n    7: 32,\n    13: 40,\n    30: 60,\n    65: 80,\n}\n\n\nclass LlamaCppModel(LLM):\n    def __init__(\n        self,\n        model_uid: str,\n        model_family: \"LLMFamilyV1\",\n        model_spec: \"LLMSpecV1\",\n        quantization: str,\n        model_path: str,\n        llamacpp_model_config: Optional[LlamaCppModelConfig] = None,\n    ):\n        super().__init__(model_uid, model_family, model_spec, quantization, model_path)\n\n        closest_size = min(\n            SIZE_TO_GPU_LAYERS.keys(),\n            key=lambda x: abs(\n                x - self.handle_model_size(model_spec.model_size_in_billions)\n            ),\n        )\n        self._gpu_layers = SIZE_TO_GPU_LAYERS[closest_size]\n        self._llamacpp_model_config: LlamaCppModelConfig = self._sanitize_model_config(\n            llamacpp_model_config\n        )\n        self._llm = None\n\n    def _can_apply_metal(self):\n        return self.quantization.lower() in [\"q4_0\", \"q4_1\", \"q4_k_s\", \"q4_k_m\"]\n\n    def _can_apply_cublas(self):\n        # TODO: figure out the quantizations supported.\n        return True\n\n    def _sanitize_model_config(\n        self, llamacpp_model_config: Optional[LlamaCppModelConfig]\n    ) -> LlamaCppModelConfig:\n        if llamacpp_model_config is None:\n            llamacpp_model_config = LlamaCppModelConfig()\n\n        if self.model_family.context_length:\n            llamacpp_model_config.setdefault(\"n_ctx\", self.model_family.context_length)\n        llamacpp_model_config.setdefault(\"embedding\", True)\n        llamacpp_model_config.setdefault(\"use_mmap\", False)\n        llamacpp_model_config.setdefault(\"use_mlock\", True)\n\n        if (\n            \"llama-2\" in self.model_family.model_name\n            and self.model_spec.model_size_in_billions == 70\n        ):\n            llamacpp_model_config[\"use_mlock\"] = False\n            llamacpp_model_config[\"n_gqa\"] = 8\n\n        if self._is_darwin_and_apple_silicon() and self._can_apply_metal():\n            # TODO: platform.processor() is not safe, need to be replaced to other method.\n            llamacpp_model_config.setdefault(\"n_gpu_layers\", 1)\n        elif self._is_linux() and self._can_apply_cublas():\n            llamacpp_model_config.setdefault(\"n_gpu_layers\", self._gpu_layers)\n\n        return llamacpp_model_config\n\n    def _sanitize_generate_config(\n        self, generate_config: Optional[LlamaCppGenerateConfig]\n    ) -> LlamaCppGenerateConfig:\n        if generate_config is None:\n            generate_config = LlamaCppGenerateConfig(\n                **CreateCompletionLlamaCpp().dict()\n            )\n        else:\n            from llama_cpp import LlamaGrammar\n\n            grammar = generate_config.get(\"grammar\")\n            if grammar is not None and not isinstance(grammar, LlamaGrammar):\n                generate_config[\"grammar\"] = LlamaGrammar.from_string(\n                    generate_config[\"grammar\"]\n                )\n            # Validate generate_config and fill default values to the generate config.\n            generate_config = LlamaCppGenerateConfig(\n                **CreateCompletionLlamaCpp(**generate_config).dict()\n            )\n        return generate_config\n\n    def _convert_ggml_to_gguf(self, model_path: str) -> str:\n        from .tools import convert\n\n        root_dir = os.path.dirname(os.path.dirname(model_path))\n        gguf_dir = os.path.join(\n            root_dir,\n            \"{}-ggufv2-{}b\".format(\n                self.model_family.model_name, self.model_spec.model_size_in_billions\n            ),\n        )\n        os.makedirs(gguf_dir, exist_ok=True)\n        gguf_path = os.path.join(\n            gguf_dir,\n            \"{}.{}.ggufv2\".format(self.model_family.model_name, self.quantization),\n        )\n        # trick for validation, use a mark file to make sure the gguf file is converted\n        mark_file = os.path.join(gguf_dir, f\"__valid_{self.quantization}\")\n        if os.path.exists(mark_file):\n            return gguf_path\n        else:\n            logger.warning(\n                \"You are using a model with ggmlv3, \"\n                \"and it will take some time to convert to ggufv2\"\n            )\n            convert(model_path, gguf_path)\n            with open(mark_file, \"w\") as f:\n                f.write(str(datetime.datetime.now()))\n            return gguf_path\n\n    def load(self):\n        try:\n            import llama_cpp\n            from llama_cpp import Llama\n\n            if llama_cpp.__version__ < \"0.2.0\":\n                raise ValueError(\n                    \"The llama_cpp version must be greater than 0.2.0. \"\n                    \"Please upgrade your version via `pip install -U llama_cpp` or refer to \"\n                    \"https://github.com/abetlen/llama-cpp-python#installation-with-openblas--cublas--clblast--metal.\"\n                )\n        except ImportError:\n            error_message = \"Failed to import module 'llama_cpp'\"\n            installation_guide = [\n                \"Please make sure 'llama_cpp' is installed. \",\n                \"You can install it by visiting the installation section of the git repo:\\n\",\n                \"https://github.com/abetlen/llama-cpp-python#installation-with-openblas--cublas--clblast--metal\",\n            ]\n\n            raise ImportError(f\"{error_message}\\n\\n{''.join(installation_guide)}\")\n\n        # handle legacy cache.\n        model_path = os.path.join(\n            self.model_path,\n            self.model_spec.model_file_name_template.format(\n                quantization=self.quantization\n            ),\n        )\n        legacy_model_file_path = os.path.join(self.model_path, \"model.bin\")\n        if os.path.exists(legacy_model_file_path):\n            model_path = legacy_model_file_path\n\n        if self.model_spec.model_format == \"ggmlv3\":\n            model_path = self._convert_ggml_to_gguf(model_path)\n\n        try:\n            self._llm = Llama(\n                model_path=model_path,\n                verbose=True,\n                **self._llamacpp_model_config,\n            )\n        except AssertionError:\n            raise RuntimeError(f\"Load model {self.model_family.model_name} failed\")\n\n    @classmethod\n    def match(\n        cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str\n    ) -> bool:\n        if llm_spec.model_format not in [\"ggmlv3\", \"ggufv2\"]:\n            return False\n        if (\n            \"chatglm\" in llm_family.model_name\n            or \"qwen\" in llm_family.model_name\n            or llm_family.model_name in CTRANSFORMERS_SUPPORTED_MODEL\n        ):\n            return False\n        if \"generate\" not in llm_family.model_ability:\n            return False\n        return True\n\n    def generate(\n        self, prompt: str, generate_config: Optional[LlamaCppGenerateConfig] = None\n    ) -> Union[Completion, Iterator[CompletionChunk]]:\n        def generator_wrapper(\n            _prompt: str,\n            _generate_config: LlamaCppGenerateConfig,\n        ) -> Iterator[CompletionChunk]:\n            assert self._llm is not None\n            for _completion_chunk in self._llm(prompt=_prompt, **_generate_config):\n                yield _completion_chunk\n\n        logger.debug(\n            \"Enter generate, prompt: %s, generate config: %s\", prompt, generate_config\n        )\n\n        generate_config = self._sanitize_generate_config(generate_config)\n\n        stream = generate_config.get(\"stream\", False)\n\n        if not stream:\n            assert self._llm is not None\n            completion = self._llm(prompt=prompt, **generate_config)\n\n            return completion\n        else:\n            return generator_wrapper(prompt, generate_config)\n\n    def create_embedding(self, input: Union[str, List[str]]) -> Embedding:\n        assert self._llm is not None\n        embedding = self._llm.create_embedding(input)\n        return embedding\n\n\nclass LlamaCppChatModel(LlamaCppModel, ChatModelMixin):\n    def __init__(\n        self,\n        model_uid: str,\n        model_family: \"LLMFamilyV1\",\n        model_spec: \"LLMSpecV1\",\n        quantization: str,\n        model_path: str,\n        llamacpp_model_config: Optional[LlamaCppModelConfig] = None,\n    ):\n        super().__init__(\n            model_uid,\n            model_family,\n            model_spec,\n            quantization,\n            model_path,\n            llamacpp_model_config,\n        )\n\n    @classmethod\n    def match(\n        cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str\n    ) -> bool:\n        if llm_spec.model_format not in [\"ggmlv3\", \"ggufv2\"]:\n            return False\n        if (\n            \"chatglm\" in llm_family.model_name\n            or llm_family.model_name in CTRANSFORMERS_SUPPORTED_MODEL\n        ):\n            return False\n        if \"chat\" not in llm_family.model_ability:\n            return False\n        return True\n\n    def _sanitize_generate_config(\n        self, generate_config: Optional[LlamaCppGenerateConfig]\n    ) -> LlamaCppGenerateConfig:\n        generate_config = super()._sanitize_generate_config(generate_config)\n        if self.model_family.prompt_style and self.model_family.prompt_style.stop:\n            generate_config[\"stop\"] = self.model_family.prompt_style.stop\n        return generate_config\n\n    def chat(\n        self,\n        prompt: str,\n        system_prompt: Optional[str] = None,\n        chat_history: Optional[List[ChatCompletionMessage]] = None,\n        generate_config: Optional[LlamaCppGenerateConfig] = None,\n    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:\n        assert self.model_family.prompt_style is not None\n        prompt_style = self.model_family.prompt_style.copy()\n        if system_prompt:\n            prompt_style.system_prompt = system_prompt\n\n        chat_history = chat_history or []\n        assert prompt_style is not None\n        tools = generate_config.pop(\"tools\", []) if generate_config else None\n        full_prompt = self.get_prompt(prompt, chat_history, prompt_style, tools=tools)\n\n        generate_config = self._sanitize_generate_config(generate_config)\n        # TODO(codingl2k1): qwen hacky to set stop for function call.\n        model_family = self.model_family.model_family or self.model_family.model_name\n        if tools and model_family in [\"qwen-chat\", \"qwen1.5-chat\"]:\n            stop = generate_config.get(\"stop\")\n            if isinstance(stop, str):\n                generate_config[\"stop\"] = [stop, \"Observation:\"]\n            elif isinstance(stop, Iterable):\n                assert not isinstance(stop, str)\n                generate_config[\"stop\"] = stop + [\"Observation:\"]\n            else:\n                generate_config[\"stop\"] = \"Observation:\"\n\n        stream = generate_config.get(\"stream\", False)\n        if stream:\n            it = self.generate(full_prompt, generate_config)\n            assert isinstance(it, Iterator)\n            return self._to_chat_completion_chunks(it)\n        else:\n            c = self.generate(full_prompt, generate_config)\n            assert not isinstance(c, Iterator)\n            if tools:\n                return self._tool_calls_completion(\n                    self.model_family, self.model_uid, c, tools\n                )\n            return self._to_chat_completion(c)\n", "path": "xinference/model/llm/ggml/llamacpp.py"}], "after_files": [{"content": "# Copyright 2022-2023 XProbe Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport datetime\nimport logging\nimport os\nfrom typing import Iterable, Iterator, List, Optional, Union\n\nfrom ....types import (\n    ChatCompletion,\n    ChatCompletionChunk,\n    ChatCompletionMessage,\n    Completion,\n    CompletionChunk,\n    CreateCompletionLlamaCpp,\n    Embedding,\n    LlamaCppGenerateConfig,\n    LlamaCppModelConfig,\n)\nfrom ..core import LLM\nfrom ..llm_family import LLMFamilyV1, LLMSpecV1\nfrom ..utils import ChatModelMixin\nfrom .ctransformers import CTRANSFORMERS_SUPPORTED_MODEL\n\nlogger = logging.getLogger(__name__)\n\n\nclass LlamaCppModel(LLM):\n    def __init__(\n        self,\n        model_uid: str,\n        model_family: \"LLMFamilyV1\",\n        model_spec: \"LLMSpecV1\",\n        quantization: str,\n        model_path: str,\n        llamacpp_model_config: Optional[LlamaCppModelConfig] = None,\n    ):\n        super().__init__(model_uid, model_family, model_spec, quantization, model_path)\n\n        self._llamacpp_model_config: LlamaCppModelConfig = self._sanitize_model_config(\n            llamacpp_model_config\n        )\n        self._llm = None\n\n    def _can_apply_metal(self):\n        return self.quantization.lower() in [\"q4_0\", \"q4_1\", \"q4_k_s\", \"q4_k_m\"]\n\n    def _can_apply_cublas(self):\n        # TODO: figure out the quantizations supported.\n        return True\n\n    def _sanitize_model_config(\n        self, llamacpp_model_config: Optional[LlamaCppModelConfig]\n    ) -> LlamaCppModelConfig:\n        if llamacpp_model_config is None:\n            llamacpp_model_config = LlamaCppModelConfig()\n\n        if self.model_family.context_length:\n            llamacpp_model_config.setdefault(\"n_ctx\", self.model_family.context_length)\n        llamacpp_model_config.setdefault(\"embedding\", True)\n        llamacpp_model_config.setdefault(\"use_mmap\", False)\n        llamacpp_model_config.setdefault(\"use_mlock\", True)\n\n        if (\n            \"llama-2\" in self.model_family.model_name\n            and self.model_spec.model_size_in_billions == 70\n        ):\n            llamacpp_model_config[\"use_mlock\"] = False\n            llamacpp_model_config[\"n_gqa\"] = 8\n\n        if self._is_darwin_and_apple_silicon() and self._can_apply_metal():\n            # TODO: platform.processor() is not safe, need to be replaced to other method.\n            llamacpp_model_config.setdefault(\"n_gpu_layers\", -1)\n        elif self._is_linux() and self._can_apply_cublas():\n            llamacpp_model_config.setdefault(\"n_gpu_layers\", -1)\n\n        return llamacpp_model_config\n\n    def _sanitize_generate_config(\n        self, generate_config: Optional[LlamaCppGenerateConfig]\n    ) -> LlamaCppGenerateConfig:\n        if generate_config is None:\n            generate_config = LlamaCppGenerateConfig(\n                **CreateCompletionLlamaCpp().dict()\n            )\n        else:\n            from llama_cpp import LlamaGrammar\n\n            grammar = generate_config.get(\"grammar\")\n            if grammar is not None and not isinstance(grammar, LlamaGrammar):\n                generate_config[\"grammar\"] = LlamaGrammar.from_string(\n                    generate_config[\"grammar\"]\n                )\n            # Validate generate_config and fill default values to the generate config.\n            generate_config = LlamaCppGenerateConfig(\n                **CreateCompletionLlamaCpp(**generate_config).dict()\n            )\n        return generate_config\n\n    def _convert_ggml_to_gguf(self, model_path: str) -> str:\n        from .tools import convert\n\n        root_dir = os.path.dirname(os.path.dirname(model_path))\n        gguf_dir = os.path.join(\n            root_dir,\n            \"{}-ggufv2-{}b\".format(\n                self.model_family.model_name, self.model_spec.model_size_in_billions\n            ),\n        )\n        os.makedirs(gguf_dir, exist_ok=True)\n        gguf_path = os.path.join(\n            gguf_dir,\n            \"{}.{}.ggufv2\".format(self.model_family.model_name, self.quantization),\n        )\n        # trick for validation, use a mark file to make sure the gguf file is converted\n        mark_file = os.path.join(gguf_dir, f\"__valid_{self.quantization}\")\n        if os.path.exists(mark_file):\n            return gguf_path\n        else:\n            logger.warning(\n                \"You are using a model with ggmlv3, \"\n                \"and it will take some time to convert to ggufv2\"\n            )\n            convert(model_path, gguf_path)\n            with open(mark_file, \"w\") as f:\n                f.write(str(datetime.datetime.now()))\n            return gguf_path\n\n    def load(self):\n        try:\n            import llama_cpp\n            from llama_cpp import Llama\n\n            if llama_cpp.__version__ < \"0.2.0\":\n                raise ValueError(\n                    \"The llama_cpp version must be greater than 0.2.0. \"\n                    \"Please upgrade your version via `pip install -U llama_cpp` or refer to \"\n                    \"https://github.com/abetlen/llama-cpp-python#installation-with-openblas--cublas--clblast--metal.\"\n                )\n        except ImportError:\n            error_message = \"Failed to import module 'llama_cpp'\"\n            installation_guide = [\n                \"Please make sure 'llama_cpp' is installed. \",\n                \"You can install it by visiting the installation section of the git repo:\\n\",\n                \"https://github.com/abetlen/llama-cpp-python#installation-with-openblas--cublas--clblast--metal\",\n            ]\n\n            raise ImportError(f\"{error_message}\\n\\n{''.join(installation_guide)}\")\n\n        # handle legacy cache.\n        model_path = os.path.join(\n            self.model_path,\n            self.model_spec.model_file_name_template.format(\n                quantization=self.quantization\n            ),\n        )\n        legacy_model_file_path = os.path.join(self.model_path, \"model.bin\")\n        if os.path.exists(legacy_model_file_path):\n            model_path = legacy_model_file_path\n\n        if self.model_spec.model_format == \"ggmlv3\":\n            model_path = self._convert_ggml_to_gguf(model_path)\n\n        try:\n            self._llm = Llama(\n                model_path=model_path,\n                verbose=True,\n                **self._llamacpp_model_config,\n            )\n        except AssertionError:\n            raise RuntimeError(f\"Load model {self.model_family.model_name} failed\")\n\n    @classmethod\n    def match(\n        cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str\n    ) -> bool:\n        if llm_spec.model_format not in [\"ggmlv3\", \"ggufv2\"]:\n            return False\n        if (\n            \"chatglm\" in llm_family.model_name\n            or \"qwen\" in llm_family.model_name\n            or llm_family.model_name in CTRANSFORMERS_SUPPORTED_MODEL\n        ):\n            return False\n        if \"generate\" not in llm_family.model_ability:\n            return False\n        return True\n\n    def generate(\n        self, prompt: str, generate_config: Optional[LlamaCppGenerateConfig] = None\n    ) -> Union[Completion, Iterator[CompletionChunk]]:\n        def generator_wrapper(\n            _prompt: str,\n            _generate_config: LlamaCppGenerateConfig,\n        ) -> Iterator[CompletionChunk]:\n            assert self._llm is not None\n            for _completion_chunk in self._llm(prompt=_prompt, **_generate_config):\n                yield _completion_chunk\n\n        logger.debug(\n            \"Enter generate, prompt: %s, generate config: %s\", prompt, generate_config\n        )\n\n        generate_config = self._sanitize_generate_config(generate_config)\n\n        stream = generate_config.get(\"stream\", False)\n\n        if not stream:\n            assert self._llm is not None\n            completion = self._llm(prompt=prompt, **generate_config)\n\n            return completion\n        else:\n            return generator_wrapper(prompt, generate_config)\n\n    def create_embedding(self, input: Union[str, List[str]]) -> Embedding:\n        assert self._llm is not None\n        embedding = self._llm.create_embedding(input)\n        return embedding\n\n\nclass LlamaCppChatModel(LlamaCppModel, ChatModelMixin):\n    def __init__(\n        self,\n        model_uid: str,\n        model_family: \"LLMFamilyV1\",\n        model_spec: \"LLMSpecV1\",\n        quantization: str,\n        model_path: str,\n        llamacpp_model_config: Optional[LlamaCppModelConfig] = None,\n    ):\n        super().__init__(\n            model_uid,\n            model_family,\n            model_spec,\n            quantization,\n            model_path,\n            llamacpp_model_config,\n        )\n\n    @classmethod\n    def match(\n        cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str\n    ) -> bool:\n        if llm_spec.model_format not in [\"ggmlv3\", \"ggufv2\"]:\n            return False\n        if (\n            \"chatglm\" in llm_family.model_name\n            or llm_family.model_name in CTRANSFORMERS_SUPPORTED_MODEL\n        ):\n            return False\n        if \"chat\" not in llm_family.model_ability:\n            return False\n        return True\n\n    def _sanitize_generate_config(\n        self, generate_config: Optional[LlamaCppGenerateConfig]\n    ) -> LlamaCppGenerateConfig:\n        generate_config = super()._sanitize_generate_config(generate_config)\n        if self.model_family.prompt_style and self.model_family.prompt_style.stop:\n            generate_config[\"stop\"] = self.model_family.prompt_style.stop\n        return generate_config\n\n    def chat(\n        self,\n        prompt: str,\n        system_prompt: Optional[str] = None,\n        chat_history: Optional[List[ChatCompletionMessage]] = None,\n        generate_config: Optional[LlamaCppGenerateConfig] = None,\n    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:\n        assert self.model_family.prompt_style is not None\n        prompt_style = self.model_family.prompt_style.copy()\n        if system_prompt:\n            prompt_style.system_prompt = system_prompt\n\n        chat_history = chat_history or []\n        assert prompt_style is not None\n        tools = generate_config.pop(\"tools\", []) if generate_config else None\n        full_prompt = self.get_prompt(prompt, chat_history, prompt_style, tools=tools)\n\n        generate_config = self._sanitize_generate_config(generate_config)\n        # TODO(codingl2k1): qwen hacky to set stop for function call.\n        model_family = self.model_family.model_family or self.model_family.model_name\n        if tools and model_family in [\"qwen-chat\", \"qwen1.5-chat\"]:\n            stop = generate_config.get(\"stop\")\n            if isinstance(stop, str):\n                generate_config[\"stop\"] = [stop, \"Observation:\"]\n            elif isinstance(stop, Iterable):\n                assert not isinstance(stop, str)\n                generate_config[\"stop\"] = stop + [\"Observation:\"]  # type: ignore\n            else:\n                generate_config[\"stop\"] = \"Observation:\"\n\n        stream = generate_config.get(\"stream\", False)\n        if stream:\n            it = self.generate(full_prompt, generate_config)\n            assert isinstance(it, Iterator)\n            return self._to_chat_completion_chunks(it)\n        else:\n            c = self.generate(full_prompt, generate_config)\n            assert not isinstance(c, Iterator)\n            if tools:\n                return self._tool_calls_completion(\n                    self.model_family, self.model_uid, c, tools\n                )\n            return self._to_chat_completion(c)\n", "path": "xinference/model/llm/ggml/llamacpp.py"}]}