We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
FEAT: support Mistral-7B-Instruct
### Is your feature request related to a problem? Please describe
https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `xinference/model/llm/ggml/ctransformers.py`
Content:
```
1 # Copyright 2022-2023 XProbe Inc.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #      http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import logging
16 import os
17 from typing import TYPE_CHECKING, Iterator, Optional, Sequence, TypedDict, Union
18 
19 if TYPE_CHECKING:
20     from ctransformers import AutoConfig
21 
22 from ....types import Completion, CompletionChunk
23 from ..core import LLM
24 from ..llm_family import LLMFamilyV1, LLMSpecV1
25 from .ctransformers_util import generate_stream
26 
27 logger = logging.getLogger(__name__)
28 
29 # all supported models for Ctransformers with their model type.
30 # Please Strictly follows this name format when inputting new model to model_family.
31 MODEL_TYPE_FOR_CTRANSFORMERS = {
32     "gpt-2": "gpt2",
33     "gpt-j": "gptj",
34     "gpt4all-j": "gptj",
35     "gpt-neox": "gpt_neox",
36     "stablelm": "gpt_neox",
37     "llama": "llama",
38     "llama-2": "llama",
39     "mpt": "mpt",
40     "dolly-v2": "dolly-v2",
41     "replit": "replit",
42     "starcoder": "starcoder",
43     "starchat": "starcoder",
44     "falcon": "falcon",
45 }
46 
47 # these two constants subjects to change for future development and ctransformers updates.
48 CTRANSFORMERS_SUPPORTED_MODEL = ["starcoder", "gpt-2"]
49 
50 CTRANSFORMERS_GPU_SUPPORT = ["llama", "llama-2", "mpt", "falcon"]
51 
52 SIZE_TO_GPU_LAYERS = {
53     3: 26,
54     7: 32,
55     13: 40,
56     30: 60,
57     65: 80,
58 }
59 
60 
61 class CtransformersModelConfig(TypedDict, total=False):
62     n_ctx: int
63     n_gpu_layers: int
64 
65 
66 class CtransformersGenerateConfig(TypedDict, total=False):
67     max_tokens: Optional[int]
68     top_k: Optional[int]
69     top_p: Optional[float]
70     temperature: Optional[float]
71     repetition_penalty: Optional[float]
72     last_n_tokens: Optional[int]
73     seed: Optional[int]
74     batch_size: Optional[int]
75     threads: Optional[int]
76     stop: Optional[Sequence[str]]
77     stream: Optional[bool]
78     reset: Optional[bool]
79 
80 
81 class CtransformersModel(LLM):
82     def __init__(
83         self,
84         model_uid: str,
85         model_family: "LLMFamilyV1",
86         model_spec: "LLMSpecV1",
87         quantization: str,
88         model_path: str,
89         ctransformers_model_config: Optional[CtransformersModelConfig],
90     ):
91         super().__init__(model_uid, model_family, model_spec, quantization, model_path)
92 
93         self._model_type = None
94         closest_size = min(
95             SIZE_TO_GPU_LAYERS.keys(),
96             key=lambda x: abs(x - model_spec.model_size_in_billions),
97         )
98 
99         self._model_family = model_family
100         self._model_uid = model_uid
101         self._llm = None
102 
103         self._gpu_layers = SIZE_TO_GPU_LAYERS[closest_size]
104         self._ctransformer_model_config = self._sanitize_model_config(
105             model_path, ctransformers_model_config
106         )
107 
108     def _sanitize_model_config(
109         self, model_path, ctransformers_model_config: Optional[CtransformersModelConfig]
110     ) -> "AutoConfig":
111         try:
112             from ctransformers import AutoConfig, Config
113         except ImportError:
114             error_message = (
115                 "Failed to import module 'ctransformers - AutoConfig and Config'"
116             )
117 
118             installation_guide = [
119                 f"Please make sure 'ctransformers' is installed.",
120                 f"You can install it by checking out the repository for command:"
121                 f"https://github.com/marella/ctransformers",
122             ]
123 
124             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
125 
126         # if the model have customized config, we update it.
127         model_config_ret = Config()
128         potential_gpu_layers = None
129         if ctransformers_model_config:
130             potential_context_length = ctransformers_model_config.pop("n_ctx", None)
131             potential_gpu_layers = ctransformers_model_config.pop("n_gpu_layers", None)
132 
133             model_config_ret.context_length = potential_context_length
134             model_config_ret.gpu_layers = potential_gpu_layers
135 
136         # if user does not define gpu layers, we have to set it with our system if applicable.
137         if potential_gpu_layers is None:
138             if self._model_family.model_name not in CTRANSFORMERS_GPU_SUPPORT:
139                 model_config_ret.gpu_layers = -1
140             elif self._is_darwin_and_apple_silicon():
141                 model_config_ret.gpu_layers = 1
142             elif self._has_cuda_device():
143                 model_config_ret.gpu_layers = self._gpu_layers
144 
145         return AutoConfig(model_config_ret)
146 
147     def _sanitize_generate_config(
148         self,
149         ctransformers_generate_config: Optional[CtransformersGenerateConfig],
150     ) -> CtransformersGenerateConfig:
151         # if the input config is not None, we try to copy the selected attributes to the ctransformersGenerateConfig.
152         if ctransformers_generate_config is None:
153             ctransformers_generate_config = CtransformersGenerateConfig()
154 
155         # for our system, the threads will have to be set to 4
156         # all other parameters, if not specified, will be set to default when generate.
157         ctransformers_generate_config.setdefault("threads", 4)
158 
159         return ctransformers_generate_config
160 
161     def load(self):
162         try:
163             from ctransformers import AutoModelForCausalLM
164         except ImportError:
165             error_message = "Failed to import module 'ctransformers'"
166 
167             installation_guide = [
168                 f"Please make sure 'ctransformers' is installed.",
169                 f"You can install it by checking out the repository for command."
170                 f"https://github.com/marella/ctransformers",
171             ]
172 
173             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
174 
175         model_path = os.path.join(
176             self.model_path,
177             self.model_spec.model_file_name_template.format(
178                 quantization=self.quantization
179             ),
180         )
181 
182         self._model_type = self._determine_model_type()
183         self._llm = AutoModelForCausalLM.from_pretrained(
184             model_path_or_repo_id=model_path,
185             model_type=self._model_type,
186             config=self._ctransformer_model_config,
187         )
188 
189     @classmethod
190     def match(
191         cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str
192     ) -> bool:
193         if llm_spec.model_format != "ggmlv3":
194             return False
195         if llm_family.model_name not in CTRANSFORMERS_SUPPORTED_MODEL:
196             return False
197         if "generate" not in llm_family.model_ability:
198             return False
199         return True
200 
201     def _determine_model_type(self):
202         if self._model_family.model_name not in MODEL_TYPE_FOR_CTRANSFORMERS:
203             raise ValueError(
204                 f"The current model {self._model_family.model_name} is not supported, check your model name. "
205             )
206         return MODEL_TYPE_FOR_CTRANSFORMERS[self._model_family.model_name]
207 
208     def generate(
209         self, prompt: str, generate_config_raw: CtransformersGenerateConfig
210     ) -> Union[Completion, Iterator[CompletionChunk]]:
211         def generator_wrapper(
212             _prompt: str,
213             _max_new_tokens: Union[int, None],
214             _generate_config: CtransformersGenerateConfig,
215         ) -> Iterator[CompletionChunk]:
216             assert self._model_uid is not None
217             for _completion_chunk, _ in generate_stream(
218                 model=self._model_uid,
219                 model_ref=self._llm,
220                 prompt=_prompt,
221                 max_new_tokens=_max_new_tokens,
222                 **_generate_config,
223             ):
224                 yield _completion_chunk
225 
226         generate_config = self._sanitize_generate_config(generate_config_raw)
227 
228         logger.debug(
229             "Enter generate, prompt: %s, generate config: %s", prompt, generate_config
230         )
231 
232         max_new_tokens = generate_config.pop("max_tokens", None)
233 
234         stream_or_not = generate_config.get("stream", False)
235         if stream_or_not:
236             return generator_wrapper(
237                 _prompt=prompt,
238                 _max_new_tokens=max_new_tokens,
239                 _generate_config=generate_config,
240             )
241         else:
242             assert self.model_uid is not None
243             completion_chunk = None
244             completion_usage = None
245             for completion_chunk, completion_usage in generate_stream(
246                 model=self.model_uid,
247                 model_ref=self._llm,
248                 prompt=prompt,
249                 max_new_tokens=max_new_tokens,
250                 **generate_config,
251             ):
252                 pass
253 
254             assert completion_chunk is not None
255             assert completion_usage is not None
256 
257             completion = Completion(
258                 id=completion_chunk["id"],
259                 object=completion_chunk["object"],
260                 created=completion_chunk["created"],
261                 model=completion_chunk["model"],
262                 choices=completion_chunk["choices"],
263                 usage=completion_usage,
264             )
265 
266             logger.debug(
267                 "Generated, completion: %s, generate config: %s",
268                 completion,
269                 generate_config,
270             )
271 
272             return completion
273
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/xinference/model/llm/ggml/ctransformers.py b/xinference/model/llm/ggml/ctransformers.py
--- a/xinference/model/llm/ggml/ctransformers.py
+++ b/xinference/model/llm/ggml/ctransformers.py
@@ -190,7 +190,7 @@
     def match(
         cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str
     ) -> bool:
-        if llm_spec.model_format != "ggmlv3":
+        if llm_spec.model_format != "ggmlv3" and llm_spec.model_format != "ggufv2":
             return False
         if llm_family.model_name not in CTRANSFORMERS_SUPPORTED_MODEL:
             return False

{"golden_diff": "diff --git a/xinference/model/llm/ggml/ctransformers.py b/xinference/model/llm/ggml/ctransformers.py\n--- a/xinference/model/llm/ggml/ctransformers.py\n+++ b/xinference/model/llm/ggml/ctransformers.py\n@@ -190,7 +190,7 @@\n     def match(\n         cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str\n     ) -> bool:\n-        if llm_spec.model_format != \"ggmlv3\":\n+        if llm_spec.model_format != \"ggmlv3\" and llm_spec.model_format != \"ggufv2\":\n             return False\n         if llm_family.model_name not in CTRANSFORMERS_SUPPORTED_MODEL:\n             return False\n", "issue": "FEAT: support Mistral-7B-Instruct\n### Is your feature request related to a problem? Please describe\r\nhttps://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1\r\n\r\n\n", "before_files": [{"content": "# Copyright 2022-2023 XProbe Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nfrom typing import TYPE_CHECKING, Iterator, Optional, Sequence, TypedDict, Union\n\nif TYPE_CHECKING:\n    from ctransformers import AutoConfig\n\nfrom ....types import Completion, CompletionChunk\nfrom ..core import LLM\nfrom ..llm_family import LLMFamilyV1, LLMSpecV1\nfrom .ctransformers_util import generate_stream\n\nlogger = logging.getLogger(__name__)\n\n# all supported models for Ctransformers with their model type.\n# Please Strictly follows this name format when inputting new model to model_family.\nMODEL_TYPE_FOR_CTRANSFORMERS = {\n    \"gpt-2\": \"gpt2\",\n    \"gpt-j\": \"gptj\",\n    \"gpt4all-j\": \"gptj\",\n    \"gpt-neox\": \"gpt_neox\",\n    \"stablelm\": \"gpt_neox\",\n    \"llama\": \"llama\",\n    \"llama-2\": \"llama\",\n    \"mpt\": \"mpt\",\n    \"dolly-v2\": \"dolly-v2\",\n    \"replit\": \"replit\",\n    \"starcoder\": \"starcoder\",\n    \"starchat\": \"starcoder\",\n    \"falcon\": \"falcon\",\n}\n\n# these two constants subjects to change for future development and ctransformers updates.\nCTRANSFORMERS_SUPPORTED_MODEL = [\"starcoder\", \"gpt-2\"]\n\nCTRANSFORMERS_GPU_SUPPORT = [\"llama\", \"llama-2\", \"mpt\", \"falcon\"]\n\nSIZE_TO_GPU_LAYERS = {\n    3: 26,\n    7: 32,\n    13: 40,\n    30: 60,\n    65: 80,\n}\n\n\nclass CtransformersModelConfig(TypedDict, total=False):\n    n_ctx: int\n    n_gpu_layers: int\n\n\nclass CtransformersGenerateConfig(TypedDict, total=False):\n    max_tokens: Optional[int]\n    top_k: Optional[int]\n    top_p: Optional[float]\n    temperature: Optional[float]\n    repetition_penalty: Optional[float]\n    last_n_tokens: Optional[int]\n    seed: Optional[int]\n    batch_size: Optional[int]\n    threads: Optional[int]\n    stop: Optional[Sequence[str]]\n    stream: Optional[bool]\n    reset: Optional[bool]\n\n\nclass CtransformersModel(LLM):\n    def __init__(\n        self,\n        model_uid: str,\n        model_family: \"LLMFamilyV1\",\n        model_spec: \"LLMSpecV1\",\n        quantization: str,\n        model_path: str,\n        ctransformers_model_config: Optional[CtransformersModelConfig],\n    ):\n        super().__init__(model_uid, model_family, model_spec, quantization, model_path)\n\n        self._model_type = None\n        closest_size = min(\n            SIZE_TO_GPU_LAYERS.keys(),\n            key=lambda x: abs(x - model_spec.model_size_in_billions),\n        )\n\n        self._model_family = model_family\n        self._model_uid = model_uid\n        self._llm = None\n\n        self._gpu_layers = SIZE_TO_GPU_LAYERS[closest_size]\n        self._ctransformer_model_config = self._sanitize_model_config(\n            model_path, ctransformers_model_config\n        )\n\n    def _sanitize_model_config(\n        self, model_path, ctransformers_model_config: Optional[CtransformersModelConfig]\n    ) -> \"AutoConfig\":\n        try:\n            from ctransformers import AutoConfig, Config\n        except ImportError:\n            error_message = (\n                \"Failed to import module 'ctransformers - AutoConfig and Config'\"\n            )\n\n            installation_guide = [\n                f\"Please make sure 'ctransformers' is installed.\",\n                f\"You can install it by checking out the repository for command:\"\n                f\"https://github.com/marella/ctransformers\",\n            ]\n\n            raise ImportError(f\"{error_message}\\n\\n{''.join(installation_guide)}\")\n\n        # if the model have customized config, we update it.\n        model_config_ret = Config()\n        potential_gpu_layers = None\n        if ctransformers_model_config:\n            potential_context_length = ctransformers_model_config.pop(\"n_ctx\", None)\n            potential_gpu_layers = ctransformers_model_config.pop(\"n_gpu_layers\", None)\n\n            model_config_ret.context_length = potential_context_length\n            model_config_ret.gpu_layers = potential_gpu_layers\n\n        # if user does not define gpu layers, we have to set it with our system if applicable.\n        if potential_gpu_layers is None:\n            if self._model_family.model_name not in CTRANSFORMERS_GPU_SUPPORT:\n                model_config_ret.gpu_layers = -1\n            elif self._is_darwin_and_apple_silicon():\n                model_config_ret.gpu_layers = 1\n            elif self._has_cuda_device():\n                model_config_ret.gpu_layers = self._gpu_layers\n\n        return AutoConfig(model_config_ret)\n\n    def _sanitize_generate_config(\n        self,\n        ctransformers_generate_config: Optional[CtransformersGenerateConfig],\n    ) -> CtransformersGenerateConfig:\n        # if the input config is not None, we try to copy the selected attributes to the ctransformersGenerateConfig.\n        if ctransformers_generate_config is None:\n            ctransformers_generate_config = CtransformersGenerateConfig()\n\n        # for our system, the threads will have to be set to 4\n        # all other parameters, if not specified, will be set to default when generate.\n        ctransformers_generate_config.setdefault(\"threads\", 4)\n\n        return ctransformers_generate_config\n\n    def load(self):\n        try:\n            from ctransformers import AutoModelForCausalLM\n        except ImportError:\n            error_message = \"Failed to import module 'ctransformers'\"\n\n            installation_guide = [\n                f\"Please make sure 'ctransformers' is installed.\",\n                f\"You can install it by checking out the repository for command.\"\n                f\"https://github.com/marella/ctransformers\",\n            ]\n\n            raise ImportError(f\"{error_message}\\n\\n{''.join(installation_guide)}\")\n\n        model_path = os.path.join(\n            self.model_path,\n            self.model_spec.model_file_name_template.format(\n                quantization=self.quantization\n            ),\n        )\n\n        self._model_type = self._determine_model_type()\n        self._llm = AutoModelForCausalLM.from_pretrained(\n            model_path_or_repo_id=model_path,\n            model_type=self._model_type,\n            config=self._ctransformer_model_config,\n        )\n\n    @classmethod\n    def match(\n        cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str\n    ) -> bool:\n        if llm_spec.model_format != \"ggmlv3\":\n            return False\n        if llm_family.model_name not in CTRANSFORMERS_SUPPORTED_MODEL:\n            return False\n        if \"generate\" not in llm_family.model_ability:\n            return False\n        return True\n\n    def _determine_model_type(self):\n        if self._model_family.model_name not in MODEL_TYPE_FOR_CTRANSFORMERS:\n            raise ValueError(\n                f\"The current model {self._model_family.model_name} is not supported, check your model name. \"\n            )\n        return MODEL_TYPE_FOR_CTRANSFORMERS[self._model_family.model_name]\n\n    def generate(\n        self, prompt: str, generate_config_raw: CtransformersGenerateConfig\n    ) -> Union[Completion, Iterator[CompletionChunk]]:\n        def generator_wrapper(\n            _prompt: str,\n            _max_new_tokens: Union[int, None],\n            _generate_config: CtransformersGenerateConfig,\n        ) -> Iterator[CompletionChunk]:\n            assert self._model_uid is not None\n            for _completion_chunk, _ in generate_stream(\n                model=self._model_uid,\n                model_ref=self._llm,\n                prompt=_prompt,\n                max_new_tokens=_max_new_tokens,\n                **_generate_config,\n            ):\n                yield _completion_chunk\n\n        generate_config = self._sanitize_generate_config(generate_config_raw)\n\n        logger.debug(\n            \"Enter generate, prompt: %s, generate config: %s\", prompt, generate_config\n        )\n\n        max_new_tokens = generate_config.pop(\"max_tokens\", None)\n\n        stream_or_not = generate_config.get(\"stream\", False)\n        if stream_or_not:\n            return generator_wrapper(\n                _prompt=prompt,\n                _max_new_tokens=max_new_tokens,\n                _generate_config=generate_config,\n            )\n        else:\n            assert self.model_uid is not None\n            completion_chunk = None\n            completion_usage = None\n            for completion_chunk, completion_usage in generate_stream(\n                model=self.model_uid,\n                model_ref=self._llm,\n                prompt=prompt,\n                max_new_tokens=max_new_tokens,\n                **generate_config,\n            ):\n                pass\n\n            assert completion_chunk is not None\n            assert completion_usage is not None\n\n            completion = Completion(\n                id=completion_chunk[\"id\"],\n                object=completion_chunk[\"object\"],\n                created=completion_chunk[\"created\"],\n                model=completion_chunk[\"model\"],\n                choices=completion_chunk[\"choices\"],\n                usage=completion_usage,\n            )\n\n            logger.debug(\n                \"Generated, completion: %s, generate config: %s\",\n                completion,\n                generate_config,\n            )\n\n            return completion\n", "path": "xinference/model/llm/ggml/ctransformers.py"}], "after_files": [{"content": "# Copyright 2022-2023 XProbe Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nfrom typing import TYPE_CHECKING, Iterator, Optional, Sequence, TypedDict, Union\n\nif TYPE_CHECKING:\n    from ctransformers import AutoConfig\n\nfrom ....types import Completion, CompletionChunk\nfrom ..core import LLM\nfrom ..llm_family import LLMFamilyV1, LLMSpecV1\nfrom .ctransformers_util import generate_stream\n\nlogger = logging.getLogger(__name__)\n\n# all supported models for Ctransformers with their model type.\n# Please Strictly follows this name format when inputting new model to model_family.\nMODEL_TYPE_FOR_CTRANSFORMERS = {\n    \"gpt-2\": \"gpt2\",\n    \"gpt-j\": \"gptj\",\n    \"gpt4all-j\": \"gptj\",\n    \"gpt-neox\": \"gpt_neox\",\n    \"stablelm\": \"gpt_neox\",\n    \"llama\": \"llama\",\n    \"llama-2\": \"llama\",\n    \"mpt\": \"mpt\",\n    \"dolly-v2\": \"dolly-v2\",\n    \"replit\": \"replit\",\n    \"starcoder\": \"starcoder\",\n    \"starchat\": \"starcoder\",\n    \"falcon\": \"falcon\",\n}\n\n# these two constants subjects to change for future development and ctransformers updates.\nCTRANSFORMERS_SUPPORTED_MODEL = [\"starcoder\", \"gpt-2\"]\n\nCTRANSFORMERS_GPU_SUPPORT = [\"llama\", \"llama-2\", \"mpt\", \"falcon\"]\n\nSIZE_TO_GPU_LAYERS = {\n    3: 26,\n    7: 32,\n    13: 40,\n    30: 60,\n    65: 80,\n}\n\n\nclass CtransformersModelConfig(TypedDict, total=False):\n    n_ctx: int\n    n_gpu_layers: int\n\n\nclass CtransformersGenerateConfig(TypedDict, total=False):\n    max_tokens: Optional[int]\n    top_k: Optional[int]\n    top_p: Optional[float]\n    temperature: Optional[float]\n    repetition_penalty: Optional[float]\n    last_n_tokens: Optional[int]\n    seed: Optional[int]\n    batch_size: Optional[int]\n    threads: Optional[int]\n    stop: Optional[Sequence[str]]\n    stream: Optional[bool]\n    reset: Optional[bool]\n\n\nclass CtransformersModel(LLM):\n    def __init__(\n        self,\n        model_uid: str,\n        model_family: \"LLMFamilyV1\",\n        model_spec: \"LLMSpecV1\",\n        quantization: str,\n        model_path: str,\n        ctransformers_model_config: Optional[CtransformersModelConfig],\n    ):\n        super().__init__(model_uid, model_family, model_spec, quantization, model_path)\n\n        self._model_type = None\n        closest_size = min(\n            SIZE_TO_GPU_LAYERS.keys(),\n            key=lambda x: abs(x - model_spec.model_size_in_billions),\n        )\n\n        self._model_family = model_family\n        self._model_uid = model_uid\n        self._llm = None\n\n        self._gpu_layers = SIZE_TO_GPU_LAYERS[closest_size]\n        self._ctransformer_model_config = self._sanitize_model_config(\n            model_path, ctransformers_model_config\n        )\n\n    def _sanitize_model_config(\n        self, model_path, ctransformers_model_config: Optional[CtransformersModelConfig]\n    ) -> \"AutoConfig\":\n        try:\n            from ctransformers import AutoConfig, Config\n        except ImportError:\n            error_message = (\n                \"Failed to import module 'ctransformers - AutoConfig and Config'\"\n            )\n\n            installation_guide = [\n                f\"Please make sure 'ctransformers' is installed.\",\n                f\"You can install it by checking out the repository for command:\"\n                f\"https://github.com/marella/ctransformers\",\n            ]\n\n            raise ImportError(f\"{error_message}\\n\\n{''.join(installation_guide)}\")\n\n        # if the model have customized config, we update it.\n        model_config_ret = Config()\n        potential_gpu_layers = None\n        if ctransformers_model_config:\n            potential_context_length = ctransformers_model_config.pop(\"n_ctx\", None)\n            potential_gpu_layers = ctransformers_model_config.pop(\"n_gpu_layers\", None)\n\n            model_config_ret.context_length = potential_context_length\n            model_config_ret.gpu_layers = potential_gpu_layers\n\n        # if user does not define gpu layers, we have to set it with our system if applicable.\n        if potential_gpu_layers is None:\n            if self._model_family.model_name not in CTRANSFORMERS_GPU_SUPPORT:\n                model_config_ret.gpu_layers = -1\n            elif self._is_darwin_and_apple_silicon():\n                model_config_ret.gpu_layers = 1\n            elif self._has_cuda_device():\n                model_config_ret.gpu_layers = self._gpu_layers\n\n        return AutoConfig(model_config_ret)\n\n    def _sanitize_generate_config(\n        self,\n        ctransformers_generate_config: Optional[CtransformersGenerateConfig],\n    ) -> CtransformersGenerateConfig:\n        # if the input config is not None, we try to copy the selected attributes to the ctransformersGenerateConfig.\n        if ctransformers_generate_config is None:\n            ctransformers_generate_config = CtransformersGenerateConfig()\n\n        # for our system, the threads will have to be set to 4\n        # all other parameters, if not specified, will be set to default when generate.\n        ctransformers_generate_config.setdefault(\"threads\", 4)\n\n        return ctransformers_generate_config\n\n    def load(self):\n        try:\n            from ctransformers import AutoModelForCausalLM\n        except ImportError:\n            error_message = \"Failed to import module 'ctransformers'\"\n\n            installation_guide = [\n                f\"Please make sure 'ctransformers' is installed.\",\n                f\"You can install it by checking out the repository for command.\"\n                f\"https://github.com/marella/ctransformers\",\n            ]\n\n            raise ImportError(f\"{error_message}\\n\\n{''.join(installation_guide)}\")\n\n        model_path = os.path.join(\n            self.model_path,\n            self.model_spec.model_file_name_template.format(\n                quantization=self.quantization\n            ),\n        )\n\n        self._model_type = self._determine_model_type()\n        self._llm = AutoModelForCausalLM.from_pretrained(\n            model_path_or_repo_id=model_path,\n            model_type=self._model_type,\n            config=self._ctransformer_model_config,\n        )\n\n    @classmethod\n    def match(\n        cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str\n    ) -> bool:\n        if llm_spec.model_format != \"ggmlv3\" and llm_spec.model_format != \"ggufv2\":\n            return False\n        if llm_family.model_name not in CTRANSFORMERS_SUPPORTED_MODEL:\n            return False\n        if \"generate\" not in llm_family.model_ability:\n            return False\n        return True\n\n    def _determine_model_type(self):\n        if self._model_family.model_name not in MODEL_TYPE_FOR_CTRANSFORMERS:\n            raise ValueError(\n                f\"The current model {self._model_family.model_name} is not supported, check your model name. \"\n            )\n        return MODEL_TYPE_FOR_CTRANSFORMERS[self._model_family.model_name]\n\n    def generate(\n        self, prompt: str, generate_config_raw: CtransformersGenerateConfig\n    ) -> Union[Completion, Iterator[CompletionChunk]]:\n        def generator_wrapper(\n            _prompt: str,\n            _max_new_tokens: Union[int, None],\n            _generate_config: CtransformersGenerateConfig,\n        ) -> Iterator[CompletionChunk]:\n            assert self._model_uid is not None\n            for _completion_chunk, _ in generate_stream(\n                model=self._model_uid,\n                model_ref=self._llm,\n                prompt=_prompt,\n                max_new_tokens=_max_new_tokens,\n                **_generate_config,\n            ):\n                yield _completion_chunk\n\n        generate_config = self._sanitize_generate_config(generate_config_raw)\n\n        logger.debug(\n            \"Enter generate, prompt: %s, generate config: %s\", prompt, generate_config\n        )\n\n        max_new_tokens = generate_config.pop(\"max_tokens\", None)\n\n        stream_or_not = generate_config.get(\"stream\", False)\n        if stream_or_not:\n            return generator_wrapper(\n                _prompt=prompt,\n                _max_new_tokens=max_new_tokens,\n                _generate_config=generate_config,\n            )\n        else:\n            assert self.model_uid is not None\n            completion_chunk = None\n            completion_usage = None\n            for completion_chunk, completion_usage in generate_stream(\n                model=self.model_uid,\n                model_ref=self._llm,\n                prompt=prompt,\n                max_new_tokens=max_new_tokens,\n                **generate_config,\n            ):\n                pass\n\n            assert completion_chunk is not None\n            assert completion_usage is not None\n\n            completion = Completion(\n                id=completion_chunk[\"id\"],\n                object=completion_chunk[\"object\"],\n                created=completion_chunk[\"created\"],\n                model=completion_chunk[\"model\"],\n                choices=completion_chunk[\"choices\"],\n                usage=completion_usage,\n            )\n\n            logger.debug(\n                \"Generated, completion: %s, generate config: %s\",\n                completion,\n                generate_config,\n            )\n\n            return completion\n", "path": "xinference/model/llm/ggml/ctransformers.py"}]}