We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
The huggingface translation task name requires an extra suffix
When I run the Huggingface translation task, got this raise ValueError('The task defaults can\'t be correctly selected. You probably meant "translation_XX_to_YY"')


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `runtimes/huggingface/mlserver_huggingface/common.py`
Content:
```
1 import os
2 import json
3 from typing import Optional, Dict
4 from distutils.util import strtobool
5 
6 import numpy as np
7 from pydantic import BaseSettings
8 from mlserver.errors import MLServerError
9 
10 from transformers.pipelines import pipeline
11 from transformers.pipelines.base import Pipeline
12 from transformers.models.auto.tokenization_auto import AutoTokenizer
13 
14 from optimum.pipelines import SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS
15 
16 
17 HUGGINGFACE_TASK_TAG = "task"
18 
19 ENV_PREFIX_HUGGINGFACE_SETTINGS = "MLSERVER_MODEL_HUGGINGFACE_"
20 HUGGINGFACE_PARAMETERS_TAG = "huggingface_parameters"
21 PARAMETERS_ENV_NAME = "PREDICTIVE_UNIT_PARAMETERS"
22 
23 
24 class InvalidTranformerInitialisation(MLServerError):
25     def __init__(self, code: int, reason: str):
26         super().__init__(
27             f"Huggingface server failed with {code}, {reason}",
28             status_code=code,
29         )
30 
31 
32 class HuggingFaceSettings(BaseSettings):
33     """
34     Parameters that apply only to alibi huggingface models
35     """
36 
37     class Config:
38         env_prefix = ENV_PREFIX_HUGGINGFACE_SETTINGS
39 
40     task: str = ""
41     pretrained_model: Optional[str] = None
42     pretrained_tokenizer: Optional[str] = None
43     optimum_model: bool = False
44     device: int = -1
45     batch_size: Optional[int] = None
46 
47 
48 def parse_parameters_from_env() -> Dict:
49     """
50     TODO
51     """
52     parameters = json.loads(os.environ.get(PARAMETERS_ENV_NAME, "[]"))
53 
54     type_dict = {
55         "INT": int,
56         "FLOAT": float,
57         "DOUBLE": float,
58         "STRING": str,
59         "BOOL": bool,
60     }
61 
62     parsed_parameters = {}
63     for param in parameters:
64         name = param.get("name")
65         value = param.get("value")
66         type_ = param.get("type")
67         if type_ == "BOOL":
68             parsed_parameters[name] = bool(strtobool(value))
69         else:
70             try:
71                 parsed_parameters[name] = type_dict[type_](value)
72             except ValueError:
73                 raise InvalidTranformerInitialisation(
74                     "Bad model parameter: "
75                     + name
76                     + " with value "
77                     + value
78                     + " can't be parsed as a "
79                     + type_,
80                     reason="MICROSERVICE_BAD_PARAMETER",
81                 )
82             except KeyError:
83                 raise InvalidTranformerInitialisation(
84                     "Bad model parameter type: "
85                     + type_
86                     + " valid are INT, FLOAT, DOUBLE, STRING, BOOL",
87                     reason="MICROSERVICE_BAD_PARAMETER",
88                 )
89     return parsed_parameters
90 
91 
92 def load_pipeline_from_settings(hf_settings: HuggingFaceSettings) -> Pipeline:
93     """
94     TODO
95     """
96     # TODO: Support URI for locally downloaded artifacts
97     # uri = model_parameters.uri
98     model = hf_settings.pretrained_model
99     tokenizer = hf_settings.pretrained_tokenizer
100     device = hf_settings.device
101 
102     if model and not tokenizer:
103         tokenizer = model
104 
105     if hf_settings.optimum_model:
106         optimum_class = SUPPORTED_OPTIMUM_TASKS[hf_settings.task]["class"][0]
107         model = optimum_class.from_pretrained(
108             hf_settings.pretrained_model,
109             from_transformers=True,
110         )
111         tokenizer = AutoTokenizer.from_pretrained(tokenizer)
112         # Device needs to be set to -1 due to known issue
113         # https://github.com/huggingface/optimum/issues/191
114         device = -1
115 
116     pp = pipeline(
117         hf_settings.task,
118         model=model,
119         tokenizer=tokenizer,
120         device=device,
121         batch_size=hf_settings.batch_size,
122     )
123 
124     # If batch_size > 0 we need to ensure tokens are padded
125     if hf_settings.batch_size:
126         pp.tokenizer.pad_token_id = [str(pp.model.config.eos_token_id)]  # type: ignore
127 
128     return pp
129 
130 
131 class NumpyEncoder(json.JSONEncoder):
132     def default(self, obj):
133         if isinstance(obj, np.ndarray):
134             return obj.tolist()
135         return json.JSONEncoder.default(self, obj)
136
```

Path: `runtimes/huggingface/mlserver_huggingface/runtime.py`
Content:
```
1 import asyncio
2 from mlserver.model import MLModel
3 from mlserver.settings import ModelSettings
4 from mlserver.types import (
5     InferenceRequest,
6     InferenceResponse,
7 )
8 from mlserver_huggingface.common import (
9     HuggingFaceSettings,
10     parse_parameters_from_env,
11     InvalidTranformerInitialisation,
12     load_pipeline_from_settings,
13 )
14 from mlserver_huggingface.codecs import HuggingfaceRequestCodec
15 from transformers.pipelines import SUPPORTED_TASKS
16 from optimum.pipelines import SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS
17 from mlserver.logging import logger
18 from mlserver_huggingface.metadata import METADATA
19 
20 
21 class HuggingFaceRuntime(MLModel):
22     """Runtime class for specific Huggingface models"""
23 
24     def __init__(self, settings: ModelSettings):
25         env_params = parse_parameters_from_env()
26         if not env_params and (
27             not settings.parameters or not settings.parameters.extra
28         ):
29             raise InvalidTranformerInitialisation(
30                 500,
31                 "Settings parameters not provided via config file nor env variables",
32             )
33 
34         extra = env_params or settings.parameters.extra  # type: ignore
35         self.hf_settings = HuggingFaceSettings(**extra)  # type: ignore
36 
37         if self.hf_settings.task not in SUPPORTED_TASKS:
38             raise InvalidTranformerInitialisation(
39                 500,
40                 (
41                     f"Invalid transformer task: {self.hf_settings.task}."
42                     f" Available tasks: {SUPPORTED_TASKS.keys()}"
43                 ),
44             )
45 
46         if self.hf_settings.optimum_model:
47             if self.hf_settings.task not in SUPPORTED_OPTIMUM_TASKS:
48                 raise InvalidTranformerInitialisation(
49                     500,
50                     (
51                         f"Invalid transformer task for "
52                         f"OPTIMUM model: {self.hf_settings.task}. "
53                         f"Supported Optimum tasks: {SUPPORTED_OPTIMUM_TASKS.keys()}"
54                     ),
55                 )
56 
57         if settings.max_batch_size != self.hf_settings.batch_size:
58             logger.warning(
59                 f"hf batch_size: {self.hf_settings.batch_size} is different "
60                 f"from MLServer max_batch_size: {settings.max_batch_size}"
61             )
62 
63         super().__init__(settings)
64 
65     async def load(self) -> bool:
66         # Loading & caching pipeline in asyncio loop to avoid blocking
67         print("=" * 80)
68         print(self.hf_settings.task)
69         print("loading model...")
70         await asyncio.get_running_loop().run_in_executor(
71             None, load_pipeline_from_settings, self.hf_settings
72         )
73         print("(re)loading model...")
74         # Now we load the cached model which should not block asyncio
75         self._model = load_pipeline_from_settings(self.hf_settings)
76         self._merge_metadata()
77         print("model has been loaded!")
78         self.ready = True
79         return self.ready
80 
81     async def predict(self, payload: InferenceRequest) -> InferenceResponse:
82         """
83         TODO
84         """
85 
86         # Adding some logging as hard to debug given the many types of input accepted
87         logger.debug("Payload %s", payload)
88 
89         # TODO: convert and validate?
90         kwargs = self.decode_request(payload, default_codec=HuggingfaceRequestCodec)
91         args = kwargs.pop("args", [])
92 
93         array_inputs = kwargs.pop("array_inputs", [])
94         if array_inputs:
95             args = [list(array_inputs)] + args
96         prediction = self._model(*args, **kwargs)
97 
98         logger.debug("Prediction %s", prediction)
99 
100         return self.encode_response(
101             payload=prediction, default_codec=HuggingfaceRequestCodec
102         )
103 
104     def _merge_metadata(self) -> None:
105         meta = METADATA.get(self.hf_settings.task)
106         if meta:
107             self.inputs += meta.get("inputs", [])  # type: ignore
108             self.outputs += meta.get("outputs", [])  # type: ignore
109
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/runtimes/huggingface/mlserver_huggingface/common.py b/runtimes/huggingface/mlserver_huggingface/common.py
--- a/runtimes/huggingface/mlserver_huggingface/common.py
+++ b/runtimes/huggingface/mlserver_huggingface/common.py
@@ -38,12 +38,22 @@
         env_prefix = ENV_PREFIX_HUGGINGFACE_SETTINGS
 
     task: str = ""
+    # Why need this filed?
+    # for translation task, required a suffix to specify source and target
+    # related issue: https://github.com/SeldonIO/MLServer/issues/947
+    task_suffix: str = ""
     pretrained_model: Optional[str] = None
     pretrained_tokenizer: Optional[str] = None
     optimum_model: bool = False
     device: int = -1
     batch_size: Optional[int] = None
 
+    @property
+    def task_name(self):
+        if self.task == "translation":
+            return f"{self.task}{self.task_suffix}"
+        return self.task
+
 
 def parse_parameters_from_env() -> Dict:
     """
@@ -114,7 +124,7 @@
         device = -1
 
     pp = pipeline(
-        hf_settings.task,
+        hf_settings.task_name,
         model=model,
         tokenizer=tokenizer,
         device=device,
diff --git a/runtimes/huggingface/mlserver_huggingface/runtime.py b/runtimes/huggingface/mlserver_huggingface/runtime.py
--- a/runtimes/huggingface/mlserver_huggingface/runtime.py
+++ b/runtimes/huggingface/mlserver_huggingface/runtime.py
@@ -65,7 +65,7 @@
     async def load(self) -> bool:
         # Loading & caching pipeline in asyncio loop to avoid blocking
         print("=" * 80)
-        print(self.hf_settings.task)
+        print(self.hf_settings.task_name)
         print("loading model...")
         await asyncio.get_running_loop().run_in_executor(
             None, load_pipeline_from_settings, self.hf_settings

{"golden_diff": "diff --git a/runtimes/huggingface/mlserver_huggingface/common.py b/runtimes/huggingface/mlserver_huggingface/common.py\n--- a/runtimes/huggingface/mlserver_huggingface/common.py\n+++ b/runtimes/huggingface/mlserver_huggingface/common.py\n@@ -38,12 +38,22 @@\n         env_prefix = ENV_PREFIX_HUGGINGFACE_SETTINGS\n \n     task: str = \"\"\n+    # Why need this filed?\n+    # for translation task, required a suffix to specify source and target\n+    # related issue: https://github.com/SeldonIO/MLServer/issues/947\n+    task_suffix: str = \"\"\n     pretrained_model: Optional[str] = None\n     pretrained_tokenizer: Optional[str] = None\n     optimum_model: bool = False\n     device: int = -1\n     batch_size: Optional[int] = None\n \n+    @property\n+    def task_name(self):\n+        if self.task == \"translation\":\n+            return f\"{self.task}{self.task_suffix}\"\n+        return self.task\n+\n \n def parse_parameters_from_env() -> Dict:\n     \"\"\"\n@@ -114,7 +124,7 @@\n         device = -1\n \n     pp = pipeline(\n-        hf_settings.task,\n+        hf_settings.task_name,\n         model=model,\n         tokenizer=tokenizer,\n         device=device,\ndiff --git a/runtimes/huggingface/mlserver_huggingface/runtime.py b/runtimes/huggingface/mlserver_huggingface/runtime.py\n--- a/runtimes/huggingface/mlserver_huggingface/runtime.py\n+++ b/runtimes/huggingface/mlserver_huggingface/runtime.py\n@@ -65,7 +65,7 @@\n     async def load(self) -> bool:\n         # Loading & caching pipeline in asyncio loop to avoid blocking\n         print(\"=\" * 80)\n-        print(self.hf_settings.task)\n+        print(self.hf_settings.task_name)\n         print(\"loading model...\")\n         await asyncio.get_running_loop().run_in_executor(\n             None, load_pipeline_from_settings, self.hf_settings\n", "issue": "The huggingface translation task name requires an extra suffix\nWhen I run the Huggingface translation task, got this raise ValueError('The task defaults can\\'t be correctly selected. You probably meant \"translation_XX_to_YY\"')\r\n\n", "before_files": [{"content": "import os\nimport json\nfrom typing import Optional, Dict\nfrom distutils.util import strtobool\n\nimport numpy as np\nfrom pydantic import BaseSettings\nfrom mlserver.errors import MLServerError\n\nfrom transformers.pipelines import pipeline\nfrom transformers.pipelines.base import Pipeline\nfrom transformers.models.auto.tokenization_auto import AutoTokenizer\n\nfrom optimum.pipelines import SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS\n\n\nHUGGINGFACE_TASK_TAG = \"task\"\n\nENV_PREFIX_HUGGINGFACE_SETTINGS = \"MLSERVER_MODEL_HUGGINGFACE_\"\nHUGGINGFACE_PARAMETERS_TAG = \"huggingface_parameters\"\nPARAMETERS_ENV_NAME = \"PREDICTIVE_UNIT_PARAMETERS\"\n\n\nclass InvalidTranformerInitialisation(MLServerError):\n    def __init__(self, code: int, reason: str):\n        super().__init__(\n            f\"Huggingface server failed with {code}, {reason}\",\n            status_code=code,\n        )\n\n\nclass HuggingFaceSettings(BaseSettings):\n    \"\"\"\n    Parameters that apply only to alibi huggingface models\n    \"\"\"\n\n    class Config:\n        env_prefix = ENV_PREFIX_HUGGINGFACE_SETTINGS\n\n    task: str = \"\"\n    pretrained_model: Optional[str] = None\n    pretrained_tokenizer: Optional[str] = None\n    optimum_model: bool = False\n    device: int = -1\n    batch_size: Optional[int] = None\n\n\ndef parse_parameters_from_env() -> Dict:\n    \"\"\"\n    TODO\n    \"\"\"\n    parameters = json.loads(os.environ.get(PARAMETERS_ENV_NAME, \"[]\"))\n\n    type_dict = {\n        \"INT\": int,\n        \"FLOAT\": float,\n        \"DOUBLE\": float,\n        \"STRING\": str,\n        \"BOOL\": bool,\n    }\n\n    parsed_parameters = {}\n    for param in parameters:\n        name = param.get(\"name\")\n        value = param.get(\"value\")\n        type_ = param.get(\"type\")\n        if type_ == \"BOOL\":\n            parsed_parameters[name] = bool(strtobool(value))\n        else:\n            try:\n                parsed_parameters[name] = type_dict[type_](value)\n            except ValueError:\n                raise InvalidTranformerInitialisation(\n                    \"Bad model parameter: \"\n                    + name\n                    + \" with value \"\n                    + value\n                    + \" can't be parsed as a \"\n                    + type_,\n                    reason=\"MICROSERVICE_BAD_PARAMETER\",\n                )\n            except KeyError:\n                raise InvalidTranformerInitialisation(\n                    \"Bad model parameter type: \"\n                    + type_\n                    + \" valid are INT, FLOAT, DOUBLE, STRING, BOOL\",\n                    reason=\"MICROSERVICE_BAD_PARAMETER\",\n                )\n    return parsed_parameters\n\n\ndef load_pipeline_from_settings(hf_settings: HuggingFaceSettings) -> Pipeline:\n    \"\"\"\n    TODO\n    \"\"\"\n    # TODO: Support URI for locally downloaded artifacts\n    # uri = model_parameters.uri\n    model = hf_settings.pretrained_model\n    tokenizer = hf_settings.pretrained_tokenizer\n    device = hf_settings.device\n\n    if model and not tokenizer:\n        tokenizer = model\n\n    if hf_settings.optimum_model:\n        optimum_class = SUPPORTED_OPTIMUM_TASKS[hf_settings.task][\"class\"][0]\n        model = optimum_class.from_pretrained(\n            hf_settings.pretrained_model,\n            from_transformers=True,\n        )\n        tokenizer = AutoTokenizer.from_pretrained(tokenizer)\n        # Device needs to be set to -1 due to known issue\n        # https://github.com/huggingface/optimum/issues/191\n        device = -1\n\n    pp = pipeline(\n        hf_settings.task,\n        model=model,\n        tokenizer=tokenizer,\n        device=device,\n        batch_size=hf_settings.batch_size,\n    )\n\n    # If batch_size > 0 we need to ensure tokens are padded\n    if hf_settings.batch_size:\n        pp.tokenizer.pad_token_id = [str(pp.model.config.eos_token_id)]  # type: ignore\n\n    return pp\n\n\nclass NumpyEncoder(json.JSONEncoder):\n    def default(self, obj):\n        if isinstance(obj, np.ndarray):\n            return obj.tolist()\n        return json.JSONEncoder.default(self, obj)\n", "path": "runtimes/huggingface/mlserver_huggingface/common.py"}, {"content": "import asyncio\nfrom mlserver.model import MLModel\nfrom mlserver.settings import ModelSettings\nfrom mlserver.types import (\n    InferenceRequest,\n    InferenceResponse,\n)\nfrom mlserver_huggingface.common import (\n    HuggingFaceSettings,\n    parse_parameters_from_env,\n    InvalidTranformerInitialisation,\n    load_pipeline_from_settings,\n)\nfrom mlserver_huggingface.codecs import HuggingfaceRequestCodec\nfrom transformers.pipelines import SUPPORTED_TASKS\nfrom optimum.pipelines import SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS\nfrom mlserver.logging import logger\nfrom mlserver_huggingface.metadata import METADATA\n\n\nclass HuggingFaceRuntime(MLModel):\n    \"\"\"Runtime class for specific Huggingface models\"\"\"\n\n    def __init__(self, settings: ModelSettings):\n        env_params = parse_parameters_from_env()\n        if not env_params and (\n            not settings.parameters or not settings.parameters.extra\n        ):\n            raise InvalidTranformerInitialisation(\n                500,\n                \"Settings parameters not provided via config file nor env variables\",\n            )\n\n        extra = env_params or settings.parameters.extra  # type: ignore\n        self.hf_settings = HuggingFaceSettings(**extra)  # type: ignore\n\n        if self.hf_settings.task not in SUPPORTED_TASKS:\n            raise InvalidTranformerInitialisation(\n                500,\n                (\n                    f\"Invalid transformer task: {self.hf_settings.task}.\"\n                    f\" Available tasks: {SUPPORTED_TASKS.keys()}\"\n                ),\n            )\n\n        if self.hf_settings.optimum_model:\n            if self.hf_settings.task not in SUPPORTED_OPTIMUM_TASKS:\n                raise InvalidTranformerInitialisation(\n                    500,\n                    (\n                        f\"Invalid transformer task for \"\n                        f\"OPTIMUM model: {self.hf_settings.task}. \"\n                        f\"Supported Optimum tasks: {SUPPORTED_OPTIMUM_TASKS.keys()}\"\n                    ),\n                )\n\n        if settings.max_batch_size != self.hf_settings.batch_size:\n            logger.warning(\n                f\"hf batch_size: {self.hf_settings.batch_size} is different \"\n                f\"from MLServer max_batch_size: {settings.max_batch_size}\"\n            )\n\n        super().__init__(settings)\n\n    async def load(self) -> bool:\n        # Loading & caching pipeline in asyncio loop to avoid blocking\n        print(\"=\" * 80)\n        print(self.hf_settings.task)\n        print(\"loading model...\")\n        await asyncio.get_running_loop().run_in_executor(\n            None, load_pipeline_from_settings, self.hf_settings\n        )\n        print(\"(re)loading model...\")\n        # Now we load the cached model which should not block asyncio\n        self._model = load_pipeline_from_settings(self.hf_settings)\n        self._merge_metadata()\n        print(\"model has been loaded!\")\n        self.ready = True\n        return self.ready\n\n    async def predict(self, payload: InferenceRequest) -> InferenceResponse:\n        \"\"\"\n        TODO\n        \"\"\"\n\n        # Adding some logging as hard to debug given the many types of input accepted\n        logger.debug(\"Payload %s\", payload)\n\n        # TODO: convert and validate?\n        kwargs = self.decode_request(payload, default_codec=HuggingfaceRequestCodec)\n        args = kwargs.pop(\"args\", [])\n\n        array_inputs = kwargs.pop(\"array_inputs\", [])\n        if array_inputs:\n            args = [list(array_inputs)] + args\n        prediction = self._model(*args, **kwargs)\n\n        logger.debug(\"Prediction %s\", prediction)\n\n        return self.encode_response(\n            payload=prediction, default_codec=HuggingfaceRequestCodec\n        )\n\n    def _merge_metadata(self) -> None:\n        meta = METADATA.get(self.hf_settings.task)\n        if meta:\n            self.inputs += meta.get(\"inputs\", [])  # type: ignore\n            self.outputs += meta.get(\"outputs\", [])  # type: ignore\n", "path": "runtimes/huggingface/mlserver_huggingface/runtime.py"}], "after_files": [{"content": "import os\nimport json\nfrom typing import Optional, Dict\nfrom distutils.util import strtobool\n\nimport numpy as np\nfrom pydantic import BaseSettings\nfrom mlserver.errors import MLServerError\n\nfrom transformers.pipelines import pipeline\nfrom transformers.pipelines.base import Pipeline\nfrom transformers.models.auto.tokenization_auto import AutoTokenizer\n\nfrom optimum.pipelines import SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS\n\n\nHUGGINGFACE_TASK_TAG = \"task\"\n\nENV_PREFIX_HUGGINGFACE_SETTINGS = \"MLSERVER_MODEL_HUGGINGFACE_\"\nHUGGINGFACE_PARAMETERS_TAG = \"huggingface_parameters\"\nPARAMETERS_ENV_NAME = \"PREDICTIVE_UNIT_PARAMETERS\"\n\n\nclass InvalidTranformerInitialisation(MLServerError):\n    def __init__(self, code: int, reason: str):\n        super().__init__(\n            f\"Huggingface server failed with {code}, {reason}\",\n            status_code=code,\n        )\n\n\nclass HuggingFaceSettings(BaseSettings):\n    \"\"\"\n    Parameters that apply only to alibi huggingface models\n    \"\"\"\n\n    class Config:\n        env_prefix = ENV_PREFIX_HUGGINGFACE_SETTINGS\n\n    task: str = \"\"\n    # Why need this filed?\n    # for translation task, required a suffix to specify source and target\n    # related issue: https://github.com/SeldonIO/MLServer/issues/947\n    task_suffix: str = \"\"\n    pretrained_model: Optional[str] = None\n    pretrained_tokenizer: Optional[str] = None\n    optimum_model: bool = False\n    device: int = -1\n    batch_size: Optional[int] = None\n\n    @property\n    def task_name(self):\n        if self.task == \"translation\":\n            return f\"{self.task}{self.task_suffix}\"\n        return self.task\n\n\ndef parse_parameters_from_env() -> Dict:\n    \"\"\"\n    TODO\n    \"\"\"\n    parameters = json.loads(os.environ.get(PARAMETERS_ENV_NAME, \"[]\"))\n\n    type_dict = {\n        \"INT\": int,\n        \"FLOAT\": float,\n        \"DOUBLE\": float,\n        \"STRING\": str,\n        \"BOOL\": bool,\n    }\n\n    parsed_parameters = {}\n    for param in parameters:\n        name = param.get(\"name\")\n        value = param.get(\"value\")\n        type_ = param.get(\"type\")\n        if type_ == \"BOOL\":\n            parsed_parameters[name] = bool(strtobool(value))\n        else:\n            try:\n                parsed_parameters[name] = type_dict[type_](value)\n            except ValueError:\n                raise InvalidTranformerInitialisation(\n                    \"Bad model parameter: \"\n                    + name\n                    + \" with value \"\n                    + value\n                    + \" can't be parsed as a \"\n                    + type_,\n                    reason=\"MICROSERVICE_BAD_PARAMETER\",\n                )\n            except KeyError:\n                raise InvalidTranformerInitialisation(\n                    \"Bad model parameter type: \"\n                    + type_\n                    + \" valid are INT, FLOAT, DOUBLE, STRING, BOOL\",\n                    reason=\"MICROSERVICE_BAD_PARAMETER\",\n                )\n    return parsed_parameters\n\n\ndef load_pipeline_from_settings(hf_settings: HuggingFaceSettings) -> Pipeline:\n    \"\"\"\n    TODO\n    \"\"\"\n    # TODO: Support URI for locally downloaded artifacts\n    # uri = model_parameters.uri\n    model = hf_settings.pretrained_model\n    tokenizer = hf_settings.pretrained_tokenizer\n    device = hf_settings.device\n\n    if model and not tokenizer:\n        tokenizer = model\n\n    if hf_settings.optimum_model:\n        optimum_class = SUPPORTED_OPTIMUM_TASKS[hf_settings.task][\"class\"][0]\n        model = optimum_class.from_pretrained(\n            hf_settings.pretrained_model,\n            from_transformers=True,\n        )\n        tokenizer = AutoTokenizer.from_pretrained(tokenizer)\n        # Device needs to be set to -1 due to known issue\n        # https://github.com/huggingface/optimum/issues/191\n        device = -1\n\n    pp = pipeline(\n        hf_settings.task_name,\n        model=model,\n        tokenizer=tokenizer,\n        device=device,\n        batch_size=hf_settings.batch_size,\n    )\n\n    # If batch_size > 0 we need to ensure tokens are padded\n    if hf_settings.batch_size:\n        pp.tokenizer.pad_token_id = [str(pp.model.config.eos_token_id)]  # type: ignore\n\n    return pp\n\n\nclass NumpyEncoder(json.JSONEncoder):\n    def default(self, obj):\n        if isinstance(obj, np.ndarray):\n            return obj.tolist()\n        return json.JSONEncoder.default(self, obj)\n", "path": "runtimes/huggingface/mlserver_huggingface/common.py"}, {"content": "import asyncio\nfrom mlserver.model import MLModel\nfrom mlserver.settings import ModelSettings\nfrom mlserver.types import (\n    InferenceRequest,\n    InferenceResponse,\n)\nfrom mlserver_huggingface.common import (\n    HuggingFaceSettings,\n    parse_parameters_from_env,\n    InvalidTranformerInitialisation,\n    load_pipeline_from_settings,\n)\nfrom mlserver_huggingface.codecs import HuggingfaceRequestCodec\nfrom transformers.pipelines import SUPPORTED_TASKS\nfrom optimum.pipelines import SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS\nfrom mlserver.logging import logger\nfrom mlserver_huggingface.metadata import METADATA\n\n\nclass HuggingFaceRuntime(MLModel):\n    \"\"\"Runtime class for specific Huggingface models\"\"\"\n\n    def __init__(self, settings: ModelSettings):\n        env_params = parse_parameters_from_env()\n        if not env_params and (\n            not settings.parameters or not settings.parameters.extra\n        ):\n            raise InvalidTranformerInitialisation(\n                500,\n                \"Settings parameters not provided via config file nor env variables\",\n            )\n\n        extra = env_params or settings.parameters.extra  # type: ignore\n        self.hf_settings = HuggingFaceSettings(**extra)  # type: ignore\n\n        if self.hf_settings.task not in SUPPORTED_TASKS:\n            raise InvalidTranformerInitialisation(\n                500,\n                (\n                    f\"Invalid transformer task: {self.hf_settings.task}.\"\n                    f\" Available tasks: {SUPPORTED_TASKS.keys()}\"\n                ),\n            )\n\n        if self.hf_settings.optimum_model:\n            if self.hf_settings.task not in SUPPORTED_OPTIMUM_TASKS:\n                raise InvalidTranformerInitialisation(\n                    500,\n                    (\n                        f\"Invalid transformer task for \"\n                        f\"OPTIMUM model: {self.hf_settings.task}. \"\n                        f\"Supported Optimum tasks: {SUPPORTED_OPTIMUM_TASKS.keys()}\"\n                    ),\n                )\n\n        if settings.max_batch_size != self.hf_settings.batch_size:\n            logger.warning(\n                f\"hf batch_size: {self.hf_settings.batch_size} is different \"\n                f\"from MLServer max_batch_size: {settings.max_batch_size}\"\n            )\n\n        super().__init__(settings)\n\n    async def load(self) -> bool:\n        # Loading & caching pipeline in asyncio loop to avoid blocking\n        print(\"=\" * 80)\n        print(self.hf_settings.task_name)\n        print(\"loading model...\")\n        await asyncio.get_running_loop().run_in_executor(\n            None, load_pipeline_from_settings, self.hf_settings\n        )\n        print(\"(re)loading model...\")\n        # Now we load the cached model which should not block asyncio\n        self._model = load_pipeline_from_settings(self.hf_settings)\n        self._merge_metadata()\n        print(\"model has been loaded!\")\n        self.ready = True\n        return self.ready\n\n    async def predict(self, payload: InferenceRequest) -> InferenceResponse:\n        \"\"\"\n        TODO\n        \"\"\"\n\n        # Adding some logging as hard to debug given the many types of input accepted\n        logger.debug(\"Payload %s\", payload)\n\n        # TODO: convert and validate?\n        kwargs = self.decode_request(payload, default_codec=HuggingfaceRequestCodec)\n        args = kwargs.pop(\"args\", [])\n\n        array_inputs = kwargs.pop(\"array_inputs\", [])\n        if array_inputs:\n            args = [list(array_inputs)] + args\n        prediction = self._model(*args, **kwargs)\n\n        logger.debug(\"Prediction %s\", prediction)\n\n        return self.encode_response(\n            payload=prediction, default_codec=HuggingfaceRequestCodec\n        )\n\n    def _merge_metadata(self) -> None:\n        meta = METADATA.get(self.hf_settings.task)\n        if meta:\n            self.inputs += meta.get(\"inputs\", [])  # type: ignore\n            self.outputs += meta.get(\"outputs\", [])  # type: ignore\n", "path": "runtimes/huggingface/mlserver_huggingface/runtime.py"}]}