We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
merging the huggingface and mlserver batch variable
As per https://github.com/SeldonIO/MLServer/pull/740#discussion_r981259626 it would be possible to just merge both huggingface batch variable into the mlserver batch variable for less redundancy in the `model-settings.json` file.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `runtimes/huggingface/mlserver_huggingface/common.py`
Content:
```
1 import os
2 import json
3 from typing import Optional, Dict
4 from distutils.util import strtobool
5 
6 import numpy as np
7 from pydantic import BaseSettings
8 from mlserver.errors import MLServerError
9 
10 from transformers.pipelines import pipeline
11 from transformers.pipelines.base import Pipeline
12 from transformers.models.auto.tokenization_auto import AutoTokenizer
13 
14 try:
15     # Optimum 1.7 changed the import name from `SUPPORTED_TASKS` to
16     # `ORT_SUPPORTED_TASKS`.
17     # We'll try to import the more recent one, falling back to the previous
18     # import name if not present.
19     # https://github.com/huggingface/optimum/blob/987b02e4f6e2a1c9325b364ff764da2e57e89902/optimum/pipelines/__init__.py#L18
20     from optimum.pipelines import ORT_SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS
21 except ImportError:
22     from optimum.pipelines import SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS
23 
24 
25 HUGGINGFACE_TASK_TAG = "task"
26 
27 ENV_PREFIX_HUGGINGFACE_SETTINGS = "MLSERVER_MODEL_HUGGINGFACE_"
28 HUGGINGFACE_PARAMETERS_TAG = "huggingface_parameters"
29 PARAMETERS_ENV_NAME = "PREDICTIVE_UNIT_PARAMETERS"
30 
31 
32 class InvalidTranformerInitialisation(MLServerError):
33     def __init__(self, code: int, reason: str):
34         super().__init__(
35             f"Huggingface server failed with {code}, {reason}",
36             status_code=code,
37         )
38 
39 
40 class HuggingFaceSettings(BaseSettings):
41     """
42     Parameters that apply only to alibi huggingface models
43     """
44 
45     class Config:
46         env_prefix = ENV_PREFIX_HUGGINGFACE_SETTINGS
47 
48     task: str = ""
49     # Why need this filed?
50     # for translation task, required a suffix to specify source and target
51     # related issue: https://github.com/SeldonIO/MLServer/issues/947
52     task_suffix: str = ""
53     pretrained_model: Optional[str] = None
54     pretrained_tokenizer: Optional[str] = None
55     optimum_model: bool = False
56     device: int = -1
57     batch_size: Optional[int] = None
58 
59     @property
60     def task_name(self):
61         if self.task == "translation":
62             return f"{self.task}{self.task_suffix}"
63         return self.task
64 
65 
66 def parse_parameters_from_env() -> Dict:
67     """
68     TODO
69     """
70     parameters = json.loads(os.environ.get(PARAMETERS_ENV_NAME, "[]"))
71 
72     type_dict = {
73         "INT": int,
74         "FLOAT": float,
75         "DOUBLE": float,
76         "STRING": str,
77         "BOOL": bool,
78     }
79 
80     parsed_parameters = {}
81     for param in parameters:
82         name = param.get("name")
83         value = param.get("value")
84         type_ = param.get("type")
85         if type_ == "BOOL":
86             parsed_parameters[name] = bool(strtobool(value))
87         else:
88             try:
89                 parsed_parameters[name] = type_dict[type_](value)
90             except ValueError:
91                 raise InvalidTranformerInitialisation(
92                     "Bad model parameter: "
93                     + name
94                     + " with value "
95                     + value
96                     + " can't be parsed as a "
97                     + type_,
98                     reason="MICROSERVICE_BAD_PARAMETER",
99                 )
100             except KeyError:
101                 raise InvalidTranformerInitialisation(
102                     "Bad model parameter type: "
103                     + type_
104                     + " valid are INT, FLOAT, DOUBLE, STRING, BOOL",
105                     reason="MICROSERVICE_BAD_PARAMETER",
106                 )
107     return parsed_parameters
108 
109 
110 def load_pipeline_from_settings(hf_settings: HuggingFaceSettings) -> Pipeline:
111     """
112     TODO
113     """
114     # TODO: Support URI for locally downloaded artifacts
115     # uri = model_parameters.uri
116     model = hf_settings.pretrained_model
117     tokenizer = hf_settings.pretrained_tokenizer
118     device = hf_settings.device
119 
120     if model and not tokenizer:
121         tokenizer = model
122 
123     if hf_settings.optimum_model:
124         optimum_class = SUPPORTED_OPTIMUM_TASKS[hf_settings.task]["class"][0]
125         model = optimum_class.from_pretrained(
126             hf_settings.pretrained_model,
127             from_transformers=True,
128         )
129         tokenizer = AutoTokenizer.from_pretrained(tokenizer)
130         # Device needs to be set to -1 due to known issue
131         # https://github.com/huggingface/optimum/issues/191
132         device = -1
133 
134     pp = pipeline(
135         hf_settings.task_name,
136         model=model,
137         tokenizer=tokenizer,
138         device=device,
139         batch_size=hf_settings.batch_size,
140     )
141 
142     # If batch_size > 0 we need to ensure tokens are padded
143     if hf_settings.batch_size:
144         pp.tokenizer.pad_token_id = [str(pp.model.config.eos_token_id)]  # type: ignore
145 
146     return pp
147 
148 
149 class NumpyEncoder(json.JSONEncoder):
150     def default(self, obj):
151         if isinstance(obj, np.ndarray):
152             return obj.tolist()
153         return json.JSONEncoder.default(self, obj)
154
```

Path: `runtimes/huggingface/mlserver_huggingface/runtime.py`
Content:
```
1 import asyncio
2 from mlserver.model import MLModel
3 from mlserver.settings import ModelSettings
4 from mlserver.types import (
5     InferenceRequest,
6     InferenceResponse,
7 )
8 from transformers.pipelines import SUPPORTED_TASKS
9 
10 from mlserver.logging import logger
11 
12 from .common import (
13     HuggingFaceSettings,
14     parse_parameters_from_env,
15     InvalidTranformerInitialisation,
16     load_pipeline_from_settings,
17     SUPPORTED_OPTIMUM_TASKS,
18 )
19 from .codecs import HuggingfaceRequestCodec
20 from .metadata import METADATA
21 
22 
23 class HuggingFaceRuntime(MLModel):
24     """Runtime class for specific Huggingface models"""
25 
26     def __init__(self, settings: ModelSettings):
27         env_params = parse_parameters_from_env()
28         if not env_params and (
29             not settings.parameters or not settings.parameters.extra
30         ):
31             raise InvalidTranformerInitialisation(
32                 500,
33                 "Settings parameters not provided via config file nor env variables",
34             )
35 
36         extra = env_params or settings.parameters.extra  # type: ignore
37         self.hf_settings = HuggingFaceSettings(**extra)  # type: ignore
38 
39         if self.hf_settings.task not in SUPPORTED_TASKS:
40             raise InvalidTranformerInitialisation(
41                 500,
42                 (
43                     f"Invalid transformer task: {self.hf_settings.task}."
44                     f" Available tasks: {SUPPORTED_TASKS.keys()}"
45                 ),
46             )
47 
48         if self.hf_settings.optimum_model:
49             if self.hf_settings.task not in SUPPORTED_OPTIMUM_TASKS:
50                 raise InvalidTranformerInitialisation(
51                     500,
52                     (
53                         f"Invalid transformer task for "
54                         f"OPTIMUM model: {self.hf_settings.task}. "
55                         f"Supported Optimum tasks: {SUPPORTED_OPTIMUM_TASKS.keys()}"
56                     ),
57                 )
58 
59         if settings.max_batch_size != self.hf_settings.batch_size:
60             logger.warning(
61                 f"hf batch_size: {self.hf_settings.batch_size} is different "
62                 f"from MLServer max_batch_size: {settings.max_batch_size}"
63             )
64 
65         super().__init__(settings)
66 
67     async def load(self) -> bool:
68         # Loading & caching pipeline in asyncio loop to avoid blocking
69         print("=" * 80)
70         print(self.hf_settings.task_name)
71         print("loading model...")
72         await asyncio.get_running_loop().run_in_executor(
73             None, load_pipeline_from_settings, self.hf_settings
74         )
75         print("(re)loading model...")
76         # Now we load the cached model which should not block asyncio
77         self._model = load_pipeline_from_settings(self.hf_settings)
78         self._merge_metadata()
79         print("model has been loaded!")
80         self.ready = True
81         return self.ready
82 
83     async def predict(self, payload: InferenceRequest) -> InferenceResponse:
84         """
85         TODO
86         """
87 
88         # Adding some logging as hard to debug given the many types of input accepted
89         logger.debug("Payload %s", payload)
90 
91         # TODO: convert and validate?
92         kwargs = self.decode_request(payload, default_codec=HuggingfaceRequestCodec)
93         args = kwargs.pop("args", [])
94 
95         array_inputs = kwargs.pop("array_inputs", [])
96         if array_inputs:
97             args = [list(array_inputs)] + args
98         prediction = self._model(*args, **kwargs)
99 
100         logger.debug("Prediction %s", prediction)
101 
102         return self.encode_response(
103             payload=prediction, default_codec=HuggingfaceRequestCodec
104         )
105 
106     def _merge_metadata(self) -> None:
107         meta = METADATA.get(self.hf_settings.task)
108         if meta:
109             self.inputs += meta.get("inputs", [])  # type: ignore
110             self.outputs += meta.get("outputs", [])  # type: ignore
111
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/runtimes/huggingface/mlserver_huggingface/common.py b/runtimes/huggingface/mlserver_huggingface/common.py
--- a/runtimes/huggingface/mlserver_huggingface/common.py
+++ b/runtimes/huggingface/mlserver_huggingface/common.py
@@ -6,6 +6,7 @@
 import numpy as np
 from pydantic import BaseSettings
 from mlserver.errors import MLServerError
+from mlserver.settings import ModelSettings
 
 from transformers.pipelines import pipeline
 from transformers.pipelines.base import Pipeline
@@ -54,7 +55,6 @@
     pretrained_tokenizer: Optional[str] = None
     optimum_model: bool = False
     device: int = -1
-    batch_size: Optional[int] = None
 
     @property
     def task_name(self):
@@ -107,7 +107,9 @@
     return parsed_parameters
 
 
-def load_pipeline_from_settings(hf_settings: HuggingFaceSettings) -> Pipeline:
+def load_pipeline_from_settings(
+    hf_settings: HuggingFaceSettings, settings: ModelSettings
+) -> Pipeline:
     """
     TODO
     """
@@ -131,16 +133,17 @@
         # https://github.com/huggingface/optimum/issues/191
         device = -1
 
+    batch_size = 1 if settings.max_batch_size == 0 else settings.max_batch_size
     pp = pipeline(
         hf_settings.task_name,
         model=model,
         tokenizer=tokenizer,
         device=device,
-        batch_size=hf_settings.batch_size,
+        batch_size=batch_size,
     )
 
-    # If batch_size > 0 we need to ensure tokens are padded
-    if hf_settings.batch_size:
+    # If max_batch_size > 0 we need to ensure tokens are padded
+    if settings.max_batch_size:
         pp.tokenizer.pad_token_id = [str(pp.model.config.eos_token_id)]  # type: ignore
 
     return pp
diff --git a/runtimes/huggingface/mlserver_huggingface/runtime.py b/runtimes/huggingface/mlserver_huggingface/runtime.py
--- a/runtimes/huggingface/mlserver_huggingface/runtime.py
+++ b/runtimes/huggingface/mlserver_huggingface/runtime.py
@@ -56,12 +56,6 @@
                     ),
                 )
 
-        if settings.max_batch_size != self.hf_settings.batch_size:
-            logger.warning(
-                f"hf batch_size: {self.hf_settings.batch_size} is different "
-                f"from MLServer max_batch_size: {settings.max_batch_size}"
-            )
-
         super().__init__(settings)
 
     async def load(self) -> bool:
@@ -70,11 +64,14 @@
         print(self.hf_settings.task_name)
         print("loading model...")
         await asyncio.get_running_loop().run_in_executor(
-            None, load_pipeline_from_settings, self.hf_settings
+            None,
+            load_pipeline_from_settings,
+            self.hf_settings,
+            self.settings,
         )
         print("(re)loading model...")
         # Now we load the cached model which should not block asyncio
-        self._model = load_pipeline_from_settings(self.hf_settings)
+        self._model = load_pipeline_from_settings(self.hf_settings, self.settings)
         self._merge_metadata()
         print("model has been loaded!")
         self.ready = True

{"golden_diff": "diff --git a/runtimes/huggingface/mlserver_huggingface/common.py b/runtimes/huggingface/mlserver_huggingface/common.py\n--- a/runtimes/huggingface/mlserver_huggingface/common.py\n+++ b/runtimes/huggingface/mlserver_huggingface/common.py\n@@ -6,6 +6,7 @@\n import numpy as np\n from pydantic import BaseSettings\n from mlserver.errors import MLServerError\n+from mlserver.settings import ModelSettings\n \n from transformers.pipelines import pipeline\n from transformers.pipelines.base import Pipeline\n@@ -54,7 +55,6 @@\n     pretrained_tokenizer: Optional[str] = None\n     optimum_model: bool = False\n     device: int = -1\n-    batch_size: Optional[int] = None\n \n     @property\n     def task_name(self):\n@@ -107,7 +107,9 @@\n     return parsed_parameters\n \n \n-def load_pipeline_from_settings(hf_settings: HuggingFaceSettings) -> Pipeline:\n+def load_pipeline_from_settings(\n+    hf_settings: HuggingFaceSettings, settings: ModelSettings\n+) -> Pipeline:\n     \"\"\"\n     TODO\n     \"\"\"\n@@ -131,16 +133,17 @@\n         # https://github.com/huggingface/optimum/issues/191\n         device = -1\n \n+    batch_size = 1 if settings.max_batch_size == 0 else settings.max_batch_size\n     pp = pipeline(\n         hf_settings.task_name,\n         model=model,\n         tokenizer=tokenizer,\n         device=device,\n-        batch_size=hf_settings.batch_size,\n+        batch_size=batch_size,\n     )\n \n-    # If batch_size > 0 we need to ensure tokens are padded\n-    if hf_settings.batch_size:\n+    # If max_batch_size > 0 we need to ensure tokens are padded\n+    if settings.max_batch_size:\n         pp.tokenizer.pad_token_id = [str(pp.model.config.eos_token_id)]  # type: ignore\n \n     return pp\ndiff --git a/runtimes/huggingface/mlserver_huggingface/runtime.py b/runtimes/huggingface/mlserver_huggingface/runtime.py\n--- a/runtimes/huggingface/mlserver_huggingface/runtime.py\n+++ b/runtimes/huggingface/mlserver_huggingface/runtime.py\n@@ -56,12 +56,6 @@\n                     ),\n                 )\n \n-        if settings.max_batch_size != self.hf_settings.batch_size:\n-            logger.warning(\n-                f\"hf batch_size: {self.hf_settings.batch_size} is different \"\n-                f\"from MLServer max_batch_size: {settings.max_batch_size}\"\n-            )\n-\n         super().__init__(settings)\n \n     async def load(self) -> bool:\n@@ -70,11 +64,14 @@\n         print(self.hf_settings.task_name)\n         print(\"loading model...\")\n         await asyncio.get_running_loop().run_in_executor(\n-            None, load_pipeline_from_settings, self.hf_settings\n+            None,\n+            load_pipeline_from_settings,\n+            self.hf_settings,\n+            self.settings,\n         )\n         print(\"(re)loading model...\")\n         # Now we load the cached model which should not block asyncio\n-        self._model = load_pipeline_from_settings(self.hf_settings)\n+        self._model = load_pipeline_from_settings(self.hf_settings, self.settings)\n         self._merge_metadata()\n         print(\"model has been loaded!\")\n         self.ready = True\n", "issue": "merging the huggingface and mlserver batch variable\nAs per https://github.com/SeldonIO/MLServer/pull/740#discussion_r981259626 it would be possible to just merge both huggingface batch variable into the mlserver batch variable for less redundancy in the `model-settings.json` file.\n", "before_files": [{"content": "import os\nimport json\nfrom typing import Optional, Dict\nfrom distutils.util import strtobool\n\nimport numpy as np\nfrom pydantic import BaseSettings\nfrom mlserver.errors import MLServerError\n\nfrom transformers.pipelines import pipeline\nfrom transformers.pipelines.base import Pipeline\nfrom transformers.models.auto.tokenization_auto import AutoTokenizer\n\ntry:\n    # Optimum 1.7 changed the import name from `SUPPORTED_TASKS` to\n    # `ORT_SUPPORTED_TASKS`.\n    # We'll try to import the more recent one, falling back to the previous\n    # import name if not present.\n    # https://github.com/huggingface/optimum/blob/987b02e4f6e2a1c9325b364ff764da2e57e89902/optimum/pipelines/__init__.py#L18\n    from optimum.pipelines import ORT_SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS\nexcept ImportError:\n    from optimum.pipelines import SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS\n\n\nHUGGINGFACE_TASK_TAG = \"task\"\n\nENV_PREFIX_HUGGINGFACE_SETTINGS = \"MLSERVER_MODEL_HUGGINGFACE_\"\nHUGGINGFACE_PARAMETERS_TAG = \"huggingface_parameters\"\nPARAMETERS_ENV_NAME = \"PREDICTIVE_UNIT_PARAMETERS\"\n\n\nclass InvalidTranformerInitialisation(MLServerError):\n    def __init__(self, code: int, reason: str):\n        super().__init__(\n            f\"Huggingface server failed with {code}, {reason}\",\n            status_code=code,\n        )\n\n\nclass HuggingFaceSettings(BaseSettings):\n    \"\"\"\n    Parameters that apply only to alibi huggingface models\n    \"\"\"\n\n    class Config:\n        env_prefix = ENV_PREFIX_HUGGINGFACE_SETTINGS\n\n    task: str = \"\"\n    # Why need this filed?\n    # for translation task, required a suffix to specify source and target\n    # related issue: https://github.com/SeldonIO/MLServer/issues/947\n    task_suffix: str = \"\"\n    pretrained_model: Optional[str] = None\n    pretrained_tokenizer: Optional[str] = None\n    optimum_model: bool = False\n    device: int = -1\n    batch_size: Optional[int] = None\n\n    @property\n    def task_name(self):\n        if self.task == \"translation\":\n            return f\"{self.task}{self.task_suffix}\"\n        return self.task\n\n\ndef parse_parameters_from_env() -> Dict:\n    \"\"\"\n    TODO\n    \"\"\"\n    parameters = json.loads(os.environ.get(PARAMETERS_ENV_NAME, \"[]\"))\n\n    type_dict = {\n        \"INT\": int,\n        \"FLOAT\": float,\n        \"DOUBLE\": float,\n        \"STRING\": str,\n        \"BOOL\": bool,\n    }\n\n    parsed_parameters = {}\n    for param in parameters:\n        name = param.get(\"name\")\n        value = param.get(\"value\")\n        type_ = param.get(\"type\")\n        if type_ == \"BOOL\":\n            parsed_parameters[name] = bool(strtobool(value))\n        else:\n            try:\n                parsed_parameters[name] = type_dict[type_](value)\n            except ValueError:\n                raise InvalidTranformerInitialisation(\n                    \"Bad model parameter: \"\n                    + name\n                    + \" with value \"\n                    + value\n                    + \" can't be parsed as a \"\n                    + type_,\n                    reason=\"MICROSERVICE_BAD_PARAMETER\",\n                )\n            except KeyError:\n                raise InvalidTranformerInitialisation(\n                    \"Bad model parameter type: \"\n                    + type_\n                    + \" valid are INT, FLOAT, DOUBLE, STRING, BOOL\",\n                    reason=\"MICROSERVICE_BAD_PARAMETER\",\n                )\n    return parsed_parameters\n\n\ndef load_pipeline_from_settings(hf_settings: HuggingFaceSettings) -> Pipeline:\n    \"\"\"\n    TODO\n    \"\"\"\n    # TODO: Support URI for locally downloaded artifacts\n    # uri = model_parameters.uri\n    model = hf_settings.pretrained_model\n    tokenizer = hf_settings.pretrained_tokenizer\n    device = hf_settings.device\n\n    if model and not tokenizer:\n        tokenizer = model\n\n    if hf_settings.optimum_model:\n        optimum_class = SUPPORTED_OPTIMUM_TASKS[hf_settings.task][\"class\"][0]\n        model = optimum_class.from_pretrained(\n            hf_settings.pretrained_model,\n            from_transformers=True,\n        )\n        tokenizer = AutoTokenizer.from_pretrained(tokenizer)\n        # Device needs to be set to -1 due to known issue\n        # https://github.com/huggingface/optimum/issues/191\n        device = -1\n\n    pp = pipeline(\n        hf_settings.task_name,\n        model=model,\n        tokenizer=tokenizer,\n        device=device,\n        batch_size=hf_settings.batch_size,\n    )\n\n    # If batch_size > 0 we need to ensure tokens are padded\n    if hf_settings.batch_size:\n        pp.tokenizer.pad_token_id = [str(pp.model.config.eos_token_id)]  # type: ignore\n\n    return pp\n\n\nclass NumpyEncoder(json.JSONEncoder):\n    def default(self, obj):\n        if isinstance(obj, np.ndarray):\n            return obj.tolist()\n        return json.JSONEncoder.default(self, obj)\n", "path": "runtimes/huggingface/mlserver_huggingface/common.py"}, {"content": "import asyncio\nfrom mlserver.model import MLModel\nfrom mlserver.settings import ModelSettings\nfrom mlserver.types import (\n    InferenceRequest,\n    InferenceResponse,\n)\nfrom transformers.pipelines import SUPPORTED_TASKS\n\nfrom mlserver.logging import logger\n\nfrom .common import (\n    HuggingFaceSettings,\n    parse_parameters_from_env,\n    InvalidTranformerInitialisation,\n    load_pipeline_from_settings,\n    SUPPORTED_OPTIMUM_TASKS,\n)\nfrom .codecs import HuggingfaceRequestCodec\nfrom .metadata import METADATA\n\n\nclass HuggingFaceRuntime(MLModel):\n    \"\"\"Runtime class for specific Huggingface models\"\"\"\n\n    def __init__(self, settings: ModelSettings):\n        env_params = parse_parameters_from_env()\n        if not env_params and (\n            not settings.parameters or not settings.parameters.extra\n        ):\n            raise InvalidTranformerInitialisation(\n                500,\n                \"Settings parameters not provided via config file nor env variables\",\n            )\n\n        extra = env_params or settings.parameters.extra  # type: ignore\n        self.hf_settings = HuggingFaceSettings(**extra)  # type: ignore\n\n        if self.hf_settings.task not in SUPPORTED_TASKS:\n            raise InvalidTranformerInitialisation(\n                500,\n                (\n                    f\"Invalid transformer task: {self.hf_settings.task}.\"\n                    f\" Available tasks: {SUPPORTED_TASKS.keys()}\"\n                ),\n            )\n\n        if self.hf_settings.optimum_model:\n            if self.hf_settings.task not in SUPPORTED_OPTIMUM_TASKS:\n                raise InvalidTranformerInitialisation(\n                    500,\n                    (\n                        f\"Invalid transformer task for \"\n                        f\"OPTIMUM model: {self.hf_settings.task}. \"\n                        f\"Supported Optimum tasks: {SUPPORTED_OPTIMUM_TASKS.keys()}\"\n                    ),\n                )\n\n        if settings.max_batch_size != self.hf_settings.batch_size:\n            logger.warning(\n                f\"hf batch_size: {self.hf_settings.batch_size} is different \"\n                f\"from MLServer max_batch_size: {settings.max_batch_size}\"\n            )\n\n        super().__init__(settings)\n\n    async def load(self) -> bool:\n        # Loading & caching pipeline in asyncio loop to avoid blocking\n        print(\"=\" * 80)\n        print(self.hf_settings.task_name)\n        print(\"loading model...\")\n        await asyncio.get_running_loop().run_in_executor(\n            None, load_pipeline_from_settings, self.hf_settings\n        )\n        print(\"(re)loading model...\")\n        # Now we load the cached model which should not block asyncio\n        self._model = load_pipeline_from_settings(self.hf_settings)\n        self._merge_metadata()\n        print(\"model has been loaded!\")\n        self.ready = True\n        return self.ready\n\n    async def predict(self, payload: InferenceRequest) -> InferenceResponse:\n        \"\"\"\n        TODO\n        \"\"\"\n\n        # Adding some logging as hard to debug given the many types of input accepted\n        logger.debug(\"Payload %s\", payload)\n\n        # TODO: convert and validate?\n        kwargs = self.decode_request(payload, default_codec=HuggingfaceRequestCodec)\n        args = kwargs.pop(\"args\", [])\n\n        array_inputs = kwargs.pop(\"array_inputs\", [])\n        if array_inputs:\n            args = [list(array_inputs)] + args\n        prediction = self._model(*args, **kwargs)\n\n        logger.debug(\"Prediction %s\", prediction)\n\n        return self.encode_response(\n            payload=prediction, default_codec=HuggingfaceRequestCodec\n        )\n\n    def _merge_metadata(self) -> None:\n        meta = METADATA.get(self.hf_settings.task)\n        if meta:\n            self.inputs += meta.get(\"inputs\", [])  # type: ignore\n            self.outputs += meta.get(\"outputs\", [])  # type: ignore\n", "path": "runtimes/huggingface/mlserver_huggingface/runtime.py"}], "after_files": [{"content": "import os\nimport json\nfrom typing import Optional, Dict\nfrom distutils.util import strtobool\n\nimport numpy as np\nfrom pydantic import BaseSettings\nfrom mlserver.errors import MLServerError\nfrom mlserver.settings import ModelSettings\n\nfrom transformers.pipelines import pipeline\nfrom transformers.pipelines.base import Pipeline\nfrom transformers.models.auto.tokenization_auto import AutoTokenizer\n\ntry:\n    # Optimum 1.7 changed the import name from `SUPPORTED_TASKS` to\n    # `ORT_SUPPORTED_TASKS`.\n    # We'll try to import the more recent one, falling back to the previous\n    # import name if not present.\n    # https://github.com/huggingface/optimum/blob/987b02e4f6e2a1c9325b364ff764da2e57e89902/optimum/pipelines/__init__.py#L18\n    from optimum.pipelines import ORT_SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS\nexcept ImportError:\n    from optimum.pipelines import SUPPORTED_TASKS as SUPPORTED_OPTIMUM_TASKS\n\n\nHUGGINGFACE_TASK_TAG = \"task\"\n\nENV_PREFIX_HUGGINGFACE_SETTINGS = \"MLSERVER_MODEL_HUGGINGFACE_\"\nHUGGINGFACE_PARAMETERS_TAG = \"huggingface_parameters\"\nPARAMETERS_ENV_NAME = \"PREDICTIVE_UNIT_PARAMETERS\"\n\n\nclass InvalidTranformerInitialisation(MLServerError):\n    def __init__(self, code: int, reason: str):\n        super().__init__(\n            f\"Huggingface server failed with {code}, {reason}\",\n            status_code=code,\n        )\n\n\nclass HuggingFaceSettings(BaseSettings):\n    \"\"\"\n    Parameters that apply only to alibi huggingface models\n    \"\"\"\n\n    class Config:\n        env_prefix = ENV_PREFIX_HUGGINGFACE_SETTINGS\n\n    task: str = \"\"\n    # Why need this filed?\n    # for translation task, required a suffix to specify source and target\n    # related issue: https://github.com/SeldonIO/MLServer/issues/947\n    task_suffix: str = \"\"\n    pretrained_model: Optional[str] = None\n    pretrained_tokenizer: Optional[str] = None\n    optimum_model: bool = False\n    device: int = -1\n\n    @property\n    def task_name(self):\n        if self.task == \"translation\":\n            return f\"{self.task}{self.task_suffix}\"\n        return self.task\n\n\ndef parse_parameters_from_env() -> Dict:\n    \"\"\"\n    TODO\n    \"\"\"\n    parameters = json.loads(os.environ.get(PARAMETERS_ENV_NAME, \"[]\"))\n\n    type_dict = {\n        \"INT\": int,\n        \"FLOAT\": float,\n        \"DOUBLE\": float,\n        \"STRING\": str,\n        \"BOOL\": bool,\n    }\n\n    parsed_parameters = {}\n    for param in parameters:\n        name = param.get(\"name\")\n        value = param.get(\"value\")\n        type_ = param.get(\"type\")\n        if type_ == \"BOOL\":\n            parsed_parameters[name] = bool(strtobool(value))\n        else:\n            try:\n                parsed_parameters[name] = type_dict[type_](value)\n            except ValueError:\n                raise InvalidTranformerInitialisation(\n                    \"Bad model parameter: \"\n                    + name\n                    + \" with value \"\n                    + value\n                    + \" can't be parsed as a \"\n                    + type_,\n                    reason=\"MICROSERVICE_BAD_PARAMETER\",\n                )\n            except KeyError:\n                raise InvalidTranformerInitialisation(\n                    \"Bad model parameter type: \"\n                    + type_\n                    + \" valid are INT, FLOAT, DOUBLE, STRING, BOOL\",\n                    reason=\"MICROSERVICE_BAD_PARAMETER\",\n                )\n    return parsed_parameters\n\n\ndef load_pipeline_from_settings(\n    hf_settings: HuggingFaceSettings, settings: ModelSettings\n) -> Pipeline:\n    \"\"\"\n    TODO\n    \"\"\"\n    # TODO: Support URI for locally downloaded artifacts\n    # uri = model_parameters.uri\n    model = hf_settings.pretrained_model\n    tokenizer = hf_settings.pretrained_tokenizer\n    device = hf_settings.device\n\n    if model and not tokenizer:\n        tokenizer = model\n\n    if hf_settings.optimum_model:\n        optimum_class = SUPPORTED_OPTIMUM_TASKS[hf_settings.task][\"class\"][0]\n        model = optimum_class.from_pretrained(\n            hf_settings.pretrained_model,\n            from_transformers=True,\n        )\n        tokenizer = AutoTokenizer.from_pretrained(tokenizer)\n        # Device needs to be set to -1 due to known issue\n        # https://github.com/huggingface/optimum/issues/191\n        device = -1\n\n    batch_size = 1 if settings.max_batch_size == 0 else settings.max_batch_size\n    pp = pipeline(\n        hf_settings.task_name,\n        model=model,\n        tokenizer=tokenizer,\n        device=device,\n        batch_size=batch_size,\n    )\n\n    # If max_batch_size > 0 we need to ensure tokens are padded\n    if settings.max_batch_size:\n        pp.tokenizer.pad_token_id = [str(pp.model.config.eos_token_id)]  # type: ignore\n\n    return pp\n\n\nclass NumpyEncoder(json.JSONEncoder):\n    def default(self, obj):\n        if isinstance(obj, np.ndarray):\n            return obj.tolist()\n        return json.JSONEncoder.default(self, obj)\n", "path": "runtimes/huggingface/mlserver_huggingface/common.py"}, {"content": "import asyncio\nfrom mlserver.model import MLModel\nfrom mlserver.settings import ModelSettings\nfrom mlserver.types import (\n    InferenceRequest,\n    InferenceResponse,\n)\nfrom transformers.pipelines import SUPPORTED_TASKS\n\nfrom mlserver.logging import logger\n\nfrom .common import (\n    HuggingFaceSettings,\n    parse_parameters_from_env,\n    InvalidTranformerInitialisation,\n    load_pipeline_from_settings,\n    SUPPORTED_OPTIMUM_TASKS,\n)\nfrom .codecs import HuggingfaceRequestCodec\nfrom .metadata import METADATA\n\n\nclass HuggingFaceRuntime(MLModel):\n    \"\"\"Runtime class for specific Huggingface models\"\"\"\n\n    def __init__(self, settings: ModelSettings):\n        env_params = parse_parameters_from_env()\n        if not env_params and (\n            not settings.parameters or not settings.parameters.extra\n        ):\n            raise InvalidTranformerInitialisation(\n                500,\n                \"Settings parameters not provided via config file nor env variables\",\n            )\n\n        extra = env_params or settings.parameters.extra  # type: ignore\n        self.hf_settings = HuggingFaceSettings(**extra)  # type: ignore\n\n        if self.hf_settings.task not in SUPPORTED_TASKS:\n            raise InvalidTranformerInitialisation(\n                500,\n                (\n                    f\"Invalid transformer task: {self.hf_settings.task}.\"\n                    f\" Available tasks: {SUPPORTED_TASKS.keys()}\"\n                ),\n            )\n\n        if self.hf_settings.optimum_model:\n            if self.hf_settings.task not in SUPPORTED_OPTIMUM_TASKS:\n                raise InvalidTranformerInitialisation(\n                    500,\n                    (\n                        f\"Invalid transformer task for \"\n                        f\"OPTIMUM model: {self.hf_settings.task}. \"\n                        f\"Supported Optimum tasks: {SUPPORTED_OPTIMUM_TASKS.keys()}\"\n                    ),\n                )\n\n        super().__init__(settings)\n\n    async def load(self) -> bool:\n        # Loading & caching pipeline in asyncio loop to avoid blocking\n        print(\"=\" * 80)\n        print(self.hf_settings.task_name)\n        print(\"loading model...\")\n        await asyncio.get_running_loop().run_in_executor(\n            None,\n            load_pipeline_from_settings,\n            self.hf_settings,\n            self.settings,\n        )\n        print(\"(re)loading model...\")\n        # Now we load the cached model which should not block asyncio\n        self._model = load_pipeline_from_settings(self.hf_settings, self.settings)\n        self._merge_metadata()\n        print(\"model has been loaded!\")\n        self.ready = True\n        return self.ready\n\n    async def predict(self, payload: InferenceRequest) -> InferenceResponse:\n        \"\"\"\n        TODO\n        \"\"\"\n\n        # Adding some logging as hard to debug given the many types of input accepted\n        logger.debug(\"Payload %s\", payload)\n\n        # TODO: convert and validate?\n        kwargs = self.decode_request(payload, default_codec=HuggingfaceRequestCodec)\n        args = kwargs.pop(\"args\", [])\n\n        array_inputs = kwargs.pop(\"array_inputs\", [])\n        if array_inputs:\n            args = [list(array_inputs)] + args\n        prediction = self._model(*args, **kwargs)\n\n        logger.debug(\"Prediction %s\", prediction)\n\n        return self.encode_response(\n            payload=prediction, default_codec=HuggingfaceRequestCodec\n        )\n\n    def _merge_metadata(self) -> None:\n        meta = METADATA.get(self.hf_settings.task)\n        if meta:\n            self.inputs += meta.get(\"inputs\", [])  # type: ignore\n            self.outputs += meta.get(\"outputs\", [])  # type: ignore\n", "path": "runtimes/huggingface/mlserver_huggingface/runtime.py"}]}