We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Allow configuring metrics endpoint name
Currently, the metrics endpoint is hardcoded to `/metrics`. It would be useful to support configuring this value, so that it can be matched by what Prometheus is expecting upstream.

For example, in Seldon Core this would something like `/prometheus` by default.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `mlserver/grpc/server.py`
Content:
```
1 from grpc import aio
2 from concurrent.futures import ThreadPoolExecutor
3 from typing import Any, List, Tuple
4 
5 from ..handlers import DataPlane, ModelRepositoryHandlers
6 from ..settings import Settings
7 
8 from .servicers import InferenceServicer, ModelRepositoryServicer
9 from .dataplane_pb2_grpc import add_GRPCInferenceServiceServicer_to_server
10 from .model_repository_pb2_grpc import add_ModelRepositoryServiceServicer_to_server
11 from .interceptors import LoggingInterceptor, PromServerInterceptor
12 from .logging import logger
13 
14 # Workers used for non-AsyncIO workloads (which aren't any in our case)
15 DefaultGrpcWorkers = 5
16 
17 
18 class GRPCServer:
19     def __init__(
20         self,
21         settings: Settings,
22         data_plane: DataPlane,
23         model_repository_handlers: ModelRepositoryHandlers,
24     ):
25         self._settings = settings
26         self._data_plane = data_plane
27         self._model_repository_handlers = model_repository_handlers
28 
29     def _create_server(self):
30         self._inference_servicer = InferenceServicer(self._data_plane)
31         self._model_repository_servicer = ModelRepositoryServicer(
32             self._model_repository_handlers
33         )
34 
35         logging_interceptor = LoggingInterceptor()
36         prom_interceptor = PromServerInterceptor()
37         self._server = aio.server(
38             ThreadPoolExecutor(max_workers=DefaultGrpcWorkers),
39             interceptors=(prom_interceptor, logging_interceptor),
40             options=self._get_options(),
41         )
42 
43         add_GRPCInferenceServiceServicer_to_server(
44             self._inference_servicer, self._server
45         )
46         add_ModelRepositoryServiceServicer_to_server(
47             self._model_repository_servicer, self._server
48         )
49 
50         self._server.add_insecure_port(
51             f"{self._settings.host}:{self._settings.grpc_port}"
52         )
53 
54         return self._server
55 
56     def _get_options(self) -> List[Tuple[str, Any]]:
57         options = []
58 
59         max_message_length = self._settings.grpc_max_message_length
60         if max_message_length is not None:
61             options += [
62                 ("grpc.max_message_length", max_message_length),
63                 ("grpc.max_send_message_length", max_message_length),
64                 ("grpc.max_receive_message_length", max_message_length),
65             ]
66 
67         return options
68 
69     async def start(self):
70         self._create_server()
71 
72         await self._server.start()
73 
74         logger.info(
75             "gRPC server running on "
76             f"http://{self._settings.host}:{self._settings.grpc_port}"
77         )
78         await self._server.wait_for_termination()
79 
80     async def stop(self):
81         # TODO: Read from config
82         await self._server.stop(grace=5)
83
```

Path: `mlserver/rest/app.py`
Content:
```
1 from typing import Callable
2 from fastapi import FastAPI
3 from fastapi.responses import Response as FastAPIResponse
4 from fastapi.routing import APIRoute as FastAPIRoute
5 from fastapi.middleware.cors import CORSMiddleware
6 from starlette_exporter import PrometheusMiddleware, handle_metrics
7 
8 from .endpoints import Endpoints, ModelRepositoryEndpoints
9 from .requests import Request
10 from .responses import Response
11 from .errors import _EXCEPTION_HANDLERS
12 
13 from ..settings import Settings
14 from ..handlers import DataPlane, ModelRepositoryHandlers
15 
16 
17 class APIRoute(FastAPIRoute):
18     """
19     Custom route to use our own Request handler.
20     """
21 
22     def get_route_handler(self) -> Callable:
23         original_route_handler = super().get_route_handler()
24 
25         async def custom_route_handler(request: Request) -> FastAPIResponse:
26             request = Request(request.scope, request.receive)
27             return await original_route_handler(request)
28 
29         return custom_route_handler
30 
31 
32 def create_app(
33     settings: Settings,
34     data_plane: DataPlane,
35     model_repository_handlers: ModelRepositoryHandlers,
36 ) -> FastAPI:
37     endpoints = Endpoints(data_plane)
38     model_repository_endpoints = ModelRepositoryEndpoints(model_repository_handlers)
39 
40     routes = [
41         # Model ready
42         APIRoute(
43             "/v2/models/{model_name}/ready",
44             endpoints.model_ready,
45         ),
46         APIRoute(
47             "/v2/models/{model_name}/versions/{model_version}/ready",
48             endpoints.model_ready,
49         ),
50         # Model infer
51         APIRoute(
52             "/v2/models/{model_name}/infer",
53             endpoints.infer,
54             methods=["POST"],
55         ),
56         APIRoute(
57             "/v2/models/{model_name}/versions/{model_version}/infer",
58             endpoints.infer,
59             methods=["POST"],
60         ),
61         # Model metadata
62         APIRoute(
63             "/v2/models/{model_name}",
64             endpoints.model_metadata,
65         ),
66         APIRoute(
67             "/v2/models/{model_name}/versions/{model_version}",
68             endpoints.model_metadata,
69         ),
70         # Liveness and readiness
71         APIRoute("/v2/health/live", endpoints.live),
72         APIRoute("/v2/health/ready", endpoints.ready),
73         # Server metadata
74         APIRoute(
75             "/v2",
76             endpoints.metadata,
77         ),
78     ]
79 
80     routes += [
81         # Model Repository API
82         APIRoute(
83             "/v2/repository/index",
84             model_repository_endpoints.index,
85             methods=["POST"],
86         ),
87         APIRoute(
88             "/v2/repository/models/{model_name}/load",
89             model_repository_endpoints.load,
90             methods=["POST"],
91         ),
92         APIRoute(
93             "/v2/repository/models/{model_name}/unload",
94             model_repository_endpoints.unload,
95             methods=["POST"],
96         ),
97     ]
98 
99     app = FastAPI(
100         debug=settings.debug,
101         routes=routes,  # type: ignore
102         default_response_class=Response,
103         exception_handlers=_EXCEPTION_HANDLERS,  # type: ignore
104     )
105 
106     if settings.cors_settings is not None:
107         app.add_middleware(
108             CORSMiddleware,
109             allow_origins=settings.cors_settings.allow_origins,
110             allow_origin_regex=settings.cors_settings.allow_origin_regex,
111             allow_credentials=settings.cors_settings.allow_credentials,
112             allow_methods=settings.cors_settings.allow_methods,
113             allow_headers=settings.cors_settings.allow_headers,
114             max_age=settings.cors_settings.max_age,
115         )
116 
117     app.add_middleware(
118         PrometheusMiddleware,
119         app_name="mlserver",
120         prefix="rest_server",
121         # TODO: Should we also exclude model's health endpoints?
122         skip_paths=["/metrics", "/v2/health/live", "/v2/health/ready"],
123     )
124     app.add_route("/metrics", handle_metrics)
125 
126     return app
127
```

Path: `mlserver/settings.py`
Content:
```
1 from typing import List, Optional
2 from pydantic import BaseSettings, PyObject
3 
4 from .version import __version__
5 from .types import MetadataTensor
6 
7 ENV_PREFIX_SETTINGS = "MLSERVER_"
8 ENV_PREFIX_MODEL_SETTINGS = "MLSERVER_MODEL_"
9 
10 
11 class CORSSettings(BaseSettings):
12     class Config:
13         env_prefix = ENV_PREFIX_SETTINGS
14 
15     """
16     A list of origins that should be permitted to make
17     cross-origin requests. E.g. ['https://example.org', 'https://www.example.org'].
18     You can use ['*'] to allow any origin
19     """
20     allow_origins: Optional[List[str]] = []
21 
22     """
23     A regex string to match against origins that
24     should be permitted to make cross-origin requests.
25     e.g. 'https:\\/\\/.*\\.example\\.org'
26     """
27     allow_origin_regex: Optional[str] = None
28 
29     """Indicate that cookies should be supported for cross-origin requests"""
30     allow_credentials: Optional[bool] = False
31 
32     """A list of HTTP methods that should be allowed for cross-origin requests"""
33     allow_methods: Optional[List[str]] = ["GET"]
34 
35     """A list of HTTP request headers that should be supported for
36     cross-origin requests"""
37     allow_headers: Optional[List[str]] = []
38 
39     """Indicate any response headers that should be made accessible to the browser"""
40     expose_headers: Optional[List[str]] = []
41 
42     """Sets a maximum time in seconds for browsers to cache CORS responses"""
43     max_age: Optional[int] = 600
44 
45 
46 class Settings(BaseSettings):
47     class Config:
48         env_prefix = ENV_PREFIX_SETTINGS
49 
50     debug: bool = True
51 
52     # Model repository settings
53     """Root of the model repository, where we will search for models."""
54     model_repository_root: str = "."
55     """Flag to load all available models automatically at startup."""
56     load_models_at_startup: bool = True
57 
58     # Server metadata
59     """Name of the server."""
60     server_name: str = "mlserver"
61     """Version of the server."""
62     server_version: str = __version__
63     """Server extensions loaded."""
64     extensions: List[str] = []
65 
66     # Server settings
67     """Host where to listen for connections."""
68     host: str = "0.0.0.0"
69     """Port where to listen for HTTP / REST connections."""
70     http_port: int = 8080
71     """Port where to listen for gRPC connections."""
72     grpc_port: int = 8081
73     """Maximum length (i.e. size) of gRPC payloads."""
74     grpc_max_message_length: Optional[int] = None
75 
76     # CORS settings
77     cors_settings: Optional[CORSSettings] = None
78 
79 
80 class ModelParameters(BaseSettings):
81     """
82     Parameters that apply only to a particular instance of a model.
83     This can include things like model weights, or arbitrary ``extra``
84     parameters particular to the underlying inference runtime.
85     The main difference with respect to ``ModelSettings`` is that parameters
86     can change on each instance (e.g. each version) of the model.
87     """
88 
89     class Config:
90         env_prefix = ENV_PREFIX_MODEL_SETTINGS
91 
92     """
93     URI where the model artifacts can be found.
94     This path must be either absolute or relative to where MLServer is running.
95     """
96     uri: Optional[str] = None
97     """Version of the model."""
98     version: Optional[str] = None
99     """Format of the model (only available on certain runtimes)."""
100     format: Optional[str] = None
101     """Default content type to use for requests and responses."""
102     content_type: Optional[str] = None
103     """Arbitrary settings, dependent on the inference runtime
104     implementation."""
105     extra: Optional[dict] = {}
106 
107 
108 class ModelSettings(BaseSettings):
109     class Config:
110         env_prefix = ENV_PREFIX_MODEL_SETTINGS
111         underscore_attrs_are_private = True
112 
113     # Source points to the file where model settings were loaded from
114     _source: Optional[str] = None
115 
116     """Name of the model."""
117     name: str = ""
118 
119     # Model metadata
120     """Framework used to train and serialise the model (e.g. sklearn)."""
121     platform: str = ""
122     """Versions of dependencies used to train the model (e.g.
123     sklearn/0.20.1)."""
124     versions: List[str] = []
125     """Metadata about the inputs accepted by the model."""
126     inputs: List[MetadataTensor] = []
127     """Metadata about the outputs returned by the model."""
128     outputs: List[MetadataTensor] = []
129 
130     # Parallel settings
131     """When parallel inference is enabled, number of workers to run inference
132     across."""
133     parallel_workers: int = 4
134 
135     # Adaptive Batching settings (disabled by default)
136     """When adaptive batching is enabled, maximum number of requests to group
137     together in a single batch."""
138     max_batch_size: int = 0
139     """When adaptive batching is enabled, maximum amount of time (in seconds)
140     to wait for enough requests to build a full batch."""
141     max_batch_time: float = 0.0
142 
143     # Custom model class implementation
144     """*Python path* to the inference runtime to use to serve this model (e.g.
145     ``mlserver_sklearn.SKLearnModel``)."""
146     implementation: PyObject = "mlserver.model.MLModel"  # type: ignore
147 
148     # Model parameters are meant to be set directly by the MLServer runtime.
149     # However, it's also possible to override them manually.
150     """Extra parameters for each instance of this model."""
151     parameters: Optional[ModelParameters] = None
152
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/mlserver/grpc/server.py b/mlserver/grpc/server.py
--- a/mlserver/grpc/server.py
+++ b/mlserver/grpc/server.py
@@ -32,11 +32,14 @@
             self._model_repository_handlers
         )
 
-        logging_interceptor = LoggingInterceptor()
-        prom_interceptor = PromServerInterceptor()
+        interceptors = [LoggingInterceptor()]
+
+        if self._settings.metrics_endpoint:
+            interceptors.append(PromServerInterceptor())
+
         self._server = aio.server(
             ThreadPoolExecutor(max_workers=DefaultGrpcWorkers),
-            interceptors=(prom_interceptor, logging_interceptor),
+            interceptors=tuple(interceptors),
             options=self._get_options(),
         )
 
diff --git a/mlserver/rest/app.py b/mlserver/rest/app.py
--- a/mlserver/rest/app.py
+++ b/mlserver/rest/app.py
@@ -114,13 +114,18 @@
             max_age=settings.cors_settings.max_age,
         )
 
-    app.add_middleware(
-        PrometheusMiddleware,
-        app_name="mlserver",
-        prefix="rest_server",
-        # TODO: Should we also exclude model's health endpoints?
-        skip_paths=["/metrics", "/v2/health/live", "/v2/health/ready"],
-    )
-    app.add_route("/metrics", handle_metrics)
+    if settings.metrics_endpoint:
+        app.add_middleware(
+            PrometheusMiddleware,
+            app_name="mlserver",
+            prefix="rest_server",
+            # TODO: Should we also exclude model's health endpoints?
+            skip_paths=[
+                settings.metrics_endpoint,
+                "/v2/health/live",
+                "/v2/health/ready",
+            ],
+        )
+        app.add_route(settings.metrics_endpoint, handle_metrics)
 
     return app
diff --git a/mlserver/settings.py b/mlserver/settings.py
--- a/mlserver/settings.py
+++ b/mlserver/settings.py
@@ -76,6 +76,13 @@
     # CORS settings
     cors_settings: Optional[CORSSettings] = None
 
+    # Metrics settings
+    """
+    Endpoint used to expose Prometheus metrics. Alternatively, can be set to
+    `None` to disable it
+    """
+    metrics_endpoint: Optional[str] = "/metrics"
+
 
 class ModelParameters(BaseSettings):
     """

{"golden_diff": "diff --git a/mlserver/grpc/server.py b/mlserver/grpc/server.py\n--- a/mlserver/grpc/server.py\n+++ b/mlserver/grpc/server.py\n@@ -32,11 +32,14 @@\n             self._model_repository_handlers\n         )\n \n-        logging_interceptor = LoggingInterceptor()\n-        prom_interceptor = PromServerInterceptor()\n+        interceptors = [LoggingInterceptor()]\n+\n+        if self._settings.metrics_endpoint:\n+            interceptors.append(PromServerInterceptor())\n+\n         self._server = aio.server(\n             ThreadPoolExecutor(max_workers=DefaultGrpcWorkers),\n-            interceptors=(prom_interceptor, logging_interceptor),\n+            interceptors=tuple(interceptors),\n             options=self._get_options(),\n         )\n \ndiff --git a/mlserver/rest/app.py b/mlserver/rest/app.py\n--- a/mlserver/rest/app.py\n+++ b/mlserver/rest/app.py\n@@ -114,13 +114,18 @@\n             max_age=settings.cors_settings.max_age,\n         )\n \n-    app.add_middleware(\n-        PrometheusMiddleware,\n-        app_name=\"mlserver\",\n-        prefix=\"rest_server\",\n-        # TODO: Should we also exclude model's health endpoints?\n-        skip_paths=[\"/metrics\", \"/v2/health/live\", \"/v2/health/ready\"],\n-    )\n-    app.add_route(\"/metrics\", handle_metrics)\n+    if settings.metrics_endpoint:\n+        app.add_middleware(\n+            PrometheusMiddleware,\n+            app_name=\"mlserver\",\n+            prefix=\"rest_server\",\n+            # TODO: Should we also exclude model's health endpoints?\n+            skip_paths=[\n+                settings.metrics_endpoint,\n+                \"/v2/health/live\",\n+                \"/v2/health/ready\",\n+            ],\n+        )\n+        app.add_route(settings.metrics_endpoint, handle_metrics)\n \n     return app\ndiff --git a/mlserver/settings.py b/mlserver/settings.py\n--- a/mlserver/settings.py\n+++ b/mlserver/settings.py\n@@ -76,6 +76,13 @@\n     # CORS settings\n     cors_settings: Optional[CORSSettings] = None\n \n+    # Metrics settings\n+    \"\"\"\n+    Endpoint used to expose Prometheus metrics. Alternatively, can be set to\n+    `None` to disable it\n+    \"\"\"\n+    metrics_endpoint: Optional[str] = \"/metrics\"\n+\n \n class ModelParameters(BaseSettings):\n     \"\"\"\n", "issue": "Allow configuring metrics endpoint name\nCurrently, the metrics endpoint is hardcoded to `/metrics`. It would be useful to support configuring this value, so that it can be matched by what Prometheus is expecting upstream.\r\n\r\nFor example, in Seldon Core this would something like `/prometheus` by default.\n", "before_files": [{"content": "from grpc import aio\nfrom concurrent.futures import ThreadPoolExecutor\nfrom typing import Any, List, Tuple\n\nfrom ..handlers import DataPlane, ModelRepositoryHandlers\nfrom ..settings import Settings\n\nfrom .servicers import InferenceServicer, ModelRepositoryServicer\nfrom .dataplane_pb2_grpc import add_GRPCInferenceServiceServicer_to_server\nfrom .model_repository_pb2_grpc import add_ModelRepositoryServiceServicer_to_server\nfrom .interceptors import LoggingInterceptor, PromServerInterceptor\nfrom .logging import logger\n\n# Workers used for non-AsyncIO workloads (which aren't any in our case)\nDefaultGrpcWorkers = 5\n\n\nclass GRPCServer:\n    def __init__(\n        self,\n        settings: Settings,\n        data_plane: DataPlane,\n        model_repository_handlers: ModelRepositoryHandlers,\n    ):\n        self._settings = settings\n        self._data_plane = data_plane\n        self._model_repository_handlers = model_repository_handlers\n\n    def _create_server(self):\n        self._inference_servicer = InferenceServicer(self._data_plane)\n        self._model_repository_servicer = ModelRepositoryServicer(\n            self._model_repository_handlers\n        )\n\n        logging_interceptor = LoggingInterceptor()\n        prom_interceptor = PromServerInterceptor()\n        self._server = aio.server(\n            ThreadPoolExecutor(max_workers=DefaultGrpcWorkers),\n            interceptors=(prom_interceptor, logging_interceptor),\n            options=self._get_options(),\n        )\n\n        add_GRPCInferenceServiceServicer_to_server(\n            self._inference_servicer, self._server\n        )\n        add_ModelRepositoryServiceServicer_to_server(\n            self._model_repository_servicer, self._server\n        )\n\n        self._server.add_insecure_port(\n            f\"{self._settings.host}:{self._settings.grpc_port}\"\n        )\n\n        return self._server\n\n    def _get_options(self) -> List[Tuple[str, Any]]:\n        options = []\n\n        max_message_length = self._settings.grpc_max_message_length\n        if max_message_length is not None:\n            options += [\n                (\"grpc.max_message_length\", max_message_length),\n                (\"grpc.max_send_message_length\", max_message_length),\n                (\"grpc.max_receive_message_length\", max_message_length),\n            ]\n\n        return options\n\n    async def start(self):\n        self._create_server()\n\n        await self._server.start()\n\n        logger.info(\n            \"gRPC server running on \"\n            f\"http://{self._settings.host}:{self._settings.grpc_port}\"\n        )\n        await self._server.wait_for_termination()\n\n    async def stop(self):\n        # TODO: Read from config\n        await self._server.stop(grace=5)\n", "path": "mlserver/grpc/server.py"}, {"content": "from typing import Callable\nfrom fastapi import FastAPI\nfrom fastapi.responses import Response as FastAPIResponse\nfrom fastapi.routing import APIRoute as FastAPIRoute\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom starlette_exporter import PrometheusMiddleware, handle_metrics\n\nfrom .endpoints import Endpoints, ModelRepositoryEndpoints\nfrom .requests import Request\nfrom .responses import Response\nfrom .errors import _EXCEPTION_HANDLERS\n\nfrom ..settings import Settings\nfrom ..handlers import DataPlane, ModelRepositoryHandlers\n\n\nclass APIRoute(FastAPIRoute):\n    \"\"\"\n    Custom route to use our own Request handler.\n    \"\"\"\n\n    def get_route_handler(self) -> Callable:\n        original_route_handler = super().get_route_handler()\n\n        async def custom_route_handler(request: Request) -> FastAPIResponse:\n            request = Request(request.scope, request.receive)\n            return await original_route_handler(request)\n\n        return custom_route_handler\n\n\ndef create_app(\n    settings: Settings,\n    data_plane: DataPlane,\n    model_repository_handlers: ModelRepositoryHandlers,\n) -> FastAPI:\n    endpoints = Endpoints(data_plane)\n    model_repository_endpoints = ModelRepositoryEndpoints(model_repository_handlers)\n\n    routes = [\n        # Model ready\n        APIRoute(\n            \"/v2/models/{model_name}/ready\",\n            endpoints.model_ready,\n        ),\n        APIRoute(\n            \"/v2/models/{model_name}/versions/{model_version}/ready\",\n            endpoints.model_ready,\n        ),\n        # Model infer\n        APIRoute(\n            \"/v2/models/{model_name}/infer\",\n            endpoints.infer,\n            methods=[\"POST\"],\n        ),\n        APIRoute(\n            \"/v2/models/{model_name}/versions/{model_version}/infer\",\n            endpoints.infer,\n            methods=[\"POST\"],\n        ),\n        # Model metadata\n        APIRoute(\n            \"/v2/models/{model_name}\",\n            endpoints.model_metadata,\n        ),\n        APIRoute(\n            \"/v2/models/{model_name}/versions/{model_version}\",\n            endpoints.model_metadata,\n        ),\n        # Liveness and readiness\n        APIRoute(\"/v2/health/live\", endpoints.live),\n        APIRoute(\"/v2/health/ready\", endpoints.ready),\n        # Server metadata\n        APIRoute(\n            \"/v2\",\n            endpoints.metadata,\n        ),\n    ]\n\n    routes += [\n        # Model Repository API\n        APIRoute(\n            \"/v2/repository/index\",\n            model_repository_endpoints.index,\n            methods=[\"POST\"],\n        ),\n        APIRoute(\n            \"/v2/repository/models/{model_name}/load\",\n            model_repository_endpoints.load,\n            methods=[\"POST\"],\n        ),\n        APIRoute(\n            \"/v2/repository/models/{model_name}/unload\",\n            model_repository_endpoints.unload,\n            methods=[\"POST\"],\n        ),\n    ]\n\n    app = FastAPI(\n        debug=settings.debug,\n        routes=routes,  # type: ignore\n        default_response_class=Response,\n        exception_handlers=_EXCEPTION_HANDLERS,  # type: ignore\n    )\n\n    if settings.cors_settings is not None:\n        app.add_middleware(\n            CORSMiddleware,\n            allow_origins=settings.cors_settings.allow_origins,\n            allow_origin_regex=settings.cors_settings.allow_origin_regex,\n            allow_credentials=settings.cors_settings.allow_credentials,\n            allow_methods=settings.cors_settings.allow_methods,\n            allow_headers=settings.cors_settings.allow_headers,\n            max_age=settings.cors_settings.max_age,\n        )\n\n    app.add_middleware(\n        PrometheusMiddleware,\n        app_name=\"mlserver\",\n        prefix=\"rest_server\",\n        # TODO: Should we also exclude model's health endpoints?\n        skip_paths=[\"/metrics\", \"/v2/health/live\", \"/v2/health/ready\"],\n    )\n    app.add_route(\"/metrics\", handle_metrics)\n\n    return app\n", "path": "mlserver/rest/app.py"}, {"content": "from typing import List, Optional\nfrom pydantic import BaseSettings, PyObject\n\nfrom .version import __version__\nfrom .types import MetadataTensor\n\nENV_PREFIX_SETTINGS = \"MLSERVER_\"\nENV_PREFIX_MODEL_SETTINGS = \"MLSERVER_MODEL_\"\n\n\nclass CORSSettings(BaseSettings):\n    class Config:\n        env_prefix = ENV_PREFIX_SETTINGS\n\n    \"\"\"\n    A list of origins that should be permitted to make\n    cross-origin requests. E.g. ['https://example.org', 'https://www.example.org'].\n    You can use ['*'] to allow any origin\n    \"\"\"\n    allow_origins: Optional[List[str]] = []\n\n    \"\"\"\n    A regex string to match against origins that\n    should be permitted to make cross-origin requests.\n    e.g. 'https:\\\\/\\\\/.*\\\\.example\\\\.org'\n    \"\"\"\n    allow_origin_regex: Optional[str] = None\n\n    \"\"\"Indicate that cookies should be supported for cross-origin requests\"\"\"\n    allow_credentials: Optional[bool] = False\n\n    \"\"\"A list of HTTP methods that should be allowed for cross-origin requests\"\"\"\n    allow_methods: Optional[List[str]] = [\"GET\"]\n\n    \"\"\"A list of HTTP request headers that should be supported for\n    cross-origin requests\"\"\"\n    allow_headers: Optional[List[str]] = []\n\n    \"\"\"Indicate any response headers that should be made accessible to the browser\"\"\"\n    expose_headers: Optional[List[str]] = []\n\n    \"\"\"Sets a maximum time in seconds for browsers to cache CORS responses\"\"\"\n    max_age: Optional[int] = 600\n\n\nclass Settings(BaseSettings):\n    class Config:\n        env_prefix = ENV_PREFIX_SETTINGS\n\n    debug: bool = True\n\n    # Model repository settings\n    \"\"\"Root of the model repository, where we will search for models.\"\"\"\n    model_repository_root: str = \".\"\n    \"\"\"Flag to load all available models automatically at startup.\"\"\"\n    load_models_at_startup: bool = True\n\n    # Server metadata\n    \"\"\"Name of the server.\"\"\"\n    server_name: str = \"mlserver\"\n    \"\"\"Version of the server.\"\"\"\n    server_version: str = __version__\n    \"\"\"Server extensions loaded.\"\"\"\n    extensions: List[str] = []\n\n    # Server settings\n    \"\"\"Host where to listen for connections.\"\"\"\n    host: str = \"0.0.0.0\"\n    \"\"\"Port where to listen for HTTP / REST connections.\"\"\"\n    http_port: int = 8080\n    \"\"\"Port where to listen for gRPC connections.\"\"\"\n    grpc_port: int = 8081\n    \"\"\"Maximum length (i.e. size) of gRPC payloads.\"\"\"\n    grpc_max_message_length: Optional[int] = None\n\n    # CORS settings\n    cors_settings: Optional[CORSSettings] = None\n\n\nclass ModelParameters(BaseSettings):\n    \"\"\"\n    Parameters that apply only to a particular instance of a model.\n    This can include things like model weights, or arbitrary ``extra``\n    parameters particular to the underlying inference runtime.\n    The main difference with respect to ``ModelSettings`` is that parameters\n    can change on each instance (e.g. each version) of the model.\n    \"\"\"\n\n    class Config:\n        env_prefix = ENV_PREFIX_MODEL_SETTINGS\n\n    \"\"\"\n    URI where the model artifacts can be found.\n    This path must be either absolute or relative to where MLServer is running.\n    \"\"\"\n    uri: Optional[str] = None\n    \"\"\"Version of the model.\"\"\"\n    version: Optional[str] = None\n    \"\"\"Format of the model (only available on certain runtimes).\"\"\"\n    format: Optional[str] = None\n    \"\"\"Default content type to use for requests and responses.\"\"\"\n    content_type: Optional[str] = None\n    \"\"\"Arbitrary settings, dependent on the inference runtime\n    implementation.\"\"\"\n    extra: Optional[dict] = {}\n\n\nclass ModelSettings(BaseSettings):\n    class Config:\n        env_prefix = ENV_PREFIX_MODEL_SETTINGS\n        underscore_attrs_are_private = True\n\n    # Source points to the file where model settings were loaded from\n    _source: Optional[str] = None\n\n    \"\"\"Name of the model.\"\"\"\n    name: str = \"\"\n\n    # Model metadata\n    \"\"\"Framework used to train and serialise the model (e.g. sklearn).\"\"\"\n    platform: str = \"\"\n    \"\"\"Versions of dependencies used to train the model (e.g.\n    sklearn/0.20.1).\"\"\"\n    versions: List[str] = []\n    \"\"\"Metadata about the inputs accepted by the model.\"\"\"\n    inputs: List[MetadataTensor] = []\n    \"\"\"Metadata about the outputs returned by the model.\"\"\"\n    outputs: List[MetadataTensor] = []\n\n    # Parallel settings\n    \"\"\"When parallel inference is enabled, number of workers to run inference\n    across.\"\"\"\n    parallel_workers: int = 4\n\n    # Adaptive Batching settings (disabled by default)\n    \"\"\"When adaptive batching is enabled, maximum number of requests to group\n    together in a single batch.\"\"\"\n    max_batch_size: int = 0\n    \"\"\"When adaptive batching is enabled, maximum amount of time (in seconds)\n    to wait for enough requests to build a full batch.\"\"\"\n    max_batch_time: float = 0.0\n\n    # Custom model class implementation\n    \"\"\"*Python path* to the inference runtime to use to serve this model (e.g.\n    ``mlserver_sklearn.SKLearnModel``).\"\"\"\n    implementation: PyObject = \"mlserver.model.MLModel\"  # type: ignore\n\n    # Model parameters are meant to be set directly by the MLServer runtime.\n    # However, it's also possible to override them manually.\n    \"\"\"Extra parameters for each instance of this model.\"\"\"\n    parameters: Optional[ModelParameters] = None\n", "path": "mlserver/settings.py"}], "after_files": [{"content": "from grpc import aio\nfrom concurrent.futures import ThreadPoolExecutor\nfrom typing import Any, List, Tuple\n\nfrom ..handlers import DataPlane, ModelRepositoryHandlers\nfrom ..settings import Settings\n\nfrom .servicers import InferenceServicer, ModelRepositoryServicer\nfrom .dataplane_pb2_grpc import add_GRPCInferenceServiceServicer_to_server\nfrom .model_repository_pb2_grpc import add_ModelRepositoryServiceServicer_to_server\nfrom .interceptors import LoggingInterceptor, PromServerInterceptor\nfrom .logging import logger\n\n# Workers used for non-AsyncIO workloads (which aren't any in our case)\nDefaultGrpcWorkers = 5\n\n\nclass GRPCServer:\n    def __init__(\n        self,\n        settings: Settings,\n        data_plane: DataPlane,\n        model_repository_handlers: ModelRepositoryHandlers,\n    ):\n        self._settings = settings\n        self._data_plane = data_plane\n        self._model_repository_handlers = model_repository_handlers\n\n    def _create_server(self):\n        self._inference_servicer = InferenceServicer(self._data_plane)\n        self._model_repository_servicer = ModelRepositoryServicer(\n            self._model_repository_handlers\n        )\n\n        interceptors = [LoggingInterceptor()]\n\n        if self._settings.metrics_endpoint:\n            interceptors.append(PromServerInterceptor())\n\n        self._server = aio.server(\n            ThreadPoolExecutor(max_workers=DefaultGrpcWorkers),\n            interceptors=tuple(interceptors),\n            options=self._get_options(),\n        )\n\n        add_GRPCInferenceServiceServicer_to_server(\n            self._inference_servicer, self._server\n        )\n        add_ModelRepositoryServiceServicer_to_server(\n            self._model_repository_servicer, self._server\n        )\n\n        self._server.add_insecure_port(\n            f\"{self._settings.host}:{self._settings.grpc_port}\"\n        )\n\n        return self._server\n\n    def _get_options(self) -> List[Tuple[str, Any]]:\n        options = []\n\n        max_message_length = self._settings.grpc_max_message_length\n        if max_message_length is not None:\n            options += [\n                (\"grpc.max_message_length\", max_message_length),\n                (\"grpc.max_send_message_length\", max_message_length),\n                (\"grpc.max_receive_message_length\", max_message_length),\n            ]\n\n        return options\n\n    async def start(self):\n        self._create_server()\n\n        await self._server.start()\n\n        logger.info(\n            \"gRPC server running on \"\n            f\"http://{self._settings.host}:{self._settings.grpc_port}\"\n        )\n        await self._server.wait_for_termination()\n\n    async def stop(self):\n        # TODO: Read from config\n        await self._server.stop(grace=5)\n", "path": "mlserver/grpc/server.py"}, {"content": "from typing import Callable\nfrom fastapi import FastAPI\nfrom fastapi.responses import Response as FastAPIResponse\nfrom fastapi.routing import APIRoute as FastAPIRoute\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom starlette_exporter import PrometheusMiddleware, handle_metrics\n\nfrom .endpoints import Endpoints, ModelRepositoryEndpoints\nfrom .requests import Request\nfrom .responses import Response\nfrom .errors import _EXCEPTION_HANDLERS\n\nfrom ..settings import Settings\nfrom ..handlers import DataPlane, ModelRepositoryHandlers\n\n\nclass APIRoute(FastAPIRoute):\n    \"\"\"\n    Custom route to use our own Request handler.\n    \"\"\"\n\n    def get_route_handler(self) -> Callable:\n        original_route_handler = super().get_route_handler()\n\n        async def custom_route_handler(request: Request) -> FastAPIResponse:\n            request = Request(request.scope, request.receive)\n            return await original_route_handler(request)\n\n        return custom_route_handler\n\n\ndef create_app(\n    settings: Settings,\n    data_plane: DataPlane,\n    model_repository_handlers: ModelRepositoryHandlers,\n) -> FastAPI:\n    endpoints = Endpoints(data_plane)\n    model_repository_endpoints = ModelRepositoryEndpoints(model_repository_handlers)\n\n    routes = [\n        # Model ready\n        APIRoute(\n            \"/v2/models/{model_name}/ready\",\n            endpoints.model_ready,\n        ),\n        APIRoute(\n            \"/v2/models/{model_name}/versions/{model_version}/ready\",\n            endpoints.model_ready,\n        ),\n        # Model infer\n        APIRoute(\n            \"/v2/models/{model_name}/infer\",\n            endpoints.infer,\n            methods=[\"POST\"],\n        ),\n        APIRoute(\n            \"/v2/models/{model_name}/versions/{model_version}/infer\",\n            endpoints.infer,\n            methods=[\"POST\"],\n        ),\n        # Model metadata\n        APIRoute(\n            \"/v2/models/{model_name}\",\n            endpoints.model_metadata,\n        ),\n        APIRoute(\n            \"/v2/models/{model_name}/versions/{model_version}\",\n            endpoints.model_metadata,\n        ),\n        # Liveness and readiness\n        APIRoute(\"/v2/health/live\", endpoints.live),\n        APIRoute(\"/v2/health/ready\", endpoints.ready),\n        # Server metadata\n        APIRoute(\n            \"/v2\",\n            endpoints.metadata,\n        ),\n    ]\n\n    routes += [\n        # Model Repository API\n        APIRoute(\n            \"/v2/repository/index\",\n            model_repository_endpoints.index,\n            methods=[\"POST\"],\n        ),\n        APIRoute(\n            \"/v2/repository/models/{model_name}/load\",\n            model_repository_endpoints.load,\n            methods=[\"POST\"],\n        ),\n        APIRoute(\n            \"/v2/repository/models/{model_name}/unload\",\n            model_repository_endpoints.unload,\n            methods=[\"POST\"],\n        ),\n    ]\n\n    app = FastAPI(\n        debug=settings.debug,\n        routes=routes,  # type: ignore\n        default_response_class=Response,\n        exception_handlers=_EXCEPTION_HANDLERS,  # type: ignore\n    )\n\n    if settings.cors_settings is not None:\n        app.add_middleware(\n            CORSMiddleware,\n            allow_origins=settings.cors_settings.allow_origins,\n            allow_origin_regex=settings.cors_settings.allow_origin_regex,\n            allow_credentials=settings.cors_settings.allow_credentials,\n            allow_methods=settings.cors_settings.allow_methods,\n            allow_headers=settings.cors_settings.allow_headers,\n            max_age=settings.cors_settings.max_age,\n        )\n\n    if settings.metrics_endpoint:\n        app.add_middleware(\n            PrometheusMiddleware,\n            app_name=\"mlserver\",\n            prefix=\"rest_server\",\n            # TODO: Should we also exclude model's health endpoints?\n            skip_paths=[\n                settings.metrics_endpoint,\n                \"/v2/health/live\",\n                \"/v2/health/ready\",\n            ],\n        )\n        app.add_route(settings.metrics_endpoint, handle_metrics)\n\n    return app\n", "path": "mlserver/rest/app.py"}, {"content": "from typing import List, Optional\nfrom pydantic import BaseSettings, PyObject\n\nfrom .version import __version__\nfrom .types import MetadataTensor\n\nENV_PREFIX_SETTINGS = \"MLSERVER_\"\nENV_PREFIX_MODEL_SETTINGS = \"MLSERVER_MODEL_\"\n\n\nclass CORSSettings(BaseSettings):\n    class Config:\n        env_prefix = ENV_PREFIX_SETTINGS\n\n    \"\"\"\n    A list of origins that should be permitted to make\n    cross-origin requests. E.g. ['https://example.org', 'https://www.example.org'].\n    You can use ['*'] to allow any origin\n    \"\"\"\n    allow_origins: Optional[List[str]] = []\n\n    \"\"\"\n    A regex string to match against origins that\n    should be permitted to make cross-origin requests.\n    e.g. 'https:\\\\/\\\\/.*\\\\.example\\\\.org'\n    \"\"\"\n    allow_origin_regex: Optional[str] = None\n\n    \"\"\"Indicate that cookies should be supported for cross-origin requests\"\"\"\n    allow_credentials: Optional[bool] = False\n\n    \"\"\"A list of HTTP methods that should be allowed for cross-origin requests\"\"\"\n    allow_methods: Optional[List[str]] = [\"GET\"]\n\n    \"\"\"A list of HTTP request headers that should be supported for\n    cross-origin requests\"\"\"\n    allow_headers: Optional[List[str]] = []\n\n    \"\"\"Indicate any response headers that should be made accessible to the browser\"\"\"\n    expose_headers: Optional[List[str]] = []\n\n    \"\"\"Sets a maximum time in seconds for browsers to cache CORS responses\"\"\"\n    max_age: Optional[int] = 600\n\n\nclass Settings(BaseSettings):\n    class Config:\n        env_prefix = ENV_PREFIX_SETTINGS\n\n    debug: bool = True\n\n    # Model repository settings\n    \"\"\"Root of the model repository, where we will search for models.\"\"\"\n    model_repository_root: str = \".\"\n    \"\"\"Flag to load all available models automatically at startup.\"\"\"\n    load_models_at_startup: bool = True\n\n    # Server metadata\n    \"\"\"Name of the server.\"\"\"\n    server_name: str = \"mlserver\"\n    \"\"\"Version of the server.\"\"\"\n    server_version: str = __version__\n    \"\"\"Server extensions loaded.\"\"\"\n    extensions: List[str] = []\n\n    # Server settings\n    \"\"\"Host where to listen for connections.\"\"\"\n    host: str = \"0.0.0.0\"\n    \"\"\"Port where to listen for HTTP / REST connections.\"\"\"\n    http_port: int = 8080\n    \"\"\"Port where to listen for gRPC connections.\"\"\"\n    grpc_port: int = 8081\n    \"\"\"Maximum length (i.e. size) of gRPC payloads.\"\"\"\n    grpc_max_message_length: Optional[int] = None\n\n    # CORS settings\n    cors_settings: Optional[CORSSettings] = None\n\n    # Metrics settings\n    \"\"\"\n    Endpoint used to expose Prometheus metrics. Alternatively, can be set to\n    `None` to disable it\n    \"\"\"\n    metrics_endpoint: Optional[str] = \"/metrics\"\n\n\nclass ModelParameters(BaseSettings):\n    \"\"\"\n    Parameters that apply only to a particular instance of a model.\n    This can include things like model weights, or arbitrary ``extra``\n    parameters particular to the underlying inference runtime.\n    The main difference with respect to ``ModelSettings`` is that parameters\n    can change on each instance (e.g. each version) of the model.\n    \"\"\"\n\n    class Config:\n        env_prefix = ENV_PREFIX_MODEL_SETTINGS\n\n    \"\"\"\n    URI where the model artifacts can be found.\n    This path must be either absolute or relative to where MLServer is running.\n    \"\"\"\n    uri: Optional[str] = None\n    \"\"\"Version of the model.\"\"\"\n    version: Optional[str] = None\n    \"\"\"Format of the model (only available on certain runtimes).\"\"\"\n    format: Optional[str] = None\n    \"\"\"Default content type to use for requests and responses.\"\"\"\n    content_type: Optional[str] = None\n    \"\"\"Arbitrary settings, dependent on the inference runtime\n    implementation.\"\"\"\n    extra: Optional[dict] = {}\n\n\nclass ModelSettings(BaseSettings):\n    class Config:\n        env_prefix = ENV_PREFIX_MODEL_SETTINGS\n        underscore_attrs_are_private = True\n\n    # Source points to the file where model settings were loaded from\n    _source: Optional[str] = None\n\n    \"\"\"Name of the model.\"\"\"\n    name: str = \"\"\n\n    # Model metadata\n    \"\"\"Framework used to train and serialise the model (e.g. sklearn).\"\"\"\n    platform: str = \"\"\n    \"\"\"Versions of dependencies used to train the model (e.g.\n    sklearn/0.20.1).\"\"\"\n    versions: List[str] = []\n    \"\"\"Metadata about the inputs accepted by the model.\"\"\"\n    inputs: List[MetadataTensor] = []\n    \"\"\"Metadata about the outputs returned by the model.\"\"\"\n    outputs: List[MetadataTensor] = []\n\n    # Parallel settings\n    \"\"\"When parallel inference is enabled, number of workers to run inference\n    across.\"\"\"\n    parallel_workers: int = 4\n\n    # Adaptive Batching settings (disabled by default)\n    \"\"\"When adaptive batching is enabled, maximum number of requests to group\n    together in a single batch.\"\"\"\n    max_batch_size: int = 0\n    \"\"\"When adaptive batching is enabled, maximum amount of time (in seconds)\n    to wait for enough requests to build a full batch.\"\"\"\n    max_batch_time: float = 0.0\n\n    # Custom model class implementation\n    \"\"\"*Python path* to the inference runtime to use to serve this model (e.g.\n    ``mlserver_sklearn.SKLearnModel``).\"\"\"\n    implementation: PyObject = \"mlserver.model.MLModel\"  # type: ignore\n\n    # Model parameters are meant to be set directly by the MLServer runtime.\n    # However, it's also possible to override them manually.\n    \"\"\"Extra parameters for each instance of this model.\"\"\"\n    parameters: Optional[ModelParameters] = None\n", "path": "mlserver/settings.py"}]}