We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Give insights to the user when we raise TooBigRows in /rows, /search, or /filter
https://huggingface.co/datasets/danielz01/xView2/discussions/2#655dc3ccde919d41622b51bd

Currently, we only show

```
Error code:   UnexpectedApiError
```

We should say that the rows contents are too heavy and that we don't show them for that reason (> 400MB in that case)

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `services/rows/src/rows/routes/rows.py`
Content:
```
1 # SPDX-License-Identifier: Apache-2.0
2 # Copyright 2022 The HuggingFace Authors.
3 
4 import logging
5 from typing import Literal, Optional, Union
6 
7 from fsspec.implementations.http import HTTPFileSystem
8 from libapi.authentication import auth_check
9 from libapi.exceptions import ApiError, UnexpectedApiError
10 from libapi.request import (
11     get_request_parameter,
12     get_request_parameter_length,
13     get_request_parameter_offset,
14 )
15 from libapi.response import create_response
16 from libapi.utils import (
17     Endpoint,
18     get_json_api_error_response,
19     get_json_error_response,
20     get_json_ok_response,
21     try_backfill_dataset_then_raise,
22 )
23 from libcommon.parquet_utils import Indexer
24 from libcommon.processing_graph import ProcessingGraph
25 from libcommon.prometheus import StepProfiler
26 from libcommon.simple_cache import CachedArtifactError, CachedArtifactNotFoundError
27 from libcommon.storage import StrPath
28 from libcommon.storage_client import StorageClient
29 from libcommon.viewer_utils.features import UNSUPPORTED_FEATURES
30 from starlette.requests import Request
31 from starlette.responses import Response
32 
33 logger = logging.getLogger(__name__)
34 
35 
36 ALL_COLUMNS_SUPPORTED_DATASETS_ALLOW_LIST: Union[Literal["all"], list[str]] = ["arabic_speech_corpus"]  # for testing
37 
38 
39 def create_rows_endpoint(
40     processing_graph: ProcessingGraph,
41     cached_assets_base_url: str,
42     storage_client: StorageClient,
43     parquet_metadata_directory: StrPath,
44     cache_max_days: int,
45     max_arrow_data_in_memory: int,
46     hf_endpoint: str,
47     blocked_datasets: list[str],
48     hf_token: Optional[str] = None,
49     hf_jwt_public_keys: Optional[list[str]] = None,
50     hf_jwt_algorithm: Optional[str] = None,
51     external_auth_url: Optional[str] = None,
52     hf_timeout_seconds: Optional[float] = None,
53     max_age_long: int = 0,
54     max_age_short: int = 0,
55 ) -> Endpoint:
56     indexer = Indexer(
57         processing_graph=processing_graph,
58         hf_token=hf_token,
59         parquet_metadata_directory=parquet_metadata_directory,
60         httpfs=HTTPFileSystem(headers={"authorization": f"Bearer {hf_token}"}),
61         max_arrow_data_in_memory=max_arrow_data_in_memory,
62         unsupported_features=UNSUPPORTED_FEATURES,
63         all_columns_supported_datasets_allow_list=ALL_COLUMNS_SUPPORTED_DATASETS_ALLOW_LIST,
64     )
65 
66     async def rows_endpoint(request: Request) -> Response:
67         await indexer.httpfs.set_session()
68         revision: Optional[str] = None
69         with StepProfiler(method="rows_endpoint", step="all"):
70             try:
71                 with StepProfiler(method="rows_endpoint", step="validate parameters"):
72                     dataset = get_request_parameter(request, "dataset", required=True)
73                     config = get_request_parameter(request, "config", required=True)
74                     split = get_request_parameter(request, "split", required=True)
75                     offset = get_request_parameter_offset(request)
76                     length = get_request_parameter_length(request)
77                     logging.info(
78                         f"/rows, dataset={dataset}, config={config}, split={split}, offset={offset}, length={length}"
79                     )
80                 with StepProfiler(method="rows_endpoint", step="check authentication"):
81                     # if auth_check fails, it will raise an exception that will be caught below
82                     await auth_check(
83                         dataset=dataset,
84                         external_auth_url=external_auth_url,
85                         request=request,
86                         hf_jwt_public_keys=hf_jwt_public_keys,
87                         hf_jwt_algorithm=hf_jwt_algorithm,
88                         hf_timeout_seconds=hf_timeout_seconds,
89                     )
90                 try:
91                     with StepProfiler(method="rows_endpoint", step="get row groups index"):
92                         rows_index = indexer.get_rows_index(
93                             dataset=dataset,
94                             config=config,
95                             split=split,
96                         )
97                         revision = rows_index.revision
98                     with StepProfiler(method="rows_endpoint", step="query the rows"):
99                         pa_table = rows_index.query(offset=offset, length=length)
100                     with StepProfiler(method="rows_endpoint", step="transform to a list"):
101                         response = await create_response(
102                             dataset=dataset,
103                             revision=revision,
104                             config=config,
105                             split=split,
106                             cached_assets_base_url=cached_assets_base_url,
107                             storage_client=storage_client,
108                             pa_table=pa_table,
109                             offset=offset,
110                             features=rows_index.parquet_index.features,
111                             unsupported_columns=rows_index.parquet_index.unsupported_columns,
112                             partial=rows_index.parquet_index.partial,
113                             num_rows_total=rows_index.parquet_index.num_rows_total,
114                         )
115                 except CachedArtifactNotFoundError:
116                     config_parquet_processing_steps = processing_graph.get_config_parquet_processing_steps()
117                     config_parquet_metadata_processing_steps = (
118                         processing_graph.get_config_parquet_metadata_processing_steps()
119                     )
120                     with StepProfiler(method="rows_endpoint", step="try backfill dataset"):
121                         try_backfill_dataset_then_raise(
122                             processing_steps=config_parquet_metadata_processing_steps
123                             + config_parquet_processing_steps,
124                             processing_graph=processing_graph,
125                             dataset=dataset,
126                             hf_endpoint=hf_endpoint,
127                             hf_timeout_seconds=hf_timeout_seconds,
128                             hf_token=hf_token,
129                             cache_max_days=cache_max_days,
130                             blocked_datasets=blocked_datasets,
131                         )
132                 with StepProfiler(method="rows_endpoint", step="generate the OK response"):
133                     return get_json_ok_response(content=response, max_age=max_age_long, revision=revision)
134             except CachedArtifactError as e:
135                 content = e.cache_entry_with_details["content"]
136                 http_status = e.cache_entry_with_details["http_status"]
137                 error_code = e.cache_entry_with_details["error_code"]
138                 return get_json_error_response(
139                     content=content,
140                     status_code=http_status,
141                     max_age=max_age_short,
142                     error_code=error_code,
143                     revision=revision,
144                 )
145             except Exception as e:
146                 error = e if isinstance(e, ApiError) else UnexpectedApiError("Unexpected error.", e)
147                 with StepProfiler(method="rows_endpoint", step="generate API error response"):
148                     return get_json_api_error_response(error=error, max_age=max_age_short, revision=revision)
149 
150     return rows_endpoint
151
```

Path: `libs/libapi/src/libapi/exceptions.py`
Content:
```
1 # SPDX-License-Identifier: Apache-2.0
2 # Copyright 2022 The HuggingFace Authors.
3 
4 import logging
5 from http import HTTPStatus
6 from typing import Literal, Optional
7 
8 from libcommon.exceptions import CustomError
9 
10 ApiErrorCode = Literal[
11     "AuthCheckHubRequestError",
12     "ExternalAuthenticatedError",
13     "ExternalUnauthenticatedError",
14     "InvalidParameter",
15     "JWTExpiredSignature",
16     "JWTInvalidClaimRead",
17     "JWTInvalidClaimSub",
18     "JWTInvalidKeyOrAlgorithm",
19     "JWTInvalidSignature",
20     "JWTKeysError",
21     "JWTMissingRequiredClaim",
22     "MissingProcessingStepsError",
23     "MissingRequiredParameter",
24     "ResponseNotFound",
25     "ResponseNotReady",
26     "SearchFeatureNotAvailableError",
27     "TransformRowsProcessingError",
28     "UnexpectedApiError",
29 ]
30 
31 
32 class ApiError(CustomError):
33     """Base class for exceptions raised by an API service."""
34 
35     def __init__(
36         self,
37         message: str,
38         status_code: HTTPStatus,
39         code: ApiErrorCode,
40         cause: Optional[BaseException] = None,
41         disclose_cause: bool = False,
42     ):
43         super().__init__(
44             message=message, status_code=status_code, code=code, cause=cause, disclose_cause=disclose_cause
45         )
46 
47 
48 class AuthCheckHubRequestError(ApiError):
49     """The external authentication check failed or timed out."""
50 
51     def __init__(self, message: str, cause: Optional[BaseException] = None):
52         super().__init__(
53             message, HTTPStatus.INTERNAL_SERVER_ERROR, "AuthCheckHubRequestError", cause=cause, disclose_cause=False
54         )
55 
56 
57 class ExternalAuthenticatedError(ApiError):
58     """The external authentication check failed while the user was authenticated.
59 
60     Even if the external authentication server returns 403 in that case, we return 404 because
61     we don't know if the dataset exist or not. It's also coherent with how the Hugging Face Hub works.
62 
63     TODO: should we return DatasetNotFoundError instead? maybe the error code is leaking existence of private datasets.
64     """
65 
66     def __init__(self, message: str):
67         super().__init__(message, HTTPStatus.NOT_FOUND, "ExternalAuthenticatedError")
68 
69 
70 class ExternalUnauthenticatedError(ApiError):
71     """The external authentication check failed while the user was unauthenticated."""
72 
73     def __init__(self, message: str):
74         super().__init__(message, HTTPStatus.UNAUTHORIZED, "ExternalUnauthenticatedError")
75 
76 
77 class InvalidParameterError(ApiError):
78     """A parameter has an invalid value."""
79 
80     def __init__(self, message: str):
81         super().__init__(message, HTTPStatus.UNPROCESSABLE_ENTITY, "InvalidParameter")
82 
83 
84 class JWTKeysError(ApiError):
85     """The public keys for decoding JWT could not be created."""
86 
87     def __init__(self, message: str, cause: Optional[BaseException] = None):
88         super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "JWTKeysError", cause=cause, disclose_cause=False)
89 
90 
91 class MissingRequiredParameterError(ApiError):
92     """A required parameter is missing."""
93 
94     def __init__(self, message: str):
95         super().__init__(message, HTTPStatus.UNPROCESSABLE_ENTITY, "MissingRequiredParameter")
96 
97 
98 class ResponseNotFoundError(ApiError):
99     """The response has not been found."""
100 
101     def __init__(self, message: str):
102         super().__init__(message, HTTPStatus.NOT_FOUND, "ResponseNotFound")
103 
104 
105 class ResponseNotReadyError(ApiError):
106     """The response has not been processed yet."""
107 
108     def __init__(self, message: str):
109         super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "ResponseNotReady")
110 
111 
112 class SearchFeatureNotAvailableError(ApiError):
113     """The split does not have search feature enabled."""
114 
115     def __init__(self, message: str, cause: Optional[BaseException] = None):
116         super().__init__(message, HTTPStatus.BAD_REQUEST, "SearchFeatureNotAvailableError", cause, True)
117 
118 
119 class TransformRowsProcessingError(ApiError):
120     """There was an error when transforming rows to list."""
121 
122     def __init__(self, message: str, cause: Optional[BaseException] = None):
123         super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "TransformRowsProcessingError", cause, True)
124 
125 
126 class JWTExpiredSignature(ApiError):
127     """The JWT signature has expired."""
128 
129     def __init__(self, message: str, cause: Optional[BaseException] = None):
130         super().__init__(message, HTTPStatus.UNAUTHORIZED, "JWTExpiredSignature", cause, True)
131 
132 
133 class JWTInvalidClaimRead(ApiError):
134     """The 'read' claim in the JWT payload is invalid."""
135 
136     def __init__(self, message: str, cause: Optional[BaseException] = None):
137         super().__init__(message, HTTPStatus.UNAUTHORIZED, "JWTInvalidClaimRead", cause, True)
138 
139 
140 class JWTInvalidClaimSub(ApiError):
141     """The 'sub' claim in the JWT payload is invalid."""
142 
143     def __init__(self, message: str, cause: Optional[BaseException] = None):
144         super().__init__(message, HTTPStatus.UNAUTHORIZED, "JWTInvalidClaimSub", cause, True)
145 
146 
147 class JWTInvalidKeyOrAlgorithm(ApiError):
148     """The key and the algorithm used to verify the JWT signature are not compatible."""
149 
150     def __init__(self, message: str, cause: Optional[BaseException] = None):
151         super().__init__(message, HTTPStatus.UNAUTHORIZED, "JWTInvalidKeyOrAlgorithm", cause, True)
152 
153 
154 class JWTInvalidSignature(ApiError):
155     """The JWT signature verification failed."""
156 
157     def __init__(self, message: str, cause: Optional[BaseException] = None):
158         super().__init__(message, HTTPStatus.UNAUTHORIZED, "JWTInvalidSignature", cause, True)
159 
160 
161 class JWTMissingRequiredClaim(ApiError):
162     """A claim is missing in the JWT payload."""
163 
164     def __init__(self, message: str, cause: Optional[BaseException] = None):
165         super().__init__(message, HTTPStatus.UNAUTHORIZED, "JWTMissingRequiredClaim", cause, True)
166 
167 
168 class UnexpectedApiError(ApiError):
169     """The server raised an unexpected error."""
170 
171     def __init__(self, message: str, cause: Optional[BaseException] = None):
172         logging.error(message, exc_info=cause)
173         super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, "UnexpectedApiError", cause)
174
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/libs/libapi/src/libapi/exceptions.py b/libs/libapi/src/libapi/exceptions.py
--- a/libs/libapi/src/libapi/exceptions.py
+++ b/libs/libapi/src/libapi/exceptions.py
@@ -116,6 +116,15 @@
         super().__init__(message, HTTPStatus.BAD_REQUEST, "SearchFeatureNotAvailableError", cause, True)
 
 
+class TooBigContentError(ApiError):
+    """The content size in bytes is bigger than the supported value."""
+
+    def __init__(self, message: str, cause: Optional[BaseException] = None):
+        super().__init__(
+            message, HTTPStatus.INTERNAL_SERVER_ERROR, "TransformRowsProcessingError", cause=cause, disclose_cause=True
+        )
+
+
 class TransformRowsProcessingError(ApiError):
     """There was an error when transforming rows to list."""
 
diff --git a/services/rows/src/rows/routes/rows.py b/services/rows/src/rows/routes/rows.py
--- a/services/rows/src/rows/routes/rows.py
+++ b/services/rows/src/rows/routes/rows.py
@@ -6,7 +6,7 @@
 
 from fsspec.implementations.http import HTTPFileSystem
 from libapi.authentication import auth_check
-from libapi.exceptions import ApiError, UnexpectedApiError
+from libapi.exceptions import ApiError, TooBigContentError, UnexpectedApiError
 from libapi.request import (
     get_request_parameter,
     get_request_parameter_length,
@@ -20,7 +20,7 @@
     get_json_ok_response,
     try_backfill_dataset_then_raise,
 )
-from libcommon.parquet_utils import Indexer
+from libcommon.parquet_utils import Indexer, TooBigRows
 from libcommon.processing_graph import ProcessingGraph
 from libcommon.prometheus import StepProfiler
 from libcommon.simple_cache import CachedArtifactError, CachedArtifactNotFoundError
@@ -96,7 +96,10 @@
                         )
                         revision = rows_index.revision
                     with StepProfiler(method="rows_endpoint", step="query the rows"):
-                        pa_table = rows_index.query(offset=offset, length=length)
+                        try:
+                            pa_table = rows_index.query(offset=offset, length=length)
+                        except TooBigRows as err:
+                            raise TooBigContentError(str(err)) from None
                     with StepProfiler(method="rows_endpoint", step="transform to a list"):
                         response = await create_response(
                             dataset=dataset,

{"golden_diff": "diff --git a/libs/libapi/src/libapi/exceptions.py b/libs/libapi/src/libapi/exceptions.py\n--- a/libs/libapi/src/libapi/exceptions.py\n+++ b/libs/libapi/src/libapi/exceptions.py\n@@ -116,6 +116,15 @@\n         super().__init__(message, HTTPStatus.BAD_REQUEST, \"SearchFeatureNotAvailableError\", cause, True)\n \n \n+class TooBigContentError(ApiError):\n+    \"\"\"The content size in bytes is bigger than the supported value.\"\"\"\n+\n+    def __init__(self, message: str, cause: Optional[BaseException] = None):\n+        super().__init__(\n+            message, HTTPStatus.INTERNAL_SERVER_ERROR, \"TransformRowsProcessingError\", cause=cause, disclose_cause=True\n+        )\n+\n+\n class TransformRowsProcessingError(ApiError):\n     \"\"\"There was an error when transforming rows to list.\"\"\"\n \ndiff --git a/services/rows/src/rows/routes/rows.py b/services/rows/src/rows/routes/rows.py\n--- a/services/rows/src/rows/routes/rows.py\n+++ b/services/rows/src/rows/routes/rows.py\n@@ -6,7 +6,7 @@\n \n from fsspec.implementations.http import HTTPFileSystem\n from libapi.authentication import auth_check\n-from libapi.exceptions import ApiError, UnexpectedApiError\n+from libapi.exceptions import ApiError, TooBigContentError, UnexpectedApiError\n from libapi.request import (\n     get_request_parameter,\n     get_request_parameter_length,\n@@ -20,7 +20,7 @@\n     get_json_ok_response,\n     try_backfill_dataset_then_raise,\n )\n-from libcommon.parquet_utils import Indexer\n+from libcommon.parquet_utils import Indexer, TooBigRows\n from libcommon.processing_graph import ProcessingGraph\n from libcommon.prometheus import StepProfiler\n from libcommon.simple_cache import CachedArtifactError, CachedArtifactNotFoundError\n@@ -96,7 +96,10 @@\n                         )\n                         revision = rows_index.revision\n                     with StepProfiler(method=\"rows_endpoint\", step=\"query the rows\"):\n-                        pa_table = rows_index.query(offset=offset, length=length)\n+                        try:\n+                            pa_table = rows_index.query(offset=offset, length=length)\n+                        except TooBigRows as err:\n+                            raise TooBigContentError(str(err)) from None\n                     with StepProfiler(method=\"rows_endpoint\", step=\"transform to a list\"):\n                         response = await create_response(\n                             dataset=dataset,\n", "issue": "Give insights to the user when we raise TooBigRows in /rows, /search, or /filter\nhttps://huggingface.co/datasets/danielz01/xView2/discussions/2#655dc3ccde919d41622b51bd\r\n\r\nCurrently, we only show\r\n\r\n```\r\nError code:   UnexpectedApiError\r\n```\r\n\r\nWe should say that the rows contents are too heavy and that we don't show them for that reason (> 400MB in that case)\n", "before_files": [{"content": "# SPDX-License-Identifier: Apache-2.0\n# Copyright 2022 The HuggingFace Authors.\n\nimport logging\nfrom typing import Literal, Optional, Union\n\nfrom fsspec.implementations.http import HTTPFileSystem\nfrom libapi.authentication import auth_check\nfrom libapi.exceptions import ApiError, UnexpectedApiError\nfrom libapi.request import (\n    get_request_parameter,\n    get_request_parameter_length,\n    get_request_parameter_offset,\n)\nfrom libapi.response import create_response\nfrom libapi.utils import (\n    Endpoint,\n    get_json_api_error_response,\n    get_json_error_response,\n    get_json_ok_response,\n    try_backfill_dataset_then_raise,\n)\nfrom libcommon.parquet_utils import Indexer\nfrom libcommon.processing_graph import ProcessingGraph\nfrom libcommon.prometheus import StepProfiler\nfrom libcommon.simple_cache import CachedArtifactError, CachedArtifactNotFoundError\nfrom libcommon.storage import StrPath\nfrom libcommon.storage_client import StorageClient\nfrom libcommon.viewer_utils.features import UNSUPPORTED_FEATURES\nfrom starlette.requests import Request\nfrom starlette.responses import Response\n\nlogger = logging.getLogger(__name__)\n\n\nALL_COLUMNS_SUPPORTED_DATASETS_ALLOW_LIST: Union[Literal[\"all\"], list[str]] = [\"arabic_speech_corpus\"]  # for testing\n\n\ndef create_rows_endpoint(\n    processing_graph: ProcessingGraph,\n    cached_assets_base_url: str,\n    storage_client: StorageClient,\n    parquet_metadata_directory: StrPath,\n    cache_max_days: int,\n    max_arrow_data_in_memory: int,\n    hf_endpoint: str,\n    blocked_datasets: list[str],\n    hf_token: Optional[str] = None,\n    hf_jwt_public_keys: Optional[list[str]] = None,\n    hf_jwt_algorithm: Optional[str] = None,\n    external_auth_url: Optional[str] = None,\n    hf_timeout_seconds: Optional[float] = None,\n    max_age_long: int = 0,\n    max_age_short: int = 0,\n) -> Endpoint:\n    indexer = Indexer(\n        processing_graph=processing_graph,\n        hf_token=hf_token,\n        parquet_metadata_directory=parquet_metadata_directory,\n        httpfs=HTTPFileSystem(headers={\"authorization\": f\"Bearer {hf_token}\"}),\n        max_arrow_data_in_memory=max_arrow_data_in_memory,\n        unsupported_features=UNSUPPORTED_FEATURES,\n        all_columns_supported_datasets_allow_list=ALL_COLUMNS_SUPPORTED_DATASETS_ALLOW_LIST,\n    )\n\n    async def rows_endpoint(request: Request) -> Response:\n        await indexer.httpfs.set_session()\n        revision: Optional[str] = None\n        with StepProfiler(method=\"rows_endpoint\", step=\"all\"):\n            try:\n                with StepProfiler(method=\"rows_endpoint\", step=\"validate parameters\"):\n                    dataset = get_request_parameter(request, \"dataset\", required=True)\n                    config = get_request_parameter(request, \"config\", required=True)\n                    split = get_request_parameter(request, \"split\", required=True)\n                    offset = get_request_parameter_offset(request)\n                    length = get_request_parameter_length(request)\n                    logging.info(\n                        f\"/rows, dataset={dataset}, config={config}, split={split}, offset={offset}, length={length}\"\n                    )\n                with StepProfiler(method=\"rows_endpoint\", step=\"check authentication\"):\n                    # if auth_check fails, it will raise an exception that will be caught below\n                    await auth_check(\n                        dataset=dataset,\n                        external_auth_url=external_auth_url,\n                        request=request,\n                        hf_jwt_public_keys=hf_jwt_public_keys,\n                        hf_jwt_algorithm=hf_jwt_algorithm,\n                        hf_timeout_seconds=hf_timeout_seconds,\n                    )\n                try:\n                    with StepProfiler(method=\"rows_endpoint\", step=\"get row groups index\"):\n                        rows_index = indexer.get_rows_index(\n                            dataset=dataset,\n                            config=config,\n                            split=split,\n                        )\n                        revision = rows_index.revision\n                    with StepProfiler(method=\"rows_endpoint\", step=\"query the rows\"):\n                        pa_table = rows_index.query(offset=offset, length=length)\n                    with StepProfiler(method=\"rows_endpoint\", step=\"transform to a list\"):\n                        response = await create_response(\n                            dataset=dataset,\n                            revision=revision,\n                            config=config,\n                            split=split,\n                            cached_assets_base_url=cached_assets_base_url,\n                            storage_client=storage_client,\n                            pa_table=pa_table,\n                            offset=offset,\n                            features=rows_index.parquet_index.features,\n                            unsupported_columns=rows_index.parquet_index.unsupported_columns,\n                            partial=rows_index.parquet_index.partial,\n                            num_rows_total=rows_index.parquet_index.num_rows_total,\n                        )\n                except CachedArtifactNotFoundError:\n                    config_parquet_processing_steps = processing_graph.get_config_parquet_processing_steps()\n                    config_parquet_metadata_processing_steps = (\n                        processing_graph.get_config_parquet_metadata_processing_steps()\n                    )\n                    with StepProfiler(method=\"rows_endpoint\", step=\"try backfill dataset\"):\n                        try_backfill_dataset_then_raise(\n                            processing_steps=config_parquet_metadata_processing_steps\n                            + config_parquet_processing_steps,\n                            processing_graph=processing_graph,\n                            dataset=dataset,\n                            hf_endpoint=hf_endpoint,\n                            hf_timeout_seconds=hf_timeout_seconds,\n                            hf_token=hf_token,\n                            cache_max_days=cache_max_days,\n                            blocked_datasets=blocked_datasets,\n                        )\n                with StepProfiler(method=\"rows_endpoint\", step=\"generate the OK response\"):\n                    return get_json_ok_response(content=response, max_age=max_age_long, revision=revision)\n            except CachedArtifactError as e:\n                content = e.cache_entry_with_details[\"content\"]\n                http_status = e.cache_entry_with_details[\"http_status\"]\n                error_code = e.cache_entry_with_details[\"error_code\"]\n                return get_json_error_response(\n                    content=content,\n                    status_code=http_status,\n                    max_age=max_age_short,\n                    error_code=error_code,\n                    revision=revision,\n                )\n            except Exception as e:\n                error = e if isinstance(e, ApiError) else UnexpectedApiError(\"Unexpected error.\", e)\n                with StepProfiler(method=\"rows_endpoint\", step=\"generate API error response\"):\n                    return get_json_api_error_response(error=error, max_age=max_age_short, revision=revision)\n\n    return rows_endpoint\n", "path": "services/rows/src/rows/routes/rows.py"}, {"content": "# SPDX-License-Identifier: Apache-2.0\n# Copyright 2022 The HuggingFace Authors.\n\nimport logging\nfrom http import HTTPStatus\nfrom typing import Literal, Optional\n\nfrom libcommon.exceptions import CustomError\n\nApiErrorCode = Literal[\n    \"AuthCheckHubRequestError\",\n    \"ExternalAuthenticatedError\",\n    \"ExternalUnauthenticatedError\",\n    \"InvalidParameter\",\n    \"JWTExpiredSignature\",\n    \"JWTInvalidClaimRead\",\n    \"JWTInvalidClaimSub\",\n    \"JWTInvalidKeyOrAlgorithm\",\n    \"JWTInvalidSignature\",\n    \"JWTKeysError\",\n    \"JWTMissingRequiredClaim\",\n    \"MissingProcessingStepsError\",\n    \"MissingRequiredParameter\",\n    \"ResponseNotFound\",\n    \"ResponseNotReady\",\n    \"SearchFeatureNotAvailableError\",\n    \"TransformRowsProcessingError\",\n    \"UnexpectedApiError\",\n]\n\n\nclass ApiError(CustomError):\n    \"\"\"Base class for exceptions raised by an API service.\"\"\"\n\n    def __init__(\n        self,\n        message: str,\n        status_code: HTTPStatus,\n        code: ApiErrorCode,\n        cause: Optional[BaseException] = None,\n        disclose_cause: bool = False,\n    ):\n        super().__init__(\n            message=message, status_code=status_code, code=code, cause=cause, disclose_cause=disclose_cause\n        )\n\n\nclass AuthCheckHubRequestError(ApiError):\n    \"\"\"The external authentication check failed or timed out.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(\n            message, HTTPStatus.INTERNAL_SERVER_ERROR, \"AuthCheckHubRequestError\", cause=cause, disclose_cause=False\n        )\n\n\nclass ExternalAuthenticatedError(ApiError):\n    \"\"\"The external authentication check failed while the user was authenticated.\n\n    Even if the external authentication server returns 403 in that case, we return 404 because\n    we don't know if the dataset exist or not. It's also coherent with how the Hugging Face Hub works.\n\n    TODO: should we return DatasetNotFoundError instead? maybe the error code is leaking existence of private datasets.\n    \"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.NOT_FOUND, \"ExternalAuthenticatedError\")\n\n\nclass ExternalUnauthenticatedError(ApiError):\n    \"\"\"The external authentication check failed while the user was unauthenticated.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"ExternalUnauthenticatedError\")\n\n\nclass InvalidParameterError(ApiError):\n    \"\"\"A parameter has an invalid value.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.UNPROCESSABLE_ENTITY, \"InvalidParameter\")\n\n\nclass JWTKeysError(ApiError):\n    \"\"\"The public keys for decoding JWT could not be created.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, \"JWTKeysError\", cause=cause, disclose_cause=False)\n\n\nclass MissingRequiredParameterError(ApiError):\n    \"\"\"A required parameter is missing.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.UNPROCESSABLE_ENTITY, \"MissingRequiredParameter\")\n\n\nclass ResponseNotFoundError(ApiError):\n    \"\"\"The response has not been found.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.NOT_FOUND, \"ResponseNotFound\")\n\n\nclass ResponseNotReadyError(ApiError):\n    \"\"\"The response has not been processed yet.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, \"ResponseNotReady\")\n\n\nclass SearchFeatureNotAvailableError(ApiError):\n    \"\"\"The split does not have search feature enabled.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.BAD_REQUEST, \"SearchFeatureNotAvailableError\", cause, True)\n\n\nclass TransformRowsProcessingError(ApiError):\n    \"\"\"There was an error when transforming rows to list.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, \"TransformRowsProcessingError\", cause, True)\n\n\nclass JWTExpiredSignature(ApiError):\n    \"\"\"The JWT signature has expired.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTExpiredSignature\", cause, True)\n\n\nclass JWTInvalidClaimRead(ApiError):\n    \"\"\"The 'read' claim in the JWT payload is invalid.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTInvalidClaimRead\", cause, True)\n\n\nclass JWTInvalidClaimSub(ApiError):\n    \"\"\"The 'sub' claim in the JWT payload is invalid.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTInvalidClaimSub\", cause, True)\n\n\nclass JWTInvalidKeyOrAlgorithm(ApiError):\n    \"\"\"The key and the algorithm used to verify the JWT signature are not compatible.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTInvalidKeyOrAlgorithm\", cause, True)\n\n\nclass JWTInvalidSignature(ApiError):\n    \"\"\"The JWT signature verification failed.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTInvalidSignature\", cause, True)\n\n\nclass JWTMissingRequiredClaim(ApiError):\n    \"\"\"A claim is missing in the JWT payload.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTMissingRequiredClaim\", cause, True)\n\n\nclass UnexpectedApiError(ApiError):\n    \"\"\"The server raised an unexpected error.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        logging.error(message, exc_info=cause)\n        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, \"UnexpectedApiError\", cause)\n", "path": "libs/libapi/src/libapi/exceptions.py"}], "after_files": [{"content": "# SPDX-License-Identifier: Apache-2.0\n# Copyright 2022 The HuggingFace Authors.\n\nimport logging\nfrom typing import Literal, Optional, Union\n\nfrom fsspec.implementations.http import HTTPFileSystem\nfrom libapi.authentication import auth_check\nfrom libapi.exceptions import ApiError, TooBigContentError, UnexpectedApiError\nfrom libapi.request import (\n    get_request_parameter,\n    get_request_parameter_length,\n    get_request_parameter_offset,\n)\nfrom libapi.response import create_response\nfrom libapi.utils import (\n    Endpoint,\n    get_json_api_error_response,\n    get_json_error_response,\n    get_json_ok_response,\n    try_backfill_dataset_then_raise,\n)\nfrom libcommon.parquet_utils import Indexer, TooBigRows\nfrom libcommon.processing_graph import ProcessingGraph\nfrom libcommon.prometheus import StepProfiler\nfrom libcommon.simple_cache import CachedArtifactError, CachedArtifactNotFoundError\nfrom libcommon.storage import StrPath\nfrom libcommon.storage_client import StorageClient\nfrom libcommon.viewer_utils.features import UNSUPPORTED_FEATURES\nfrom starlette.requests import Request\nfrom starlette.responses import Response\n\nlogger = logging.getLogger(__name__)\n\n\nALL_COLUMNS_SUPPORTED_DATASETS_ALLOW_LIST: Union[Literal[\"all\"], list[str]] = [\"arabic_speech_corpus\"]  # for testing\n\n\ndef create_rows_endpoint(\n    processing_graph: ProcessingGraph,\n    cached_assets_base_url: str,\n    storage_client: StorageClient,\n    parquet_metadata_directory: StrPath,\n    cache_max_days: int,\n    max_arrow_data_in_memory: int,\n    hf_endpoint: str,\n    blocked_datasets: list[str],\n    hf_token: Optional[str] = None,\n    hf_jwt_public_keys: Optional[list[str]] = None,\n    hf_jwt_algorithm: Optional[str] = None,\n    external_auth_url: Optional[str] = None,\n    hf_timeout_seconds: Optional[float] = None,\n    max_age_long: int = 0,\n    max_age_short: int = 0,\n) -> Endpoint:\n    indexer = Indexer(\n        processing_graph=processing_graph,\n        hf_token=hf_token,\n        parquet_metadata_directory=parquet_metadata_directory,\n        httpfs=HTTPFileSystem(headers={\"authorization\": f\"Bearer {hf_token}\"}),\n        max_arrow_data_in_memory=max_arrow_data_in_memory,\n        unsupported_features=UNSUPPORTED_FEATURES,\n        all_columns_supported_datasets_allow_list=ALL_COLUMNS_SUPPORTED_DATASETS_ALLOW_LIST,\n    )\n\n    async def rows_endpoint(request: Request) -> Response:\n        await indexer.httpfs.set_session()\n        revision: Optional[str] = None\n        with StepProfiler(method=\"rows_endpoint\", step=\"all\"):\n            try:\n                with StepProfiler(method=\"rows_endpoint\", step=\"validate parameters\"):\n                    dataset = get_request_parameter(request, \"dataset\", required=True)\n                    config = get_request_parameter(request, \"config\", required=True)\n                    split = get_request_parameter(request, \"split\", required=True)\n                    offset = get_request_parameter_offset(request)\n                    length = get_request_parameter_length(request)\n                    logging.info(\n                        f\"/rows, dataset={dataset}, config={config}, split={split}, offset={offset}, length={length}\"\n                    )\n                with StepProfiler(method=\"rows_endpoint\", step=\"check authentication\"):\n                    # if auth_check fails, it will raise an exception that will be caught below\n                    await auth_check(\n                        dataset=dataset,\n                        external_auth_url=external_auth_url,\n                        request=request,\n                        hf_jwt_public_keys=hf_jwt_public_keys,\n                        hf_jwt_algorithm=hf_jwt_algorithm,\n                        hf_timeout_seconds=hf_timeout_seconds,\n                    )\n                try:\n                    with StepProfiler(method=\"rows_endpoint\", step=\"get row groups index\"):\n                        rows_index = indexer.get_rows_index(\n                            dataset=dataset,\n                            config=config,\n                            split=split,\n                        )\n                        revision = rows_index.revision\n                    with StepProfiler(method=\"rows_endpoint\", step=\"query the rows\"):\n                        try:\n                            pa_table = rows_index.query(offset=offset, length=length)\n                        except TooBigRows as err:\n                            raise TooBigContentError(str(err)) from None\n                    with StepProfiler(method=\"rows_endpoint\", step=\"transform to a list\"):\n                        response = await create_response(\n                            dataset=dataset,\n                            revision=revision,\n                            config=config,\n                            split=split,\n                            cached_assets_base_url=cached_assets_base_url,\n                            storage_client=storage_client,\n                            pa_table=pa_table,\n                            offset=offset,\n                            features=rows_index.parquet_index.features,\n                            unsupported_columns=rows_index.parquet_index.unsupported_columns,\n                            partial=rows_index.parquet_index.partial,\n                            num_rows_total=rows_index.parquet_index.num_rows_total,\n                        )\n                except CachedArtifactNotFoundError:\n                    config_parquet_processing_steps = processing_graph.get_config_parquet_processing_steps()\n                    config_parquet_metadata_processing_steps = (\n                        processing_graph.get_config_parquet_metadata_processing_steps()\n                    )\n                    with StepProfiler(method=\"rows_endpoint\", step=\"try backfill dataset\"):\n                        try_backfill_dataset_then_raise(\n                            processing_steps=config_parquet_metadata_processing_steps\n                            + config_parquet_processing_steps,\n                            processing_graph=processing_graph,\n                            dataset=dataset,\n                            hf_endpoint=hf_endpoint,\n                            hf_timeout_seconds=hf_timeout_seconds,\n                            hf_token=hf_token,\n                            cache_max_days=cache_max_days,\n                            blocked_datasets=blocked_datasets,\n                        )\n                with StepProfiler(method=\"rows_endpoint\", step=\"generate the OK response\"):\n                    return get_json_ok_response(content=response, max_age=max_age_long, revision=revision)\n            except CachedArtifactError as e:\n                content = e.cache_entry_with_details[\"content\"]\n                http_status = e.cache_entry_with_details[\"http_status\"]\n                error_code = e.cache_entry_with_details[\"error_code\"]\n                return get_json_error_response(\n                    content=content,\n                    status_code=http_status,\n                    max_age=max_age_short,\n                    error_code=error_code,\n                    revision=revision,\n                )\n            except Exception as e:\n                error = e if isinstance(e, ApiError) else UnexpectedApiError(\"Unexpected error.\", e)\n                with StepProfiler(method=\"rows_endpoint\", step=\"generate API error response\"):\n                    return get_json_api_error_response(error=error, max_age=max_age_short, revision=revision)\n\n    return rows_endpoint\n", "path": "services/rows/src/rows/routes/rows.py"}, {"content": "# SPDX-License-Identifier: Apache-2.0\n# Copyright 2022 The HuggingFace Authors.\n\nimport logging\nfrom http import HTTPStatus\nfrom typing import Literal, Optional\n\nfrom libcommon.exceptions import CustomError\n\nApiErrorCode = Literal[\n    \"AuthCheckHubRequestError\",\n    \"ExternalAuthenticatedError\",\n    \"ExternalUnauthenticatedError\",\n    \"InvalidParameter\",\n    \"JWTExpiredSignature\",\n    \"JWTInvalidClaimRead\",\n    \"JWTInvalidClaimSub\",\n    \"JWTInvalidKeyOrAlgorithm\",\n    \"JWTInvalidSignature\",\n    \"JWTKeysError\",\n    \"JWTMissingRequiredClaim\",\n    \"MissingProcessingStepsError\",\n    \"MissingRequiredParameter\",\n    \"ResponseNotFound\",\n    \"ResponseNotReady\",\n    \"SearchFeatureNotAvailableError\",\n    \"TransformRowsProcessingError\",\n    \"UnexpectedApiError\",\n]\n\n\nclass ApiError(CustomError):\n    \"\"\"Base class for exceptions raised by an API service.\"\"\"\n\n    def __init__(\n        self,\n        message: str,\n        status_code: HTTPStatus,\n        code: ApiErrorCode,\n        cause: Optional[BaseException] = None,\n        disclose_cause: bool = False,\n    ):\n        super().__init__(\n            message=message, status_code=status_code, code=code, cause=cause, disclose_cause=disclose_cause\n        )\n\n\nclass AuthCheckHubRequestError(ApiError):\n    \"\"\"The external authentication check failed or timed out.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(\n            message, HTTPStatus.INTERNAL_SERVER_ERROR, \"AuthCheckHubRequestError\", cause=cause, disclose_cause=False\n        )\n\n\nclass ExternalAuthenticatedError(ApiError):\n    \"\"\"The external authentication check failed while the user was authenticated.\n\n    Even if the external authentication server returns 403 in that case, we return 404 because\n    we don't know if the dataset exist or not. It's also coherent with how the Hugging Face Hub works.\n\n    TODO: should we return DatasetNotFoundError instead? maybe the error code is leaking existence of private datasets.\n    \"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.NOT_FOUND, \"ExternalAuthenticatedError\")\n\n\nclass ExternalUnauthenticatedError(ApiError):\n    \"\"\"The external authentication check failed while the user was unauthenticated.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"ExternalUnauthenticatedError\")\n\n\nclass InvalidParameterError(ApiError):\n    \"\"\"A parameter has an invalid value.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.UNPROCESSABLE_ENTITY, \"InvalidParameter\")\n\n\nclass JWTKeysError(ApiError):\n    \"\"\"The public keys for decoding JWT could not be created.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, \"JWTKeysError\", cause=cause, disclose_cause=False)\n\n\nclass MissingRequiredParameterError(ApiError):\n    \"\"\"A required parameter is missing.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.UNPROCESSABLE_ENTITY, \"MissingRequiredParameter\")\n\n\nclass ResponseNotFoundError(ApiError):\n    \"\"\"The response has not been found.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.NOT_FOUND, \"ResponseNotFound\")\n\n\nclass ResponseNotReadyError(ApiError):\n    \"\"\"The response has not been processed yet.\"\"\"\n\n    def __init__(self, message: str):\n        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, \"ResponseNotReady\")\n\n\nclass SearchFeatureNotAvailableError(ApiError):\n    \"\"\"The split does not have search feature enabled.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.BAD_REQUEST, \"SearchFeatureNotAvailableError\", cause, True)\n\n\nclass TooBigContentError(ApiError):\n    \"\"\"The content size in bytes is bigger than the supported value.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(\n            message, HTTPStatus.INTERNAL_SERVER_ERROR, \"TransformRowsProcessingError\", cause=cause, disclose_cause=True\n        )\n\n\nclass TransformRowsProcessingError(ApiError):\n    \"\"\"There was an error when transforming rows to list.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, \"TransformRowsProcessingError\", cause, True)\n\n\nclass JWTExpiredSignature(ApiError):\n    \"\"\"The JWT signature has expired.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTExpiredSignature\", cause, True)\n\n\nclass JWTInvalidClaimRead(ApiError):\n    \"\"\"The 'read' claim in the JWT payload is invalid.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTInvalidClaimRead\", cause, True)\n\n\nclass JWTInvalidClaimSub(ApiError):\n    \"\"\"The 'sub' claim in the JWT payload is invalid.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTInvalidClaimSub\", cause, True)\n\n\nclass JWTInvalidKeyOrAlgorithm(ApiError):\n    \"\"\"The key and the algorithm used to verify the JWT signature are not compatible.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTInvalidKeyOrAlgorithm\", cause, True)\n\n\nclass JWTInvalidSignature(ApiError):\n    \"\"\"The JWT signature verification failed.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTInvalidSignature\", cause, True)\n\n\nclass JWTMissingRequiredClaim(ApiError):\n    \"\"\"A claim is missing in the JWT payload.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        super().__init__(message, HTTPStatus.UNAUTHORIZED, \"JWTMissingRequiredClaim\", cause, True)\n\n\nclass UnexpectedApiError(ApiError):\n    \"\"\"The server raised an unexpected error.\"\"\"\n\n    def __init__(self, message: str, cause: Optional[BaseException] = None):\n        logging.error(message, exc_info=cause)\n        super().__init__(message, HTTPStatus.INTERNAL_SERVER_ERROR, \"UnexpectedApiError\", cause)\n", "path": "libs/libapi/src/libapi/exceptions.py"}]}