We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Regenerate dataset-info instead of loading it?
Currently, getting the rows with `/rows` requires a previous (internal) call to `/infos` to get the features (type of the columns). But sometimes the dataset-info.json file is missing, or not coherent with the dataset script (for example: https://huggingface.co/datasets/lhoestq/custom_squad/tree/main), while we are using `datasets.get_dataset_infos()`, which only loads the exported dataset-info.json files:

https://github.com/huggingface/datasets-preview-backend/blob/c2a78e7ce8e36cdf579fea805535fa9ef84a2027/src/datasets_preview_backend/queries/infos.py#L45


https://github.com/huggingface/datasets/blob/26ff41aa3a642e46489db9e95be1e9a8c4e64bea/src/datasets/inspect.py#L115


We might want to call `._info()` from the builder to get the info, and features, instead of relying on the dataset-info.json file.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/datasets_preview_backend/queries/infos.py`
Content:
```
1 from dataclasses import asdict
2 from typing import List, Optional, cast
3 
4 from datasets import get_dataset_infos
5 
6 from datasets_preview_backend.cache import memoize  # type: ignore
7 from datasets_preview_backend.config import CACHE_TTL_SECONDS, cache
8 from datasets_preview_backend.constants import DATASETS_BLOCKLIST
9 from datasets_preview_backend.exceptions import Status400Error, Status404Error
10 from datasets_preview_backend.queries.configs import get_configs_response
11 from datasets_preview_backend.responses import CachedResponse
12 from datasets_preview_backend.types import (
13     ConfigsContent,
14     InfoItem,
15     InfosContent,
16     StatusErrorContent,
17 )
18 
19 
20 def get_infos(dataset: str, config: Optional[str] = None, token: Optional[str] = None) -> InfosContent:
21     if not isinstance(dataset, str) and dataset is not None:
22         raise TypeError("dataset argument should be a string")
23     if dataset is None:
24         raise Status400Error("'dataset' is a required query parameter.")
25     if dataset in DATASETS_BLOCKLIST:
26         raise Status400Error("this dataset is not supported for now.")
27     if config is not None:
28         if not isinstance(config, str):
29             raise TypeError("config argument should be a string")
30         configs = [config]
31     else:
32         content = get_configs_response(dataset=dataset, token=token).content
33         if "configs" not in content:
34             error = cast(StatusErrorContent, content)
35             if "status_code" in error and error["status_code"] == 404:
36                 raise Status404Error("configurations could not be found")
37             raise Status400Error("configurations could not be found")
38         configs_content = cast(ConfigsContent, content)
39         configs = [configItem["config"] for configItem in configs_content["configs"]]
40 
41     infoItems: List[InfoItem] = []
42     # Note that we raise on the first error
43     for config in configs:
44         try:
45             dataset_info_dict = get_dataset_infos(dataset, use_auth_token=token)
46             if config not in dataset_info_dict:
47                 # this is an error in the dataset-info.json
48                 raise Exception("The config is not listed in the datasets infos")
49             info = asdict(dataset_info_dict[config])
50             if "splits" in info:
51                 info["splits"] = {split_name: split_info for split_name, split_info in info["splits"].items()}
52         except FileNotFoundError as err:
53             raise Status404Error("The config info could not be found.") from err
54         except Exception as err:
55             raise Status400Error("The config info could not be parsed from the dataset.") from err
56         infoItems.append({"dataset": dataset, "config": config, "info": info})
57 
58     return {"infos": infoItems}
59 
60 
61 @memoize(cache, expire=CACHE_TTL_SECONDS)  # type:ignore
62 def get_infos_response(*, dataset: str, config: Optional[str] = None, token: Optional[str] = None) -> CachedResponse:
63     try:
64         response = CachedResponse(get_infos(dataset, config, token))
65     except (Status400Error, Status404Error) as err:
66         response = CachedResponse(err.as_content(), err.status_code)
67     return response
68 
69 
70 def get_refreshed_infos(dataset: str, config: Optional[str] = None, token: Optional[str] = None) -> InfosContent:
71     return cast(InfosContent, get_infos_response(dataset, config, token, _refresh=True)["content"])
72
```

Path: `src/datasets_preview_backend/queries/rows.py`
Content:
```
1 import logging
2 import re
3 from typing import List, Optional, cast
4 
5 from datasets import IterableDataset, load_dataset
6 
7 from datasets_preview_backend.cache import memoize  # type: ignore
8 from datasets_preview_backend.config import CACHE_TTL_SECONDS, EXTRACT_ROWS_LIMIT, cache
9 from datasets_preview_backend.constants import DATASETS_BLOCKLIST
10 from datasets_preview_backend.exceptions import Status400Error, Status404Error
11 from datasets_preview_backend.queries.configs import get_configs_response
12 from datasets_preview_backend.queries.infos import get_infos_response
13 from datasets_preview_backend.queries.splits import get_splits_response
14 from datasets_preview_backend.responses import CachedResponse
15 from datasets_preview_backend.types import (
16     ConfigsContent,
17     FeatureItem,
18     InfosContent,
19     RowItem,
20     RowsContent,
21     SplitsContent,
22     StatusErrorContent,
23 )
24 
25 logger = logging.getLogger(__name__)
26 
27 
28 def get_rows(
29     dataset: str, config: Optional[str] = None, split: Optional[str] = None, token: Optional[str] = None
30 ) -> RowsContent:
31     if not isinstance(dataset, str) and dataset is not None:
32         raise TypeError("dataset argument should be a string")
33     if dataset is None:
34         raise Status400Error("'dataset' is a required query parameter.")
35     if dataset in DATASETS_BLOCKLIST:
36         raise Status400Error("this dataset is not supported for now.")
37     if config is None:
38         # split is ignored if config is not passed
39         logger.warning("split argument is ignored since config is not provided")
40         split = None
41     elif not isinstance(config, str):
42         raise TypeError("config argument should be a string")
43     if not isinstance(split, str) and split is not None:
44         raise TypeError("split argument should be a string")
45     num_rows = EXTRACT_ROWS_LIMIT
46 
47     rowItems: List[RowItem] = []
48     featureItems: List[FeatureItem] = []
49 
50     if config is not None and split is not None:
51         try:
52             iterable_dataset = load_dataset(dataset, name=config, split=split, streaming=True, use_auth_token=token)
53             if not isinstance(iterable_dataset, IterableDataset):
54                 raise TypeError("load_dataset should return an IterableDataset")
55             rows = list(iterable_dataset.take(num_rows))
56         except FileNotFoundError as err:
57             raise Status404Error("The split for the dataset config could not be found.") from err
58         except NotImplementedError as err:
59             # TODO: check what has changed once https://github.com/huggingface/datasets/pull/2662 is merged
60             try:
61                 regex = re.compile(r"Extraction protocol for file at .*?((\.\w+)?\.\w+)* is not implemented yet")
62                 match = regex.match(str(err))
63                 if match is None:
64                     raise Exception("No match")
65                 extension = match.group(1)
66             except Exception:
67                 raise Status400Error("The rows could not be extracted from the split of the dataset config.") from err
68             else:
69                 raise Status400Error(
70                     "The rows could not be extracted from the split of the dataset config because extension"
71                     f" {extension} is not supported."
72                 ) from err
73         except ValueError as err:
74             if (
75                 str(err).startswith(f"BuilderConfig {config} not found.")
76                 or str(err).startswith("Config name is missing.")
77                 or str(err).startswith("Bad split")
78             ):
79                 raise Status404Error("The dataset config could not be found.") from err
80             else:
81                 raise Status400Error("The rows could not be extracted from the split of the dataset config.") from err
82         except Exception as err:
83             raise Status400Error("The rows could not be extracted from the split of the dataset config.") from err
84 
85         if len(rows) != num_rows:
86             logger.warning(
87                 f"could not read all the required rows ({len(rows)} / {num_rows}) from dataset {dataset} -"
88                 f" {config} - {split}"
89             )
90 
91         rowItems = [{"dataset": dataset, "config": config, "split": split, "row": row} for row in rows]
92 
93         content = get_infos_response(dataset=dataset, config=config, token=token).content
94         if "infos" not in content:
95             error = cast(StatusErrorContent, content)
96             if "status_code" in error and error["status_code"] == 404:
97                 raise Status404Error("features could not be found")
98             raise Status400Error("features could not be found")
99         infos_content = cast(InfosContent, content)
100         infoItems = [infoItem["info"] for infoItem in infos_content["infos"]]
101 
102         if len(infoItems) != 1:
103             raise Exception("a dataset config should have exactly one info")
104         infoItem = infoItems[0]
105         if "features" not in infoItem:
106             raise Status400Error("a dataset config info should contain a 'features' property")
107         localFeaturesItems: List[FeatureItem] = [
108             {"dataset": dataset, "config": config, "features": infoItem["features"]}
109         ]
110 
111         return {"features": localFeaturesItems, "rows": rowItems}
112 
113     if config is None:
114         content = get_configs_response(dataset=dataset, token=token).content
115         if "configs" not in content:
116             error = cast(StatusErrorContent, content)
117             if "status_code" in error and error["status_code"] == 404:
118                 raise Status404Error("configurations could not be found")
119             raise Status400Error("configurations could not be found")
120         configs_content = cast(ConfigsContent, content)
121         configs = [configItem["config"] for configItem in configs_content["configs"]]
122     else:
123         configs = [config]
124 
125     # Note that we raise on the first error
126     for config in configs:
127         content = get_splits_response(dataset=dataset, config=config, token=token).content
128         if "splits" not in content:
129             error = cast(StatusErrorContent, content)
130             if "status_code" in error and error["status_code"] == 404:
131                 raise Status404Error("splits could not be found")
132             raise Status400Error("splits could not be found")
133         splits_content = cast(SplitsContent, content)
134         splits = [splitItem["split"] for splitItem in splits_content["splits"]]
135 
136         for split in splits:
137             content = get_rows_response(dataset=dataset, config=config, split=split, token=token).content
138             if "rows" not in content:
139                 error = cast(StatusErrorContent, content)
140                 if "status_code" in error and error["status_code"] == 404:
141                     raise Status404Error("rows could not be found")
142                 raise Status400Error("rows could not be found")
143             rows_content = cast(RowsContent, content)
144             rowItems += rows_content["rows"]
145             for featureItem in rows_content["features"]:
146                 # there should be only one element. Anyway, let's loop
147                 if featureItem not in featureItems:
148                     featureItems.append(featureItem)
149 
150     return {"features": featureItems, "rows": rowItems}
151 
152 
153 @memoize(cache, expire=CACHE_TTL_SECONDS)  # type:ignore
154 def get_rows_response(
155     *, dataset: str, config: Optional[str] = None, split: Optional[str] = None, token: Optional[str] = None
156 ) -> CachedResponse:
157     try:
158         response = CachedResponse(get_rows(dataset, config, split, token))
159     except (Status400Error, Status404Error) as err:
160         response = CachedResponse(err.as_content(), err.status_code)
161     return response
162 
163 
164 def get_refreshed_rows(
165     dataset: str, config: Optional[str] = None, split: Optional[str] = None, token: Optional[str] = None
166 ) -> RowsContent:
167     return cast(RowsContent, get_rows_response(dataset, config, split, token, _refresh=True)["content"])
168
```

Path: `src/datasets_preview_backend/types.py`
Content:
```
1 from typing import Any, Dict, List, TypedDict, Union
2 
3 
4 class DatasetItem(TypedDict):
5     dataset: str
6 
7 
8 class InfoItem(TypedDict):
9     dataset: str
10     config: str
11     info: Dict[str, Any]
12 
13 
14 class ConfigItem(TypedDict):
15     dataset: str
16     config: str
17 
18 
19 class SplitItem(TypedDict):
20     dataset: str
21     config: str
22     split: str
23 
24 
25 class RowItem(TypedDict):
26     dataset: str
27     config: str
28     split: str
29     row: Any
30 
31 
32 class FeatureItem(TypedDict):
33     dataset: str
34     config: str
35     features: Any
36 
37 
38 # Content of endpoint responses
39 
40 
41 class DatasetsContent(TypedDict):
42     datasets: List[DatasetItem]
43 
44 
45 class InfosContent(TypedDict):
46     infos: List[InfoItem]
47 
48 
49 class ConfigsContent(TypedDict):
50     configs: List[ConfigItem]
51 
52 
53 class SplitsContent(TypedDict):
54     splits: List[SplitItem]
55 
56 
57 class RowsContent(TypedDict):
58     features: List[FeatureItem]
59     rows: List[RowItem]
60 
61 
62 class StatusErrorContent(TypedDict):
63     status_code: int
64     exception: str
65     message: str
66     cause: str
67     cause_message: str
68 
69 
70 Content = Union[
71     ConfigsContent,
72     DatasetsContent,
73     InfosContent,
74     RowsContent,
75     SplitsContent,
76     StatusErrorContent,
77 ]
78
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/datasets_preview_backend/queries/infos.py b/src/datasets_preview_backend/queries/infos.py
--- a/src/datasets_preview_backend/queries/infos.py
+++ b/src/datasets_preview_backend/queries/infos.py
@@ -1,7 +1,7 @@
 from dataclasses import asdict
 from typing import List, Optional, cast
 
-from datasets import get_dataset_infos
+from datasets import load_dataset_builder
 
 from datasets_preview_backend.cache import memoize  # type: ignore
 from datasets_preview_backend.config import CACHE_TTL_SECONDS, cache
@@ -42,12 +42,9 @@
     # Note that we raise on the first error
     for config in configs:
         try:
-            dataset_info_dict = get_dataset_infos(dataset, use_auth_token=token)
-            if config not in dataset_info_dict:
-                # this is an error in the dataset-info.json
-                raise Exception("The config is not listed in the datasets infos")
-            info = asdict(dataset_info_dict[config])
-            if "splits" in info:
+            builder = load_dataset_builder(dataset, name=config, use_auth_token=token)
+            info = asdict(builder.info)
+            if "splits" in info and info["splits"] is not None:
                 info["splits"] = {split_name: split_info for split_name, split_info in info["splits"].items()}
         except FileNotFoundError as err:
             raise Status404Error("The config info could not be found.") from err
diff --git a/src/datasets_preview_backend/queries/rows.py b/src/datasets_preview_backend/queries/rows.py
--- a/src/datasets_preview_backend/queries/rows.py
+++ b/src/datasets_preview_backend/queries/rows.py
@@ -14,7 +14,7 @@
 from datasets_preview_backend.responses import CachedResponse
 from datasets_preview_backend.types import (
     ConfigsContent,
-    FeatureItem,
+    FeaturesItem,
     InfosContent,
     RowItem,
     RowsContent,
@@ -45,7 +45,7 @@
     num_rows = EXTRACT_ROWS_LIMIT
 
     rowItems: List[RowItem] = []
-    featureItems: List[FeatureItem] = []
+    featuresItems: List[FeaturesItem] = []
 
     if config is not None and split is not None:
         try:
@@ -104,7 +104,7 @@
         infoItem = infoItems[0]
         if "features" not in infoItem:
             raise Status400Error("a dataset config info should contain a 'features' property")
-        localFeaturesItems: List[FeatureItem] = [
+        localFeaturesItems: List[FeaturesItem] = [
             {"dataset": dataset, "config": config, "features": infoItem["features"]}
         ]
 
@@ -142,12 +142,12 @@
                 raise Status400Error("rows could not be found")
             rows_content = cast(RowsContent, content)
             rowItems += rows_content["rows"]
-            for featureItem in rows_content["features"]:
+            for featuresItem in rows_content["features"]:
                 # there should be only one element. Anyway, let's loop
-                if featureItem not in featureItems:
-                    featureItems.append(featureItem)
+                if featuresItem not in featuresItems:
+                    featuresItems.append(featuresItem)
 
-    return {"features": featureItems, "rows": rowItems}
+    return {"features": featuresItems, "rows": rowItems}
 
 
 @memoize(cache, expire=CACHE_TTL_SECONDS)  # type:ignore
diff --git a/src/datasets_preview_backend/types.py b/src/datasets_preview_backend/types.py
--- a/src/datasets_preview_backend/types.py
+++ b/src/datasets_preview_backend/types.py
@@ -29,7 +29,7 @@
     row: Any
 
 
-class FeatureItem(TypedDict):
+class FeaturesItem(TypedDict):
     dataset: str
     config: str
     features: Any
@@ -55,7 +55,7 @@
 
 
 class RowsContent(TypedDict):
-    features: List[FeatureItem]
+    features: List[FeaturesItem]
     rows: List[RowItem]

{"golden_diff": "diff --git a/src/datasets_preview_backend/queries/infos.py b/src/datasets_preview_backend/queries/infos.py\n--- a/src/datasets_preview_backend/queries/infos.py\n+++ b/src/datasets_preview_backend/queries/infos.py\n@@ -1,7 +1,7 @@\n from dataclasses import asdict\n from typing import List, Optional, cast\n \n-from datasets import get_dataset_infos\n+from datasets import load_dataset_builder\n \n from datasets_preview_backend.cache import memoize  # type: ignore\n from datasets_preview_backend.config import CACHE_TTL_SECONDS, cache\n@@ -42,12 +42,9 @@\n     # Note that we raise on the first error\n     for config in configs:\n         try:\n-            dataset_info_dict = get_dataset_infos(dataset, use_auth_token=token)\n-            if config not in dataset_info_dict:\n-                # this is an error in the dataset-info.json\n-                raise Exception(\"The config is not listed in the datasets infos\")\n-            info = asdict(dataset_info_dict[config])\n-            if \"splits\" in info:\n+            builder = load_dataset_builder(dataset, name=config, use_auth_token=token)\n+            info = asdict(builder.info)\n+            if \"splits\" in info and info[\"splits\"] is not None:\n                 info[\"splits\"] = {split_name: split_info for split_name, split_info in info[\"splits\"].items()}\n         except FileNotFoundError as err:\n             raise Status404Error(\"The config info could not be found.\") from err\ndiff --git a/src/datasets_preview_backend/queries/rows.py b/src/datasets_preview_backend/queries/rows.py\n--- a/src/datasets_preview_backend/queries/rows.py\n+++ b/src/datasets_preview_backend/queries/rows.py\n@@ -14,7 +14,7 @@\n from datasets_preview_backend.responses import CachedResponse\n from datasets_preview_backend.types import (\n     ConfigsContent,\n-    FeatureItem,\n+    FeaturesItem,\n     InfosContent,\n     RowItem,\n     RowsContent,\n@@ -45,7 +45,7 @@\n     num_rows = EXTRACT_ROWS_LIMIT\n \n     rowItems: List[RowItem] = []\n-    featureItems: List[FeatureItem] = []\n+    featuresItems: List[FeaturesItem] = []\n \n     if config is not None and split is not None:\n         try:\n@@ -104,7 +104,7 @@\n         infoItem = infoItems[0]\n         if \"features\" not in infoItem:\n             raise Status400Error(\"a dataset config info should contain a 'features' property\")\n-        localFeaturesItems: List[FeatureItem] = [\n+        localFeaturesItems: List[FeaturesItem] = [\n             {\"dataset\": dataset, \"config\": config, \"features\": infoItem[\"features\"]}\n         ]\n \n@@ -142,12 +142,12 @@\n                 raise Status400Error(\"rows could not be found\")\n             rows_content = cast(RowsContent, content)\n             rowItems += rows_content[\"rows\"]\n-            for featureItem in rows_content[\"features\"]:\n+            for featuresItem in rows_content[\"features\"]:\n                 # there should be only one element. Anyway, let's loop\n-                if featureItem not in featureItems:\n-                    featureItems.append(featureItem)\n+                if featuresItem not in featuresItems:\n+                    featuresItems.append(featuresItem)\n \n-    return {\"features\": featureItems, \"rows\": rowItems}\n+    return {\"features\": featuresItems, \"rows\": rowItems}\n \n \n @memoize(cache, expire=CACHE_TTL_SECONDS)  # type:ignore\ndiff --git a/src/datasets_preview_backend/types.py b/src/datasets_preview_backend/types.py\n--- a/src/datasets_preview_backend/types.py\n+++ b/src/datasets_preview_backend/types.py\n@@ -29,7 +29,7 @@\n     row: Any\n \n \n-class FeatureItem(TypedDict):\n+class FeaturesItem(TypedDict):\n     dataset: str\n     config: str\n     features: Any\n@@ -55,7 +55,7 @@\n \n \n class RowsContent(TypedDict):\n-    features: List[FeatureItem]\n+    features: List[FeaturesItem]\n     rows: List[RowItem]\n", "issue": "Regenerate dataset-info instead of loading it?\nCurrently, getting the rows with `/rows` requires a previous (internal) call to `/infos` to get the features (type of the columns). But sometimes the dataset-info.json file is missing, or not coherent with the dataset script (for example: https://huggingface.co/datasets/lhoestq/custom_squad/tree/main), while we are using `datasets.get_dataset_infos()`, which only loads the exported dataset-info.json files:\r\n\r\nhttps://github.com/huggingface/datasets-preview-backend/blob/c2a78e7ce8e36cdf579fea805535fa9ef84a2027/src/datasets_preview_backend/queries/infos.py#L45\r\n\r\n\r\nhttps://github.com/huggingface/datasets/blob/26ff41aa3a642e46489db9e95be1e9a8c4e64bea/src/datasets/inspect.py#L115\r\n\r\n\r\nWe might want to call `._info()` from the builder to get the info, and features, instead of relying on the dataset-info.json file.\r\n\n", "before_files": [{"content": "from dataclasses import asdict\nfrom typing import List, Optional, cast\n\nfrom datasets import get_dataset_infos\n\nfrom datasets_preview_backend.cache import memoize  # type: ignore\nfrom datasets_preview_backend.config import CACHE_TTL_SECONDS, cache\nfrom datasets_preview_backend.constants import DATASETS_BLOCKLIST\nfrom datasets_preview_backend.exceptions import Status400Error, Status404Error\nfrom datasets_preview_backend.queries.configs import get_configs_response\nfrom datasets_preview_backend.responses import CachedResponse\nfrom datasets_preview_backend.types import (\n    ConfigsContent,\n    InfoItem,\n    InfosContent,\n    StatusErrorContent,\n)\n\n\ndef get_infos(dataset: str, config: Optional[str] = None, token: Optional[str] = None) -> InfosContent:\n    if not isinstance(dataset, str) and dataset is not None:\n        raise TypeError(\"dataset argument should be a string\")\n    if dataset is None:\n        raise Status400Error(\"'dataset' is a required query parameter.\")\n    if dataset in DATASETS_BLOCKLIST:\n        raise Status400Error(\"this dataset is not supported for now.\")\n    if config is not None:\n        if not isinstance(config, str):\n            raise TypeError(\"config argument should be a string\")\n        configs = [config]\n    else:\n        content = get_configs_response(dataset=dataset, token=token).content\n        if \"configs\" not in content:\n            error = cast(StatusErrorContent, content)\n            if \"status_code\" in error and error[\"status_code\"] == 404:\n                raise Status404Error(\"configurations could not be found\")\n            raise Status400Error(\"configurations could not be found\")\n        configs_content = cast(ConfigsContent, content)\n        configs = [configItem[\"config\"] for configItem in configs_content[\"configs\"]]\n\n    infoItems: List[InfoItem] = []\n    # Note that we raise on the first error\n    for config in configs:\n        try:\n            dataset_info_dict = get_dataset_infos(dataset, use_auth_token=token)\n            if config not in dataset_info_dict:\n                # this is an error in the dataset-info.json\n                raise Exception(\"The config is not listed in the datasets infos\")\n            info = asdict(dataset_info_dict[config])\n            if \"splits\" in info:\n                info[\"splits\"] = {split_name: split_info for split_name, split_info in info[\"splits\"].items()}\n        except FileNotFoundError as err:\n            raise Status404Error(\"The config info could not be found.\") from err\n        except Exception as err:\n            raise Status400Error(\"The config info could not be parsed from the dataset.\") from err\n        infoItems.append({\"dataset\": dataset, \"config\": config, \"info\": info})\n\n    return {\"infos\": infoItems}\n\n\n@memoize(cache, expire=CACHE_TTL_SECONDS)  # type:ignore\ndef get_infos_response(*, dataset: str, config: Optional[str] = None, token: Optional[str] = None) -> CachedResponse:\n    try:\n        response = CachedResponse(get_infos(dataset, config, token))\n    except (Status400Error, Status404Error) as err:\n        response = CachedResponse(err.as_content(), err.status_code)\n    return response\n\n\ndef get_refreshed_infos(dataset: str, config: Optional[str] = None, token: Optional[str] = None) -> InfosContent:\n    return cast(InfosContent, get_infos_response(dataset, config, token, _refresh=True)[\"content\"])\n", "path": "src/datasets_preview_backend/queries/infos.py"}, {"content": "import logging\nimport re\nfrom typing import List, Optional, cast\n\nfrom datasets import IterableDataset, load_dataset\n\nfrom datasets_preview_backend.cache import memoize  # type: ignore\nfrom datasets_preview_backend.config import CACHE_TTL_SECONDS, EXTRACT_ROWS_LIMIT, cache\nfrom datasets_preview_backend.constants import DATASETS_BLOCKLIST\nfrom datasets_preview_backend.exceptions import Status400Error, Status404Error\nfrom datasets_preview_backend.queries.configs import get_configs_response\nfrom datasets_preview_backend.queries.infos import get_infos_response\nfrom datasets_preview_backend.queries.splits import get_splits_response\nfrom datasets_preview_backend.responses import CachedResponse\nfrom datasets_preview_backend.types import (\n    ConfigsContent,\n    FeatureItem,\n    InfosContent,\n    RowItem,\n    RowsContent,\n    SplitsContent,\n    StatusErrorContent,\n)\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_rows(\n    dataset: str, config: Optional[str] = None, split: Optional[str] = None, token: Optional[str] = None\n) -> RowsContent:\n    if not isinstance(dataset, str) and dataset is not None:\n        raise TypeError(\"dataset argument should be a string\")\n    if dataset is None:\n        raise Status400Error(\"'dataset' is a required query parameter.\")\n    if dataset in DATASETS_BLOCKLIST:\n        raise Status400Error(\"this dataset is not supported for now.\")\n    if config is None:\n        # split is ignored if config is not passed\n        logger.warning(\"split argument is ignored since config is not provided\")\n        split = None\n    elif not isinstance(config, str):\n        raise TypeError(\"config argument should be a string\")\n    if not isinstance(split, str) and split is not None:\n        raise TypeError(\"split argument should be a string\")\n    num_rows = EXTRACT_ROWS_LIMIT\n\n    rowItems: List[RowItem] = []\n    featureItems: List[FeatureItem] = []\n\n    if config is not None and split is not None:\n        try:\n            iterable_dataset = load_dataset(dataset, name=config, split=split, streaming=True, use_auth_token=token)\n            if not isinstance(iterable_dataset, IterableDataset):\n                raise TypeError(\"load_dataset should return an IterableDataset\")\n            rows = list(iterable_dataset.take(num_rows))\n        except FileNotFoundError as err:\n            raise Status404Error(\"The split for the dataset config could not be found.\") from err\n        except NotImplementedError as err:\n            # TODO: check what has changed once https://github.com/huggingface/datasets/pull/2662 is merged\n            try:\n                regex = re.compile(r\"Extraction protocol for file at .*?((\\.\\w+)?\\.\\w+)* is not implemented yet\")\n                match = regex.match(str(err))\n                if match is None:\n                    raise Exception(\"No match\")\n                extension = match.group(1)\n            except Exception:\n                raise Status400Error(\"The rows could not be extracted from the split of the dataset config.\") from err\n            else:\n                raise Status400Error(\n                    \"The rows could not be extracted from the split of the dataset config because extension\"\n                    f\" {extension} is not supported.\"\n                ) from err\n        except ValueError as err:\n            if (\n                str(err).startswith(f\"BuilderConfig {config} not found.\")\n                or str(err).startswith(\"Config name is missing.\")\n                or str(err).startswith(\"Bad split\")\n            ):\n                raise Status404Error(\"The dataset config could not be found.\") from err\n            else:\n                raise Status400Error(\"The rows could not be extracted from the split of the dataset config.\") from err\n        except Exception as err:\n            raise Status400Error(\"The rows could not be extracted from the split of the dataset config.\") from err\n\n        if len(rows) != num_rows:\n            logger.warning(\n                f\"could not read all the required rows ({len(rows)} / {num_rows}) from dataset {dataset} -\"\n                f\" {config} - {split}\"\n            )\n\n        rowItems = [{\"dataset\": dataset, \"config\": config, \"split\": split, \"row\": row} for row in rows]\n\n        content = get_infos_response(dataset=dataset, config=config, token=token).content\n        if \"infos\" not in content:\n            error = cast(StatusErrorContent, content)\n            if \"status_code\" in error and error[\"status_code\"] == 404:\n                raise Status404Error(\"features could not be found\")\n            raise Status400Error(\"features could not be found\")\n        infos_content = cast(InfosContent, content)\n        infoItems = [infoItem[\"info\"] for infoItem in infos_content[\"infos\"]]\n\n        if len(infoItems) != 1:\n            raise Exception(\"a dataset config should have exactly one info\")\n        infoItem = infoItems[0]\n        if \"features\" not in infoItem:\n            raise Status400Error(\"a dataset config info should contain a 'features' property\")\n        localFeaturesItems: List[FeatureItem] = [\n            {\"dataset\": dataset, \"config\": config, \"features\": infoItem[\"features\"]}\n        ]\n\n        return {\"features\": localFeaturesItems, \"rows\": rowItems}\n\n    if config is None:\n        content = get_configs_response(dataset=dataset, token=token).content\n        if \"configs\" not in content:\n            error = cast(StatusErrorContent, content)\n            if \"status_code\" in error and error[\"status_code\"] == 404:\n                raise Status404Error(\"configurations could not be found\")\n            raise Status400Error(\"configurations could not be found\")\n        configs_content = cast(ConfigsContent, content)\n        configs = [configItem[\"config\"] for configItem in configs_content[\"configs\"]]\n    else:\n        configs = [config]\n\n    # Note that we raise on the first error\n    for config in configs:\n        content = get_splits_response(dataset=dataset, config=config, token=token).content\n        if \"splits\" not in content:\n            error = cast(StatusErrorContent, content)\n            if \"status_code\" in error and error[\"status_code\"] == 404:\n                raise Status404Error(\"splits could not be found\")\n            raise Status400Error(\"splits could not be found\")\n        splits_content = cast(SplitsContent, content)\n        splits = [splitItem[\"split\"] for splitItem in splits_content[\"splits\"]]\n\n        for split in splits:\n            content = get_rows_response(dataset=dataset, config=config, split=split, token=token).content\n            if \"rows\" not in content:\n                error = cast(StatusErrorContent, content)\n                if \"status_code\" in error and error[\"status_code\"] == 404:\n                    raise Status404Error(\"rows could not be found\")\n                raise Status400Error(\"rows could not be found\")\n            rows_content = cast(RowsContent, content)\n            rowItems += rows_content[\"rows\"]\n            for featureItem in rows_content[\"features\"]:\n                # there should be only one element. Anyway, let's loop\n                if featureItem not in featureItems:\n                    featureItems.append(featureItem)\n\n    return {\"features\": featureItems, \"rows\": rowItems}\n\n\n@memoize(cache, expire=CACHE_TTL_SECONDS)  # type:ignore\ndef get_rows_response(\n    *, dataset: str, config: Optional[str] = None, split: Optional[str] = None, token: Optional[str] = None\n) -> CachedResponse:\n    try:\n        response = CachedResponse(get_rows(dataset, config, split, token))\n    except (Status400Error, Status404Error) as err:\n        response = CachedResponse(err.as_content(), err.status_code)\n    return response\n\n\ndef get_refreshed_rows(\n    dataset: str, config: Optional[str] = None, split: Optional[str] = None, token: Optional[str] = None\n) -> RowsContent:\n    return cast(RowsContent, get_rows_response(dataset, config, split, token, _refresh=True)[\"content\"])\n", "path": "src/datasets_preview_backend/queries/rows.py"}, {"content": "from typing import Any, Dict, List, TypedDict, Union\n\n\nclass DatasetItem(TypedDict):\n    dataset: str\n\n\nclass InfoItem(TypedDict):\n    dataset: str\n    config: str\n    info: Dict[str, Any]\n\n\nclass ConfigItem(TypedDict):\n    dataset: str\n    config: str\n\n\nclass SplitItem(TypedDict):\n    dataset: str\n    config: str\n    split: str\n\n\nclass RowItem(TypedDict):\n    dataset: str\n    config: str\n    split: str\n    row: Any\n\n\nclass FeatureItem(TypedDict):\n    dataset: str\n    config: str\n    features: Any\n\n\n# Content of endpoint responses\n\n\nclass DatasetsContent(TypedDict):\n    datasets: List[DatasetItem]\n\n\nclass InfosContent(TypedDict):\n    infos: List[InfoItem]\n\n\nclass ConfigsContent(TypedDict):\n    configs: List[ConfigItem]\n\n\nclass SplitsContent(TypedDict):\n    splits: List[SplitItem]\n\n\nclass RowsContent(TypedDict):\n    features: List[FeatureItem]\n    rows: List[RowItem]\n\n\nclass StatusErrorContent(TypedDict):\n    status_code: int\n    exception: str\n    message: str\n    cause: str\n    cause_message: str\n\n\nContent = Union[\n    ConfigsContent,\n    DatasetsContent,\n    InfosContent,\n    RowsContent,\n    SplitsContent,\n    StatusErrorContent,\n]\n", "path": "src/datasets_preview_backend/types.py"}], "after_files": [{"content": "from dataclasses import asdict\nfrom typing import List, Optional, cast\n\nfrom datasets import load_dataset_builder\n\nfrom datasets_preview_backend.cache import memoize  # type: ignore\nfrom datasets_preview_backend.config import CACHE_TTL_SECONDS, cache\nfrom datasets_preview_backend.constants import DATASETS_BLOCKLIST\nfrom datasets_preview_backend.exceptions import Status400Error, Status404Error\nfrom datasets_preview_backend.queries.configs import get_configs_response\nfrom datasets_preview_backend.responses import CachedResponse\nfrom datasets_preview_backend.types import (\n    ConfigsContent,\n    InfoItem,\n    InfosContent,\n    StatusErrorContent,\n)\n\n\ndef get_infos(dataset: str, config: Optional[str] = None, token: Optional[str] = None) -> InfosContent:\n    if not isinstance(dataset, str) and dataset is not None:\n        raise TypeError(\"dataset argument should be a string\")\n    if dataset is None:\n        raise Status400Error(\"'dataset' is a required query parameter.\")\n    if dataset in DATASETS_BLOCKLIST:\n        raise Status400Error(\"this dataset is not supported for now.\")\n    if config is not None:\n        if not isinstance(config, str):\n            raise TypeError(\"config argument should be a string\")\n        configs = [config]\n    else:\n        content = get_configs_response(dataset=dataset, token=token).content\n        if \"configs\" not in content:\n            error = cast(StatusErrorContent, content)\n            if \"status_code\" in error and error[\"status_code\"] == 404:\n                raise Status404Error(\"configurations could not be found\")\n            raise Status400Error(\"configurations could not be found\")\n        configs_content = cast(ConfigsContent, content)\n        configs = [configItem[\"config\"] for configItem in configs_content[\"configs\"]]\n\n    infoItems: List[InfoItem] = []\n    # Note that we raise on the first error\n    for config in configs:\n        try:\n            builder = load_dataset_builder(dataset, name=config, use_auth_token=token)\n            info = asdict(builder.info)\n            if \"splits\" in info and info[\"splits\"] is not None:\n                info[\"splits\"] = {split_name: split_info for split_name, split_info in info[\"splits\"].items()}\n        except FileNotFoundError as err:\n            raise Status404Error(\"The config info could not be found.\") from err\n        except Exception as err:\n            raise Status400Error(\"The config info could not be parsed from the dataset.\") from err\n        infoItems.append({\"dataset\": dataset, \"config\": config, \"info\": info})\n\n    return {\"infos\": infoItems}\n\n\n@memoize(cache, expire=CACHE_TTL_SECONDS)  # type:ignore\ndef get_infos_response(*, dataset: str, config: Optional[str] = None, token: Optional[str] = None) -> CachedResponse:\n    try:\n        response = CachedResponse(get_infos(dataset, config, token))\n    except (Status400Error, Status404Error) as err:\n        response = CachedResponse(err.as_content(), err.status_code)\n    return response\n\n\ndef get_refreshed_infos(dataset: str, config: Optional[str] = None, token: Optional[str] = None) -> InfosContent:\n    return cast(InfosContent, get_infos_response(dataset, config, token, _refresh=True)[\"content\"])\n", "path": "src/datasets_preview_backend/queries/infos.py"}, {"content": "import logging\nimport re\nfrom typing import List, Optional, cast\n\nfrom datasets import IterableDataset, load_dataset\n\nfrom datasets_preview_backend.cache import memoize  # type: ignore\nfrom datasets_preview_backend.config import CACHE_TTL_SECONDS, EXTRACT_ROWS_LIMIT, cache\nfrom datasets_preview_backend.constants import DATASETS_BLOCKLIST\nfrom datasets_preview_backend.exceptions import Status400Error, Status404Error\nfrom datasets_preview_backend.queries.configs import get_configs_response\nfrom datasets_preview_backend.queries.infos import get_infos_response\nfrom datasets_preview_backend.queries.splits import get_splits_response\nfrom datasets_preview_backend.responses import CachedResponse\nfrom datasets_preview_backend.types import (\n    ConfigsContent,\n    FeaturesItem,\n    InfosContent,\n    RowItem,\n    RowsContent,\n    SplitsContent,\n    StatusErrorContent,\n)\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_rows(\n    dataset: str, config: Optional[str] = None, split: Optional[str] = None, token: Optional[str] = None\n) -> RowsContent:\n    if not isinstance(dataset, str) and dataset is not None:\n        raise TypeError(\"dataset argument should be a string\")\n    if dataset is None:\n        raise Status400Error(\"'dataset' is a required query parameter.\")\n    if dataset in DATASETS_BLOCKLIST:\n        raise Status400Error(\"this dataset is not supported for now.\")\n    if config is None:\n        # split is ignored if config is not passed\n        logger.warning(\"split argument is ignored since config is not provided\")\n        split = None\n    elif not isinstance(config, str):\n        raise TypeError(\"config argument should be a string\")\n    if not isinstance(split, str) and split is not None:\n        raise TypeError(\"split argument should be a string\")\n    num_rows = EXTRACT_ROWS_LIMIT\n\n    rowItems: List[RowItem] = []\n    featuresItems: List[FeaturesItem] = []\n\n    if config is not None and split is not None:\n        try:\n            iterable_dataset = load_dataset(dataset, name=config, split=split, streaming=True, use_auth_token=token)\n            if not isinstance(iterable_dataset, IterableDataset):\n                raise TypeError(\"load_dataset should return an IterableDataset\")\n            rows = list(iterable_dataset.take(num_rows))\n        except FileNotFoundError as err:\n            raise Status404Error(\"The split for the dataset config could not be found.\") from err\n        except NotImplementedError as err:\n            # TODO: check what has changed once https://github.com/huggingface/datasets/pull/2662 is merged\n            try:\n                regex = re.compile(r\"Extraction protocol for file at .*?((\\.\\w+)?\\.\\w+)* is not implemented yet\")\n                match = regex.match(str(err))\n                if match is None:\n                    raise Exception(\"No match\")\n                extension = match.group(1)\n            except Exception:\n                raise Status400Error(\"The rows could not be extracted from the split of the dataset config.\") from err\n            else:\n                raise Status400Error(\n                    \"The rows could not be extracted from the split of the dataset config because extension\"\n                    f\" {extension} is not supported.\"\n                ) from err\n        except ValueError as err:\n            if (\n                str(err).startswith(f\"BuilderConfig {config} not found.\")\n                or str(err).startswith(\"Config name is missing.\")\n                or str(err).startswith(\"Bad split\")\n            ):\n                raise Status404Error(\"The dataset config could not be found.\") from err\n            else:\n                raise Status400Error(\"The rows could not be extracted from the split of the dataset config.\") from err\n        except Exception as err:\n            raise Status400Error(\"The rows could not be extracted from the split of the dataset config.\") from err\n\n        if len(rows) != num_rows:\n            logger.warning(\n                f\"could not read all the required rows ({len(rows)} / {num_rows}) from dataset {dataset} -\"\n                f\" {config} - {split}\"\n            )\n\n        rowItems = [{\"dataset\": dataset, \"config\": config, \"split\": split, \"row\": row} for row in rows]\n\n        content = get_infos_response(dataset=dataset, config=config, token=token).content\n        if \"infos\" not in content:\n            error = cast(StatusErrorContent, content)\n            if \"status_code\" in error and error[\"status_code\"] == 404:\n                raise Status404Error(\"features could not be found\")\n            raise Status400Error(\"features could not be found\")\n        infos_content = cast(InfosContent, content)\n        infoItems = [infoItem[\"info\"] for infoItem in infos_content[\"infos\"]]\n\n        if len(infoItems) != 1:\n            raise Exception(\"a dataset config should have exactly one info\")\n        infoItem = infoItems[0]\n        if \"features\" not in infoItem:\n            raise Status400Error(\"a dataset config info should contain a 'features' property\")\n        localFeaturesItems: List[FeaturesItem] = [\n            {\"dataset\": dataset, \"config\": config, \"features\": infoItem[\"features\"]}\n        ]\n\n        return {\"features\": localFeaturesItems, \"rows\": rowItems}\n\n    if config is None:\n        content = get_configs_response(dataset=dataset, token=token).content\n        if \"configs\" not in content:\n            error = cast(StatusErrorContent, content)\n            if \"status_code\" in error and error[\"status_code\"] == 404:\n                raise Status404Error(\"configurations could not be found\")\n            raise Status400Error(\"configurations could not be found\")\n        configs_content = cast(ConfigsContent, content)\n        configs = [configItem[\"config\"] for configItem in configs_content[\"configs\"]]\n    else:\n        configs = [config]\n\n    # Note that we raise on the first error\n    for config in configs:\n        content = get_splits_response(dataset=dataset, config=config, token=token).content\n        if \"splits\" not in content:\n            error = cast(StatusErrorContent, content)\n            if \"status_code\" in error and error[\"status_code\"] == 404:\n                raise Status404Error(\"splits could not be found\")\n            raise Status400Error(\"splits could not be found\")\n        splits_content = cast(SplitsContent, content)\n        splits = [splitItem[\"split\"] for splitItem in splits_content[\"splits\"]]\n\n        for split in splits:\n            content = get_rows_response(dataset=dataset, config=config, split=split, token=token).content\n            if \"rows\" not in content:\n                error = cast(StatusErrorContent, content)\n                if \"status_code\" in error and error[\"status_code\"] == 404:\n                    raise Status404Error(\"rows could not be found\")\n                raise Status400Error(\"rows could not be found\")\n            rows_content = cast(RowsContent, content)\n            rowItems += rows_content[\"rows\"]\n            for featuresItem in rows_content[\"features\"]:\n                # there should be only one element. Anyway, let's loop\n                if featuresItem not in featuresItems:\n                    featuresItems.append(featuresItem)\n\n    return {\"features\": featuresItems, \"rows\": rowItems}\n\n\n@memoize(cache, expire=CACHE_TTL_SECONDS)  # type:ignore\ndef get_rows_response(\n    *, dataset: str, config: Optional[str] = None, split: Optional[str] = None, token: Optional[str] = None\n) -> CachedResponse:\n    try:\n        response = CachedResponse(get_rows(dataset, config, split, token))\n    except (Status400Error, Status404Error) as err:\n        response = CachedResponse(err.as_content(), err.status_code)\n    return response\n\n\ndef get_refreshed_rows(\n    dataset: str, config: Optional[str] = None, split: Optional[str] = None, token: Optional[str] = None\n) -> RowsContent:\n    return cast(RowsContent, get_rows_response(dataset, config, split, token, _refresh=True)[\"content\"])\n", "path": "src/datasets_preview_backend/queries/rows.py"}, {"content": "from typing import Any, Dict, List, TypedDict, Union\n\n\nclass DatasetItem(TypedDict):\n    dataset: str\n\n\nclass InfoItem(TypedDict):\n    dataset: str\n    config: str\n    info: Dict[str, Any]\n\n\nclass ConfigItem(TypedDict):\n    dataset: str\n    config: str\n\n\nclass SplitItem(TypedDict):\n    dataset: str\n    config: str\n    split: str\n\n\nclass RowItem(TypedDict):\n    dataset: str\n    config: str\n    split: str\n    row: Any\n\n\nclass FeaturesItem(TypedDict):\n    dataset: str\n    config: str\n    features: Any\n\n\n# Content of endpoint responses\n\n\nclass DatasetsContent(TypedDict):\n    datasets: List[DatasetItem]\n\n\nclass InfosContent(TypedDict):\n    infos: List[InfoItem]\n\n\nclass ConfigsContent(TypedDict):\n    configs: List[ConfigItem]\n\n\nclass SplitsContent(TypedDict):\n    splits: List[SplitItem]\n\n\nclass RowsContent(TypedDict):\n    features: List[FeaturesItem]\n    rows: List[RowItem]\n\n\nclass StatusErrorContent(TypedDict):\n    status_code: int\n    exception: str\n    message: str\n    cause: str\n    cause_message: str\n\n\nContent = Union[\n    ConfigsContent,\n    DatasetsContent,\n    InfosContent,\n    RowsContent,\n    SplitsContent,\n    StatusErrorContent,\n]\n", "path": "src/datasets_preview_backend/types.py"}]}