We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
INaturalist download is broken for version="2021_train"
*Originally reported in https://discuss.pytorch.org/t/inaturalist-download-crashes-towards-the-end/149006*

> *INaturalist* download proceeds fine up to the end, then crashes with this error:
> 
> ```py
> File "/scratch/user/myenvs/env/lib/python3.8/site-packages/torchvision/datasets/utils.py", line 152, in download_url
>     raise RuntimeError("File not found or corrupted.")
> ```
> Using torchvision automatic downloading of the `2021_train` dataset.

---

The error points to

https://github.com/pytorch/vision/blob/05eae32f9663bbecad10a8d367ccbec50130e2f5/torchvision/datasets/utils.py#L151-L152

So my guess is that the MD5 checksum that we have on record is wrong:

https://github.com/pytorch/vision/blob/05eae32f9663bbecad10a8d367ccbec50130e2f5/torchvision/datasets/inaturalist.py#L24

I'll download and check. Given that this archive is > 200GB, this will take a while. I've tried with `version="2021_valid"` and this works fine. Thus, I'm guessing this is not a systematic problem.

cc @pmeier @YosuaMichael

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `torchvision/datasets/inaturalist.py`
Content:
```
1 import os
2 import os.path
3 from typing import Any, Callable, Dict, List, Optional, Union, Tuple
4 
5 from PIL import Image
6 
7 from .utils import download_and_extract_archive, verify_str_arg
8 from .vision import VisionDataset
9 
10 CATEGORIES_2021 = ["kingdom", "phylum", "class", "order", "family", "genus"]
11 
12 DATASET_URLS = {
13     "2017": "https://ml-inat-competition-datasets.s3.amazonaws.com/2017/train_val_images.tar.gz",
14     "2018": "https://ml-inat-competition-datasets.s3.amazonaws.com/2018/train_val2018.tar.gz",
15     "2019": "https://ml-inat-competition-datasets.s3.amazonaws.com/2019/train_val2019.tar.gz",
16     "2021_train": "https://ml-inat-competition-datasets.s3.amazonaws.com/2021/train.tar.gz",
17     "2021_train_mini": "https://ml-inat-competition-datasets.s3.amazonaws.com/2021/train_mini.tar.gz",
18     "2021_valid": "https://ml-inat-competition-datasets.s3.amazonaws.com/2021/val.tar.gz",
19 }
20 
21 DATASET_MD5 = {
22     "2017": "7c784ea5e424efaec655bd392f87301f",
23     "2018": "b1c6952ce38f31868cc50ea72d066cc3",
24     "2019": "c60a6e2962c9b8ccbd458d12c8582644",
25     "2021_train": "38a7bb733f7a09214d44293460ec0021",
26     "2021_train_mini": "db6ed8330e634445efc8fec83ae81442",
27     "2021_valid": "f6f6e0e242e3d4c9569ba56400938afc",
28 }
29 
30 
31 class INaturalist(VisionDataset):
32     """`iNaturalist <https://github.com/visipedia/inat_comp>`_ Dataset.
33 
34     Args:
35         root (string): Root directory of dataset where the image files are stored.
36             This class does not require/use annotation files.
37         version (string, optional): Which version of the dataset to download/use. One of
38             '2017', '2018', '2019', '2021_train', '2021_train_mini', '2021_valid'.
39             Default: `2021_train`.
40         target_type (string or list, optional): Type of target to use, for 2021 versions, one of:
41 
42             - ``full``: the full category (species)
43             - ``kingdom``: e.g. "Animalia"
44             - ``phylum``: e.g. "Arthropoda"
45             - ``class``: e.g. "Insecta"
46             - ``order``: e.g. "Coleoptera"
47             - ``family``: e.g. "Cleridae"
48             - ``genus``: e.g. "Trichodes"
49 
50             for 2017-2019 versions, one of:
51 
52             - ``full``: the full (numeric) category
53             - ``super``: the super category, e.g. "Amphibians"
54 
55             Can also be a list to output a tuple with all specified target types.
56             Defaults to ``full``.
57         transform (callable, optional): A function/transform that takes in an PIL image
58             and returns a transformed version. E.g, ``transforms.RandomCrop``
59         target_transform (callable, optional): A function/transform that takes in the
60             target and transforms it.
61         download (bool, optional): If true, downloads the dataset from the internet and
62             puts it in root directory. If dataset is already downloaded, it is not
63             downloaded again.
64     """
65 
66     def __init__(
67         self,
68         root: str,
69         version: str = "2021_train",
70         target_type: Union[List[str], str] = "full",
71         transform: Optional[Callable] = None,
72         target_transform: Optional[Callable] = None,
73         download: bool = False,
74     ) -> None:
75         self.version = verify_str_arg(version, "version", DATASET_URLS.keys())
76 
77         super().__init__(os.path.join(root, version), transform=transform, target_transform=target_transform)
78 
79         os.makedirs(root, exist_ok=True)
80         if download:
81             self.download()
82 
83         if not self._check_integrity():
84             raise RuntimeError("Dataset not found or corrupted. You can use download=True to download it")
85 
86         self.all_categories: List[str] = []
87 
88         # map: category type -> name of category -> index
89         self.categories_index: Dict[str, Dict[str, int]] = {}
90 
91         # list indexed by category id, containing mapping from category type -> index
92         self.categories_map: List[Dict[str, int]] = []
93 
94         if not isinstance(target_type, list):
95             target_type = [target_type]
96         if self.version[:4] == "2021":
97             self.target_type = [verify_str_arg(t, "target_type", ("full", *CATEGORIES_2021)) for t in target_type]
98             self._init_2021()
99         else:
100             self.target_type = [verify_str_arg(t, "target_type", ("full", "super")) for t in target_type]
101             self._init_pre2021()
102 
103         # index of all files: (full category id, filename)
104         self.index: List[Tuple[int, str]] = []
105 
106         for dir_index, dir_name in enumerate(self.all_categories):
107             files = os.listdir(os.path.join(self.root, dir_name))
108             for fname in files:
109                 self.index.append((dir_index, fname))
110 
111     def _init_2021(self) -> None:
112         """Initialize based on 2021 layout"""
113 
114         self.all_categories = sorted(os.listdir(self.root))
115 
116         # map: category type -> name of category -> index
117         self.categories_index = {k: {} for k in CATEGORIES_2021}
118 
119         for dir_index, dir_name in enumerate(self.all_categories):
120             pieces = dir_name.split("_")
121             if len(pieces) != 8:
122                 raise RuntimeError(f"Unexpected category name {dir_name}, wrong number of pieces")
123             if pieces[0] != f"{dir_index:05d}":
124                 raise RuntimeError(f"Unexpected category id {pieces[0]}, expecting {dir_index:05d}")
125             cat_map = {}
126             for cat, name in zip(CATEGORIES_2021, pieces[1:7]):
127                 if name in self.categories_index[cat]:
128                     cat_id = self.categories_index[cat][name]
129                 else:
130                     cat_id = len(self.categories_index[cat])
131                     self.categories_index[cat][name] = cat_id
132                 cat_map[cat] = cat_id
133             self.categories_map.append(cat_map)
134 
135     def _init_pre2021(self) -> None:
136         """Initialize based on 2017-2019 layout"""
137 
138         # map: category type -> name of category -> index
139         self.categories_index = {"super": {}}
140 
141         cat_index = 0
142         super_categories = sorted(os.listdir(self.root))
143         for sindex, scat in enumerate(super_categories):
144             self.categories_index["super"][scat] = sindex
145             subcategories = sorted(os.listdir(os.path.join(self.root, scat)))
146             for subcat in subcategories:
147                 if self.version == "2017":
148                     # this version does not use ids as directory names
149                     subcat_i = cat_index
150                     cat_index += 1
151                 else:
152                     try:
153                         subcat_i = int(subcat)
154                     except ValueError:
155                         raise RuntimeError(f"Unexpected non-numeric dir name: {subcat}")
156                 if subcat_i >= len(self.categories_map):
157                     old_len = len(self.categories_map)
158                     self.categories_map.extend([{}] * (subcat_i - old_len + 1))
159                     self.all_categories.extend([""] * (subcat_i - old_len + 1))
160                 if self.categories_map[subcat_i]:
161                     raise RuntimeError(f"Duplicate category {subcat}")
162                 self.categories_map[subcat_i] = {"super": sindex}
163                 self.all_categories[subcat_i] = os.path.join(scat, subcat)
164 
165         # validate the dictionary
166         for cindex, c in enumerate(self.categories_map):
167             if not c:
168                 raise RuntimeError(f"Missing category {cindex}")
169 
170     def __getitem__(self, index: int) -> Tuple[Any, Any]:
171         """
172         Args:
173             index (int): Index
174 
175         Returns:
176             tuple: (image, target) where the type of target specified by target_type.
177         """
178 
179         cat_id, fname = self.index[index]
180         img = Image.open(os.path.join(self.root, self.all_categories[cat_id], fname))
181 
182         target: Any = []
183         for t in self.target_type:
184             if t == "full":
185                 target.append(cat_id)
186             else:
187                 target.append(self.categories_map[cat_id][t])
188         target = tuple(target) if len(target) > 1 else target[0]
189 
190         if self.transform is not None:
191             img = self.transform(img)
192 
193         if self.target_transform is not None:
194             target = self.target_transform(target)
195 
196         return img, target
197 
198     def __len__(self) -> int:
199         return len(self.index)
200 
201     def category_name(self, category_type: str, category_id: int) -> str:
202         """
203         Args:
204             category_type(str): one of "full", "kingdom", "phylum", "class", "order", "family", "genus" or "super"
205             category_id(int): an index (class id) from this category
206 
207         Returns:
208             the name of the category
209         """
210         if category_type == "full":
211             return self.all_categories[category_id]
212         else:
213             if category_type not in self.categories_index:
214                 raise ValueError(f"Invalid category type '{category_type}'")
215             else:
216                 for name, id in self.categories_index[category_type].items():
217                     if id == category_id:
218                         return name
219                 raise ValueError(f"Invalid category id {category_id} for {category_type}")
220 
221     def _check_integrity(self) -> bool:
222         return os.path.exists(self.root) and len(os.listdir(self.root)) > 0
223 
224     def download(self) -> None:
225         if self._check_integrity():
226             raise RuntimeError(
227                 f"The directory {self.root} already exists. "
228                 f"If you want to re-download or re-extract the images, delete the directory."
229             )
230 
231         base_root = os.path.dirname(self.root)
232 
233         download_and_extract_archive(
234             DATASET_URLS[self.version], base_root, filename=f"{self.version}.tgz", md5=DATASET_MD5[self.version]
235         )
236 
237         orig_dir_name = os.path.join(base_root, os.path.basename(DATASET_URLS[self.version]).rstrip(".tar.gz"))
238         if not os.path.exists(orig_dir_name):
239             raise RuntimeError(f"Unable to find downloaded files at {orig_dir_name}")
240         os.rename(orig_dir_name, self.root)
241         print(f"Dataset version '{self.version}' has been downloaded and prepared for use")
242
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/torchvision/datasets/inaturalist.py b/torchvision/datasets/inaturalist.py
--- a/torchvision/datasets/inaturalist.py
+++ b/torchvision/datasets/inaturalist.py
@@ -22,7 +22,7 @@
     "2017": "7c784ea5e424efaec655bd392f87301f",
     "2018": "b1c6952ce38f31868cc50ea72d066cc3",
     "2019": "c60a6e2962c9b8ccbd458d12c8582644",
-    "2021_train": "38a7bb733f7a09214d44293460ec0021",
+    "2021_train": "e0526d53c7f7b2e3167b2b43bb2690ed",
     "2021_train_mini": "db6ed8330e634445efc8fec83ae81442",
     "2021_valid": "f6f6e0e242e3d4c9569ba56400938afc",
 }

{"golden_diff": "diff --git a/torchvision/datasets/inaturalist.py b/torchvision/datasets/inaturalist.py\n--- a/torchvision/datasets/inaturalist.py\n+++ b/torchvision/datasets/inaturalist.py\n@@ -22,7 +22,7 @@\n     \"2017\": \"7c784ea5e424efaec655bd392f87301f\",\n     \"2018\": \"b1c6952ce38f31868cc50ea72d066cc3\",\n     \"2019\": \"c60a6e2962c9b8ccbd458d12c8582644\",\n-    \"2021_train\": \"38a7bb733f7a09214d44293460ec0021\",\n+    \"2021_train\": \"e0526d53c7f7b2e3167b2b43bb2690ed\",\n     \"2021_train_mini\": \"db6ed8330e634445efc8fec83ae81442\",\n     \"2021_valid\": \"f6f6e0e242e3d4c9569ba56400938afc\",\n }\n", "issue": "INaturalist download is broken for version=\"2021_train\"\n*Originally reported in https://discuss.pytorch.org/t/inaturalist-download-crashes-towards-the-end/149006*\r\n\r\n> *INaturalist* download proceeds fine up to the end, then crashes with this error:\r\n> \r\n> ```py\r\n> File \"/scratch/user/myenvs/env/lib/python3.8/site-packages/torchvision/datasets/utils.py\", line 152, in download_url\r\n>     raise RuntimeError(\"File not found or corrupted.\")\r\n> ```\r\n> Using torchvision automatic downloading of the `2021_train` dataset.\r\n\r\n---\r\n\r\nThe error points to\r\n\r\nhttps://github.com/pytorch/vision/blob/05eae32f9663bbecad10a8d367ccbec50130e2f5/torchvision/datasets/utils.py#L151-L152\r\n\r\nSo my guess is that the MD5 checksum that we have on record is wrong:\r\n\r\nhttps://github.com/pytorch/vision/blob/05eae32f9663bbecad10a8d367ccbec50130e2f5/torchvision/datasets/inaturalist.py#L24\r\n\r\nI'll download and check. Given that this archive is > 200GB, this will take a while. I've tried with `version=\"2021_valid\"` and this works fine. Thus, I'm guessing this is not a systematic problem.\n\ncc @pmeier @YosuaMichael\n", "before_files": [{"content": "import os\nimport os.path\nfrom typing import Any, Callable, Dict, List, Optional, Union, Tuple\n\nfrom PIL import Image\n\nfrom .utils import download_and_extract_archive, verify_str_arg\nfrom .vision import VisionDataset\n\nCATEGORIES_2021 = [\"kingdom\", \"phylum\", \"class\", \"order\", \"family\", \"genus\"]\n\nDATASET_URLS = {\n    \"2017\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2017/train_val_images.tar.gz\",\n    \"2018\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2018/train_val2018.tar.gz\",\n    \"2019\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2019/train_val2019.tar.gz\",\n    \"2021_train\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2021/train.tar.gz\",\n    \"2021_train_mini\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2021/train_mini.tar.gz\",\n    \"2021_valid\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2021/val.tar.gz\",\n}\n\nDATASET_MD5 = {\n    \"2017\": \"7c784ea5e424efaec655bd392f87301f\",\n    \"2018\": \"b1c6952ce38f31868cc50ea72d066cc3\",\n    \"2019\": \"c60a6e2962c9b8ccbd458d12c8582644\",\n    \"2021_train\": \"38a7bb733f7a09214d44293460ec0021\",\n    \"2021_train_mini\": \"db6ed8330e634445efc8fec83ae81442\",\n    \"2021_valid\": \"f6f6e0e242e3d4c9569ba56400938afc\",\n}\n\n\nclass INaturalist(VisionDataset):\n    \"\"\"`iNaturalist <https://github.com/visipedia/inat_comp>`_ Dataset.\n\n    Args:\n        root (string): Root directory of dataset where the image files are stored.\n            This class does not require/use annotation files.\n        version (string, optional): Which version of the dataset to download/use. One of\n            '2017', '2018', '2019', '2021_train', '2021_train_mini', '2021_valid'.\n            Default: `2021_train`.\n        target_type (string or list, optional): Type of target to use, for 2021 versions, one of:\n\n            - ``full``: the full category (species)\n            - ``kingdom``: e.g. \"Animalia\"\n            - ``phylum``: e.g. \"Arthropoda\"\n            - ``class``: e.g. \"Insecta\"\n            - ``order``: e.g. \"Coleoptera\"\n            - ``family``: e.g. \"Cleridae\"\n            - ``genus``: e.g. \"Trichodes\"\n\n            for 2017-2019 versions, one of:\n\n            - ``full``: the full (numeric) category\n            - ``super``: the super category, e.g. \"Amphibians\"\n\n            Can also be a list to output a tuple with all specified target types.\n            Defaults to ``full``.\n        transform (callable, optional): A function/transform that takes in an PIL image\n            and returns a transformed version. E.g, ``transforms.RandomCrop``\n        target_transform (callable, optional): A function/transform that takes in the\n            target and transforms it.\n        download (bool, optional): If true, downloads the dataset from the internet and\n            puts it in root directory. If dataset is already downloaded, it is not\n            downloaded again.\n    \"\"\"\n\n    def __init__(\n        self,\n        root: str,\n        version: str = \"2021_train\",\n        target_type: Union[List[str], str] = \"full\",\n        transform: Optional[Callable] = None,\n        target_transform: Optional[Callable] = None,\n        download: bool = False,\n    ) -> None:\n        self.version = verify_str_arg(version, \"version\", DATASET_URLS.keys())\n\n        super().__init__(os.path.join(root, version), transform=transform, target_transform=target_transform)\n\n        os.makedirs(root, exist_ok=True)\n        if download:\n            self.download()\n\n        if not self._check_integrity():\n            raise RuntimeError(\"Dataset not found or corrupted. You can use download=True to download it\")\n\n        self.all_categories: List[str] = []\n\n        # map: category type -> name of category -> index\n        self.categories_index: Dict[str, Dict[str, int]] = {}\n\n        # list indexed by category id, containing mapping from category type -> index\n        self.categories_map: List[Dict[str, int]] = []\n\n        if not isinstance(target_type, list):\n            target_type = [target_type]\n        if self.version[:4] == \"2021\":\n            self.target_type = [verify_str_arg(t, \"target_type\", (\"full\", *CATEGORIES_2021)) for t in target_type]\n            self._init_2021()\n        else:\n            self.target_type = [verify_str_arg(t, \"target_type\", (\"full\", \"super\")) for t in target_type]\n            self._init_pre2021()\n\n        # index of all files: (full category id, filename)\n        self.index: List[Tuple[int, str]] = []\n\n        for dir_index, dir_name in enumerate(self.all_categories):\n            files = os.listdir(os.path.join(self.root, dir_name))\n            for fname in files:\n                self.index.append((dir_index, fname))\n\n    def _init_2021(self) -> None:\n        \"\"\"Initialize based on 2021 layout\"\"\"\n\n        self.all_categories = sorted(os.listdir(self.root))\n\n        # map: category type -> name of category -> index\n        self.categories_index = {k: {} for k in CATEGORIES_2021}\n\n        for dir_index, dir_name in enumerate(self.all_categories):\n            pieces = dir_name.split(\"_\")\n            if len(pieces) != 8:\n                raise RuntimeError(f\"Unexpected category name {dir_name}, wrong number of pieces\")\n            if pieces[0] != f\"{dir_index:05d}\":\n                raise RuntimeError(f\"Unexpected category id {pieces[0]}, expecting {dir_index:05d}\")\n            cat_map = {}\n            for cat, name in zip(CATEGORIES_2021, pieces[1:7]):\n                if name in self.categories_index[cat]:\n                    cat_id = self.categories_index[cat][name]\n                else:\n                    cat_id = len(self.categories_index[cat])\n                    self.categories_index[cat][name] = cat_id\n                cat_map[cat] = cat_id\n            self.categories_map.append(cat_map)\n\n    def _init_pre2021(self) -> None:\n        \"\"\"Initialize based on 2017-2019 layout\"\"\"\n\n        # map: category type -> name of category -> index\n        self.categories_index = {\"super\": {}}\n\n        cat_index = 0\n        super_categories = sorted(os.listdir(self.root))\n        for sindex, scat in enumerate(super_categories):\n            self.categories_index[\"super\"][scat] = sindex\n            subcategories = sorted(os.listdir(os.path.join(self.root, scat)))\n            for subcat in subcategories:\n                if self.version == \"2017\":\n                    # this version does not use ids as directory names\n                    subcat_i = cat_index\n                    cat_index += 1\n                else:\n                    try:\n                        subcat_i = int(subcat)\n                    except ValueError:\n                        raise RuntimeError(f\"Unexpected non-numeric dir name: {subcat}\")\n                if subcat_i >= len(self.categories_map):\n                    old_len = len(self.categories_map)\n                    self.categories_map.extend([{}] * (subcat_i - old_len + 1))\n                    self.all_categories.extend([\"\"] * (subcat_i - old_len + 1))\n                if self.categories_map[subcat_i]:\n                    raise RuntimeError(f\"Duplicate category {subcat}\")\n                self.categories_map[subcat_i] = {\"super\": sindex}\n                self.all_categories[subcat_i] = os.path.join(scat, subcat)\n\n        # validate the dictionary\n        for cindex, c in enumerate(self.categories_map):\n            if not c:\n                raise RuntimeError(f\"Missing category {cindex}\")\n\n    def __getitem__(self, index: int) -> Tuple[Any, Any]:\n        \"\"\"\n        Args:\n            index (int): Index\n\n        Returns:\n            tuple: (image, target) where the type of target specified by target_type.\n        \"\"\"\n\n        cat_id, fname = self.index[index]\n        img = Image.open(os.path.join(self.root, self.all_categories[cat_id], fname))\n\n        target: Any = []\n        for t in self.target_type:\n            if t == \"full\":\n                target.append(cat_id)\n            else:\n                target.append(self.categories_map[cat_id][t])\n        target = tuple(target) if len(target) > 1 else target[0]\n\n        if self.transform is not None:\n            img = self.transform(img)\n\n        if self.target_transform is not None:\n            target = self.target_transform(target)\n\n        return img, target\n\n    def __len__(self) -> int:\n        return len(self.index)\n\n    def category_name(self, category_type: str, category_id: int) -> str:\n        \"\"\"\n        Args:\n            category_type(str): one of \"full\", \"kingdom\", \"phylum\", \"class\", \"order\", \"family\", \"genus\" or \"super\"\n            category_id(int): an index (class id) from this category\n\n        Returns:\n            the name of the category\n        \"\"\"\n        if category_type == \"full\":\n            return self.all_categories[category_id]\n        else:\n            if category_type not in self.categories_index:\n                raise ValueError(f\"Invalid category type '{category_type}'\")\n            else:\n                for name, id in self.categories_index[category_type].items():\n                    if id == category_id:\n                        return name\n                raise ValueError(f\"Invalid category id {category_id} for {category_type}\")\n\n    def _check_integrity(self) -> bool:\n        return os.path.exists(self.root) and len(os.listdir(self.root)) > 0\n\n    def download(self) -> None:\n        if self._check_integrity():\n            raise RuntimeError(\n                f\"The directory {self.root} already exists. \"\n                f\"If you want to re-download or re-extract the images, delete the directory.\"\n            )\n\n        base_root = os.path.dirname(self.root)\n\n        download_and_extract_archive(\n            DATASET_URLS[self.version], base_root, filename=f\"{self.version}.tgz\", md5=DATASET_MD5[self.version]\n        )\n\n        orig_dir_name = os.path.join(base_root, os.path.basename(DATASET_URLS[self.version]).rstrip(\".tar.gz\"))\n        if not os.path.exists(orig_dir_name):\n            raise RuntimeError(f\"Unable to find downloaded files at {orig_dir_name}\")\n        os.rename(orig_dir_name, self.root)\n        print(f\"Dataset version '{self.version}' has been downloaded and prepared for use\")\n", "path": "torchvision/datasets/inaturalist.py"}], "after_files": [{"content": "import os\nimport os.path\nfrom typing import Any, Callable, Dict, List, Optional, Union, Tuple\n\nfrom PIL import Image\n\nfrom .utils import download_and_extract_archive, verify_str_arg\nfrom .vision import VisionDataset\n\nCATEGORIES_2021 = [\"kingdom\", \"phylum\", \"class\", \"order\", \"family\", \"genus\"]\n\nDATASET_URLS = {\n    \"2017\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2017/train_val_images.tar.gz\",\n    \"2018\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2018/train_val2018.tar.gz\",\n    \"2019\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2019/train_val2019.tar.gz\",\n    \"2021_train\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2021/train.tar.gz\",\n    \"2021_train_mini\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2021/train_mini.tar.gz\",\n    \"2021_valid\": \"https://ml-inat-competition-datasets.s3.amazonaws.com/2021/val.tar.gz\",\n}\n\nDATASET_MD5 = {\n    \"2017\": \"7c784ea5e424efaec655bd392f87301f\",\n    \"2018\": \"b1c6952ce38f31868cc50ea72d066cc3\",\n    \"2019\": \"c60a6e2962c9b8ccbd458d12c8582644\",\n    \"2021_train\": \"e0526d53c7f7b2e3167b2b43bb2690ed\",\n    \"2021_train_mini\": \"db6ed8330e634445efc8fec83ae81442\",\n    \"2021_valid\": \"f6f6e0e242e3d4c9569ba56400938afc\",\n}\n\n\nclass INaturalist(VisionDataset):\n    \"\"\"`iNaturalist <https://github.com/visipedia/inat_comp>`_ Dataset.\n\n    Args:\n        root (string): Root directory of dataset where the image files are stored.\n            This class does not require/use annotation files.\n        version (string, optional): Which version of the dataset to download/use. One of\n            '2017', '2018', '2019', '2021_train', '2021_train_mini', '2021_valid'.\n            Default: `2021_train`.\n        target_type (string or list, optional): Type of target to use, for 2021 versions, one of:\n\n            - ``full``: the full category (species)\n            - ``kingdom``: e.g. \"Animalia\"\n            - ``phylum``: e.g. \"Arthropoda\"\n            - ``class``: e.g. \"Insecta\"\n            - ``order``: e.g. \"Coleoptera\"\n            - ``family``: e.g. \"Cleridae\"\n            - ``genus``: e.g. \"Trichodes\"\n\n            for 2017-2019 versions, one of:\n\n            - ``full``: the full (numeric) category\n            - ``super``: the super category, e.g. \"Amphibians\"\n\n            Can also be a list to output a tuple with all specified target types.\n            Defaults to ``full``.\n        transform (callable, optional): A function/transform that takes in an PIL image\n            and returns a transformed version. E.g, ``transforms.RandomCrop``\n        target_transform (callable, optional): A function/transform that takes in the\n            target and transforms it.\n        download (bool, optional): If true, downloads the dataset from the internet and\n            puts it in root directory. If dataset is already downloaded, it is not\n            downloaded again.\n    \"\"\"\n\n    def __init__(\n        self,\n        root: str,\n        version: str = \"2021_train\",\n        target_type: Union[List[str], str] = \"full\",\n        transform: Optional[Callable] = None,\n        target_transform: Optional[Callable] = None,\n        download: bool = False,\n    ) -> None:\n        self.version = verify_str_arg(version, \"version\", DATASET_URLS.keys())\n\n        super().__init__(os.path.join(root, version), transform=transform, target_transform=target_transform)\n\n        os.makedirs(root, exist_ok=True)\n        if download:\n            self.download()\n\n        if not self._check_integrity():\n            raise RuntimeError(\"Dataset not found or corrupted. You can use download=True to download it\")\n\n        self.all_categories: List[str] = []\n\n        # map: category type -> name of category -> index\n        self.categories_index: Dict[str, Dict[str, int]] = {}\n\n        # list indexed by category id, containing mapping from category type -> index\n        self.categories_map: List[Dict[str, int]] = []\n\n        if not isinstance(target_type, list):\n            target_type = [target_type]\n        if self.version[:4] == \"2021\":\n            self.target_type = [verify_str_arg(t, \"target_type\", (\"full\", *CATEGORIES_2021)) for t in target_type]\n            self._init_2021()\n        else:\n            self.target_type = [verify_str_arg(t, \"target_type\", (\"full\", \"super\")) for t in target_type]\n            self._init_pre2021()\n\n        # index of all files: (full category id, filename)\n        self.index: List[Tuple[int, str]] = []\n\n        for dir_index, dir_name in enumerate(self.all_categories):\n            files = os.listdir(os.path.join(self.root, dir_name))\n            for fname in files:\n                self.index.append((dir_index, fname))\n\n    def _init_2021(self) -> None:\n        \"\"\"Initialize based on 2021 layout\"\"\"\n\n        self.all_categories = sorted(os.listdir(self.root))\n\n        # map: category type -> name of category -> index\n        self.categories_index = {k: {} for k in CATEGORIES_2021}\n\n        for dir_index, dir_name in enumerate(self.all_categories):\n            pieces = dir_name.split(\"_\")\n            if len(pieces) != 8:\n                raise RuntimeError(f\"Unexpected category name {dir_name}, wrong number of pieces\")\n            if pieces[0] != f\"{dir_index:05d}\":\n                raise RuntimeError(f\"Unexpected category id {pieces[0]}, expecting {dir_index:05d}\")\n            cat_map = {}\n            for cat, name in zip(CATEGORIES_2021, pieces[1:7]):\n                if name in self.categories_index[cat]:\n                    cat_id = self.categories_index[cat][name]\n                else:\n                    cat_id = len(self.categories_index[cat])\n                    self.categories_index[cat][name] = cat_id\n                cat_map[cat] = cat_id\n            self.categories_map.append(cat_map)\n\n    def _init_pre2021(self) -> None:\n        \"\"\"Initialize based on 2017-2019 layout\"\"\"\n\n        # map: category type -> name of category -> index\n        self.categories_index = {\"super\": {}}\n\n        cat_index = 0\n        super_categories = sorted(os.listdir(self.root))\n        for sindex, scat in enumerate(super_categories):\n            self.categories_index[\"super\"][scat] = sindex\n            subcategories = sorted(os.listdir(os.path.join(self.root, scat)))\n            for subcat in subcategories:\n                if self.version == \"2017\":\n                    # this version does not use ids as directory names\n                    subcat_i = cat_index\n                    cat_index += 1\n                else:\n                    try:\n                        subcat_i = int(subcat)\n                    except ValueError:\n                        raise RuntimeError(f\"Unexpected non-numeric dir name: {subcat}\")\n                if subcat_i >= len(self.categories_map):\n                    old_len = len(self.categories_map)\n                    self.categories_map.extend([{}] * (subcat_i - old_len + 1))\n                    self.all_categories.extend([\"\"] * (subcat_i - old_len + 1))\n                if self.categories_map[subcat_i]:\n                    raise RuntimeError(f\"Duplicate category {subcat}\")\n                self.categories_map[subcat_i] = {\"super\": sindex}\n                self.all_categories[subcat_i] = os.path.join(scat, subcat)\n\n        # validate the dictionary\n        for cindex, c in enumerate(self.categories_map):\n            if not c:\n                raise RuntimeError(f\"Missing category {cindex}\")\n\n    def __getitem__(self, index: int) -> Tuple[Any, Any]:\n        \"\"\"\n        Args:\n            index (int): Index\n\n        Returns:\n            tuple: (image, target) where the type of target specified by target_type.\n        \"\"\"\n\n        cat_id, fname = self.index[index]\n        img = Image.open(os.path.join(self.root, self.all_categories[cat_id], fname))\n\n        target: Any = []\n        for t in self.target_type:\n            if t == \"full\":\n                target.append(cat_id)\n            else:\n                target.append(self.categories_map[cat_id][t])\n        target = tuple(target) if len(target) > 1 else target[0]\n\n        if self.transform is not None:\n            img = self.transform(img)\n\n        if self.target_transform is not None:\n            target = self.target_transform(target)\n\n        return img, target\n\n    def __len__(self) -> int:\n        return len(self.index)\n\n    def category_name(self, category_type: str, category_id: int) -> str:\n        \"\"\"\n        Args:\n            category_type(str): one of \"full\", \"kingdom\", \"phylum\", \"class\", \"order\", \"family\", \"genus\" or \"super\"\n            category_id(int): an index (class id) from this category\n\n        Returns:\n            the name of the category\n        \"\"\"\n        if category_type == \"full\":\n            return self.all_categories[category_id]\n        else:\n            if category_type not in self.categories_index:\n                raise ValueError(f\"Invalid category type '{category_type}'\")\n            else:\n                for name, id in self.categories_index[category_type].items():\n                    if id == category_id:\n                        return name\n                raise ValueError(f\"Invalid category id {category_id} for {category_type}\")\n\n    def _check_integrity(self) -> bool:\n        return os.path.exists(self.root) and len(os.listdir(self.root)) > 0\n\n    def download(self) -> None:\n        if self._check_integrity():\n            raise RuntimeError(\n                f\"The directory {self.root} already exists. \"\n                f\"If you want to re-download or re-extract the images, delete the directory.\"\n            )\n\n        base_root = os.path.dirname(self.root)\n\n        download_and_extract_archive(\n            DATASET_URLS[self.version], base_root, filename=f\"{self.version}.tgz\", md5=DATASET_MD5[self.version]\n        )\n\n        orig_dir_name = os.path.join(base_root, os.path.basename(DATASET_URLS[self.version]).rstrip(\".tar.gz\"))\n        if not os.path.exists(orig_dir_name):\n            raise RuntimeError(f\"Unable to find downloaded files at {orig_dir_name}\")\n        os.rename(orig_dir_name, self.root)\n        print(f\"Dataset version '{self.version}' has been downloaded and prepared for use\")\n", "path": "torchvision/datasets/inaturalist.py"}]}