We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Lightly Dataset is slow at indexing larger datasets
`LightlyDataset` is significantly slower than a simple `os.listdir` (500ms vs 50ms with a folder of 100k images). Let's figure out whether we can improve.

**Tasks:**

- [x] Figure out where we spend most of the time
- [x] Figure out whether we can improve

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `lightly/data/_helpers.py`
Content:
```
1 """ Helper Functions """
2 
3 # Copyright (c) 2020. Lightly AG and its affiliates.
4 # All Rights Reserved
5 
6 import os
7 from torchvision import datasets
8 
9 from lightly.data._image import DatasetFolder
10 
11 try:
12     from lightly.data._video import VideoDataset
13     VIDEO_DATASET_AVAILABLE = True
14 except Exception as e:
15     VIDEO_DATASET_AVAILABLE = False
16     VIDEO_DATASET_ERRORMSG = e
17 
18 
19 IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp',
20                   '.pgm', '.tif', '.tiff', '.webp')
21 
22 VIDEO_EXTENSIONS = ('.mp4', '.mov', '.avi', '.mpg',
23                     '.hevc', '.m4v', '.webm', '.mpeg')
24 
25 
26 def _contains_videos(root: str, extensions: tuple):
27     """Checks whether directory contains video files.
28 
29     Args:
30         root: Root directory path.
31 
32     Returns:
33         True if root contains subdirectories else false.
34     """
35     list_dir = os.listdir(root)
36     is_video = \
37         [f.lower().endswith(extensions) for f in list_dir]
38     return any(is_video)
39 
40 
41 def _is_lightly_output_dir(dirname: str):
42     """Checks whether the directory is a lightly_output directory.
43 
44     Args:
45         dirname: Directory to check.
46 
47     Returns:
48         True if dirname is "lightly_outputs" else false.
49 
50     """
51     return 'lightly_outputs' in dirname
52 
53 
54 def _contains_subdirs(root: str):
55     """Checks whether directory contains subdirectories.
56 
57     Args:
58         root: Root directory path.
59 
60     Returns:
61         True if root contains subdirectories else false.
62 
63     """
64     list_dir = os.listdir(root)
65     list_dir = list(filter(lambda x: not _is_lightly_output_dir(x), list_dir))
66     is_dir = \
67         [os.path.isdir(os.path.join(root, f)) for f in list_dir]
68     return any(is_dir)
69 
70 
71 def _load_dataset_from_folder(root: str, transform):
72     """Initializes dataset from folder.
73 
74     Args:
75         root: (str) Root directory path
76         transform: (torchvision.transforms.Compose) image transformations
77 
78     Returns:
79         Dataset consisting of images in the root directory.
80 
81     """
82 
83     # if there is a video in the input directory but we do not have
84     # the right dependencies, raise a ValueError
85     contains_videos = _contains_videos(root, VIDEO_EXTENSIONS)
86     if contains_videos and not VIDEO_DATASET_AVAILABLE:
87         raise ValueError(f'The input directory {root} contains videos '
88                          'but the VideoDataset is not available. \n'
89                          'Make sure you have installed the right '
90                          'dependencies. The error from the imported '
91                          f'module was: {VIDEO_DATASET_ERRORMSG}')
92 
93     if contains_videos:
94         # root contains videos -> create a video dataset
95         dataset = VideoDataset(root,
96                                extensions=VIDEO_EXTENSIONS,
97                                transform=transform)
98     elif _contains_subdirs(root):
99         # root contains subdirectories -> create an image folder dataset
100         dataset = datasets.ImageFolder(root,
101                                        transform=transform)
102     else:
103         # root contains plain images -> create a folder dataset
104         dataset = DatasetFolder(root,
105                                 extensions=IMG_EXTENSIONS,
106                                 transform=transform)
107 
108     return dataset
109 
110 
111 def _load_dataset(input_dir: str,
112                   transform=None):
113     """Initializes dataset from torchvision or from folder.
114 
115     Args:
116         root: (str) Directory where dataset is stored
117         name: (str) Name of the dataset (e.g. cifar10, cifar100)
118         train: (bool) Use the training set
119         download: (bool) Download the dataset
120         transform: (torchvision.transforms.Compose) image transformations
121         from_folder: (str) Path to directory holding the images to load.
122 
123     Returns:
124         A torchvision dataset
125 
126     Raises:
127         ValueError: If the specified dataset doesn't exist
128 
129     """
130 
131     if not os.path.exists(input_dir):
132         raise ValueError(f'The input directory {input_dir} does not exist!')
133 
134     return _load_dataset_from_folder(input_dir, transform)
135
```

Path: `lightly/data/_image.py`
Content:
```
1 """ Image Dataset """
2 
3 # Copyright (c) 2020. Lightly AG and its affiliates.
4 # All Rights Reserved
5 
6 import os
7 import torchvision.datasets as datasets
8 
9 from lightly.data._image_loaders import default_loader
10 
11 
12 def _make_dataset(directory, extensions=None, is_valid_file=None):
13     """Returns a list of all image files with targets in the directory.
14 
15     Args:
16         directory:
17             Root directory path (should not contain subdirectories!).
18         extensions:
19             Tuple of valid extensions.
20         is_valid_file:
21             Used to find valid files.
22 
23     Returns:
24         List of instance tuples: (path_i, target_i = 0).
25 
26     """
27 
28     # handle is_valid_file and extensions the same way torchvision handles them:
29     # https://pytorch.org/docs/stable/_modules/torchvision/datasets/folder.html#ImageFolder
30     both_none = extensions is None and is_valid_file is None
31     both_something = extensions is not None and is_valid_file is not None
32     if both_none or both_something:
33         raise ValueError('Both extensions and is_valid_file cannot be None or '
34                          'not None at the same time')
35 
36     if extensions is not None:
37         def _is_valid_file(filename):
38             return filename.lower().endswith(extensions)
39 
40     if is_valid_file is not None:
41         _is_valid_file = is_valid_file
42 
43     instances = []
44     for fname in os.listdir(directory):
45 
46         if not _is_valid_file(fname):
47             continue
48 
49         path = os.path.join(directory, fname)
50         item = (path, 0)
51         instances.append(item)
52 
53     return instances
54 
55 
56 class DatasetFolder(datasets.VisionDataset):
57     """Implements a dataset folder.
58     
59     DatasetFolder based on torchvisions implementation.
60     (https://pytorch.org/docs/stable/torchvision/datasets.html#datasetfolder)
61 
62     Attributes:
63         root:
64             Root directory path
65         loader:
66             Function that loads file at path
67         extensions:
68             Tuple of allowed extensions
69         transform:
70             Function that takes a PIL image and returns transformed version
71         target_transform:
72             As transform but for targets
73         is_valid_file:
74             Used to check corrupt files
75 
76     Raises:
77         RuntimeError: If no supported files are found in root.
78 
79     """
80 
81     def __init__(self,
82                  root: str,
83                  loader=default_loader,
84                  extensions=None,
85                  transform=None,
86                  target_transform=None,
87                  is_valid_file=None):
88 
89         super(DatasetFolder, self).__init__(root,
90                                             transform=transform,
91                                             target_transform=target_transform)
92 
93         samples = _make_dataset(self.root, extensions, is_valid_file)
94         if len(samples) == 0:
95             msg = 'Found 0 files in folder: {}\n'.format(self.root)
96             if extensions is not None:
97                 msg += 'Supported extensions are: {}'.format(
98                     ','.join(extensions))
99             raise RuntimeError(msg)
100 
101         self.loader = loader
102         self.extensions = extensions
103 
104         self.samples = samples
105         self.targets = [s[1] for s in samples]
106 
107     def __getitem__(self, index: int):
108         """Returns item at index.
109 
110         Args:
111             index:
112                 Index of the sample to retrieve.
113 
114         Returns:
115             A tuple (sample, target) where target is 0.
116 
117         """
118 
119         path, target = self.samples[index]
120         sample = self.loader(path)
121         if self.transform is not None:
122             sample = self.transform(sample)
123         if self.target_transform is not None:
124             target = self.target_transform(target)
125 
126         return sample, target
127 
128     def __len__(self):
129         """Returns the number of samples in the dataset.
130 
131         """
132         return len(self.samples)
133
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/lightly/data/_helpers.py b/lightly/data/_helpers.py
--- a/lightly/data/_helpers.py
+++ b/lightly/data/_helpers.py
@@ -32,10 +32,8 @@
     Returns:
         True if root contains subdirectories else false.
     """
-    list_dir = os.listdir(root)
-    is_video = \
-        [f.lower().endswith(extensions) for f in list_dir]
-    return any(is_video)
+    with os.scandir(root) as scan_dir:
+        return any(f.name.lower().endswith(extensions) for f in scan_dir)
 
 
 def _is_lightly_output_dir(dirname: str):
@@ -61,11 +59,9 @@
         True if root contains subdirectories else false.
 
     """
-    list_dir = os.listdir(root)
-    list_dir = list(filter(lambda x: not _is_lightly_output_dir(x), list_dir))
-    is_dir = \
-        [os.path.isdir(os.path.join(root, f)) for f in list_dir]
-    return any(is_dir)
+    with os.scandir(root) as scan_dir:
+        return any(not _is_lightly_output_dir(f.name) for f in scan_dir \
+            if f.is_dir())
 
 
 def _load_dataset_from_folder(root: str, transform):
diff --git a/lightly/data/_image.py b/lightly/data/_image.py
--- a/lightly/data/_image.py
+++ b/lightly/data/_image.py
@@ -3,13 +3,15 @@
 # Copyright (c) 2020. Lightly AG and its affiliates.
 # All Rights Reserved
 
+from typing import List, Tuple
+
 import os
 import torchvision.datasets as datasets
 
 from lightly.data._image_loaders import default_loader
 
 
-def _make_dataset(directory, extensions=None, is_valid_file=None):
+def _make_dataset(directory, extensions=None, is_valid_file=None) -> List[Tuple[str, int]]:
     """Returns a list of all image files with targets in the directory.
 
     Args:
@@ -41,16 +43,17 @@
         _is_valid_file = is_valid_file
 
     instances = []
-    for fname in os.listdir(directory):
+    for f in os.scandir(directory):
 
-        if not _is_valid_file(fname):
+        if not _is_valid_file(f.name):
             continue
 
-        path = os.path.join(directory, fname)
-        item = (path, 0)
+        # convention: the label of all images is 0, based on the fact that
+        # they are all in the same directory
+        item = (f.path, 0)
         instances.append(item)
 
-    return instances
+    return sorted(instances, key=lambda x: x[0]) # sort by path
 
 
 class DatasetFolder(datasets.VisionDataset):

{"golden_diff": "diff --git a/lightly/data/_helpers.py b/lightly/data/_helpers.py\n--- a/lightly/data/_helpers.py\n+++ b/lightly/data/_helpers.py\n@@ -32,10 +32,8 @@\n     Returns:\n         True if root contains subdirectories else false.\n     \"\"\"\n-    list_dir = os.listdir(root)\n-    is_video = \\\n-        [f.lower().endswith(extensions) for f in list_dir]\n-    return any(is_video)\n+    with os.scandir(root) as scan_dir:\n+        return any(f.name.lower().endswith(extensions) for f in scan_dir)\n \n \n def _is_lightly_output_dir(dirname: str):\n@@ -61,11 +59,9 @@\n         True if root contains subdirectories else false.\n \n     \"\"\"\n-    list_dir = os.listdir(root)\n-    list_dir = list(filter(lambda x: not _is_lightly_output_dir(x), list_dir))\n-    is_dir = \\\n-        [os.path.isdir(os.path.join(root, f)) for f in list_dir]\n-    return any(is_dir)\n+    with os.scandir(root) as scan_dir:\n+        return any(not _is_lightly_output_dir(f.name) for f in scan_dir \\\n+            if f.is_dir())\n \n \n def _load_dataset_from_folder(root: str, transform):\ndiff --git a/lightly/data/_image.py b/lightly/data/_image.py\n--- a/lightly/data/_image.py\n+++ b/lightly/data/_image.py\n@@ -3,13 +3,15 @@\n # Copyright (c) 2020. Lightly AG and its affiliates.\n # All Rights Reserved\n \n+from typing import List, Tuple\n+\n import os\n import torchvision.datasets as datasets\n \n from lightly.data._image_loaders import default_loader\n \n \n-def _make_dataset(directory, extensions=None, is_valid_file=None):\n+def _make_dataset(directory, extensions=None, is_valid_file=None) -> List[Tuple[str, int]]:\n     \"\"\"Returns a list of all image files with targets in the directory.\n \n     Args:\n@@ -41,16 +43,17 @@\n         _is_valid_file = is_valid_file\n \n     instances = []\n-    for fname in os.listdir(directory):\n+    for f in os.scandir(directory):\n \n-        if not _is_valid_file(fname):\n+        if not _is_valid_file(f.name):\n             continue\n \n-        path = os.path.join(directory, fname)\n-        item = (path, 0)\n+        # convention: the label of all images is 0, based on the fact that\n+        # they are all in the same directory\n+        item = (f.path, 0)\n         instances.append(item)\n \n-    return instances\n+    return sorted(instances, key=lambda x: x[0]) # sort by path\n \n \n class DatasetFolder(datasets.VisionDataset):\n", "issue": "Lightly Dataset is slow at indexing larger datasets\n`LightlyDataset` is significantly slower than a simple `os.listdir` (500ms vs 50ms with a folder of 100k images). Let's figure out whether we can improve.\r\n\r\n**Tasks:**\r\n\r\n- [x] Figure out where we spend most of the time\r\n- [x] Figure out whether we can improve\n", "before_files": [{"content": "\"\"\" Helper Functions \"\"\"\n\n# Copyright (c) 2020. Lightly AG and its affiliates.\n# All Rights Reserved\n\nimport os\nfrom torchvision import datasets\n\nfrom lightly.data._image import DatasetFolder\n\ntry:\n    from lightly.data._video import VideoDataset\n    VIDEO_DATASET_AVAILABLE = True\nexcept Exception as e:\n    VIDEO_DATASET_AVAILABLE = False\n    VIDEO_DATASET_ERRORMSG = e\n\n\nIMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp',\n                  '.pgm', '.tif', '.tiff', '.webp')\n\nVIDEO_EXTENSIONS = ('.mp4', '.mov', '.avi', '.mpg',\n                    '.hevc', '.m4v', '.webm', '.mpeg')\n\n\ndef _contains_videos(root: str, extensions: tuple):\n    \"\"\"Checks whether directory contains video files.\n\n    Args:\n        root: Root directory path.\n\n    Returns:\n        True if root contains subdirectories else false.\n    \"\"\"\n    list_dir = os.listdir(root)\n    is_video = \\\n        [f.lower().endswith(extensions) for f in list_dir]\n    return any(is_video)\n\n\ndef _is_lightly_output_dir(dirname: str):\n    \"\"\"Checks whether the directory is a lightly_output directory.\n\n    Args:\n        dirname: Directory to check.\n\n    Returns:\n        True if dirname is \"lightly_outputs\" else false.\n\n    \"\"\"\n    return 'lightly_outputs' in dirname\n\n\ndef _contains_subdirs(root: str):\n    \"\"\"Checks whether directory contains subdirectories.\n\n    Args:\n        root: Root directory path.\n\n    Returns:\n        True if root contains subdirectories else false.\n\n    \"\"\"\n    list_dir = os.listdir(root)\n    list_dir = list(filter(lambda x: not _is_lightly_output_dir(x), list_dir))\n    is_dir = \\\n        [os.path.isdir(os.path.join(root, f)) for f in list_dir]\n    return any(is_dir)\n\n\ndef _load_dataset_from_folder(root: str, transform):\n    \"\"\"Initializes dataset from folder.\n\n    Args:\n        root: (str) Root directory path\n        transform: (torchvision.transforms.Compose) image transformations\n\n    Returns:\n        Dataset consisting of images in the root directory.\n\n    \"\"\"\n\n    # if there is a video in the input directory but we do not have\n    # the right dependencies, raise a ValueError\n    contains_videos = _contains_videos(root, VIDEO_EXTENSIONS)\n    if contains_videos and not VIDEO_DATASET_AVAILABLE:\n        raise ValueError(f'The input directory {root} contains videos '\n                         'but the VideoDataset is not available. \\n'\n                         'Make sure you have installed the right '\n                         'dependencies. The error from the imported '\n                         f'module was: {VIDEO_DATASET_ERRORMSG}')\n\n    if contains_videos:\n        # root contains videos -> create a video dataset\n        dataset = VideoDataset(root,\n                               extensions=VIDEO_EXTENSIONS,\n                               transform=transform)\n    elif _contains_subdirs(root):\n        # root contains subdirectories -> create an image folder dataset\n        dataset = datasets.ImageFolder(root,\n                                       transform=transform)\n    else:\n        # root contains plain images -> create a folder dataset\n        dataset = DatasetFolder(root,\n                                extensions=IMG_EXTENSIONS,\n                                transform=transform)\n\n    return dataset\n\n\ndef _load_dataset(input_dir: str,\n                  transform=None):\n    \"\"\"Initializes dataset from torchvision or from folder.\n\n    Args:\n        root: (str) Directory where dataset is stored\n        name: (str) Name of the dataset (e.g. cifar10, cifar100)\n        train: (bool) Use the training set\n        download: (bool) Download the dataset\n        transform: (torchvision.transforms.Compose) image transformations\n        from_folder: (str) Path to directory holding the images to load.\n\n    Returns:\n        A torchvision dataset\n\n    Raises:\n        ValueError: If the specified dataset doesn't exist\n\n    \"\"\"\n\n    if not os.path.exists(input_dir):\n        raise ValueError(f'The input directory {input_dir} does not exist!')\n\n    return _load_dataset_from_folder(input_dir, transform)\n", "path": "lightly/data/_helpers.py"}, {"content": "\"\"\" Image Dataset \"\"\"\n\n# Copyright (c) 2020. Lightly AG and its affiliates.\n# All Rights Reserved\n\nimport os\nimport torchvision.datasets as datasets\n\nfrom lightly.data._image_loaders import default_loader\n\n\ndef _make_dataset(directory, extensions=None, is_valid_file=None):\n    \"\"\"Returns a list of all image files with targets in the directory.\n\n    Args:\n        directory:\n            Root directory path (should not contain subdirectories!).\n        extensions:\n            Tuple of valid extensions.\n        is_valid_file:\n            Used to find valid files.\n\n    Returns:\n        List of instance tuples: (path_i, target_i = 0).\n\n    \"\"\"\n\n    # handle is_valid_file and extensions the same way torchvision handles them:\n    # https://pytorch.org/docs/stable/_modules/torchvision/datasets/folder.html#ImageFolder\n    both_none = extensions is None and is_valid_file is None\n    both_something = extensions is not None and is_valid_file is not None\n    if both_none or both_something:\n        raise ValueError('Both extensions and is_valid_file cannot be None or '\n                         'not None at the same time')\n\n    if extensions is not None:\n        def _is_valid_file(filename):\n            return filename.lower().endswith(extensions)\n\n    if is_valid_file is not None:\n        _is_valid_file = is_valid_file\n\n    instances = []\n    for fname in os.listdir(directory):\n\n        if not _is_valid_file(fname):\n            continue\n\n        path = os.path.join(directory, fname)\n        item = (path, 0)\n        instances.append(item)\n\n    return instances\n\n\nclass DatasetFolder(datasets.VisionDataset):\n    \"\"\"Implements a dataset folder.\n    \n    DatasetFolder based on torchvisions implementation.\n    (https://pytorch.org/docs/stable/torchvision/datasets.html#datasetfolder)\n\n    Attributes:\n        root:\n            Root directory path\n        loader:\n            Function that loads file at path\n        extensions:\n            Tuple of allowed extensions\n        transform:\n            Function that takes a PIL image and returns transformed version\n        target_transform:\n            As transform but for targets\n        is_valid_file:\n            Used to check corrupt files\n\n    Raises:\n        RuntimeError: If no supported files are found in root.\n\n    \"\"\"\n\n    def __init__(self,\n                 root: str,\n                 loader=default_loader,\n                 extensions=None,\n                 transform=None,\n                 target_transform=None,\n                 is_valid_file=None):\n\n        super(DatasetFolder, self).__init__(root,\n                                            transform=transform,\n                                            target_transform=target_transform)\n\n        samples = _make_dataset(self.root, extensions, is_valid_file)\n        if len(samples) == 0:\n            msg = 'Found 0 files in folder: {}\\n'.format(self.root)\n            if extensions is not None:\n                msg += 'Supported extensions are: {}'.format(\n                    ','.join(extensions))\n            raise RuntimeError(msg)\n\n        self.loader = loader\n        self.extensions = extensions\n\n        self.samples = samples\n        self.targets = [s[1] for s in samples]\n\n    def __getitem__(self, index: int):\n        \"\"\"Returns item at index.\n\n        Args:\n            index:\n                Index of the sample to retrieve.\n\n        Returns:\n            A tuple (sample, target) where target is 0.\n\n        \"\"\"\n\n        path, target = self.samples[index]\n        sample = self.loader(path)\n        if self.transform is not None:\n            sample = self.transform(sample)\n        if self.target_transform is not None:\n            target = self.target_transform(target)\n\n        return sample, target\n\n    def __len__(self):\n        \"\"\"Returns the number of samples in the dataset.\n\n        \"\"\"\n        return len(self.samples)\n", "path": "lightly/data/_image.py"}], "after_files": [{"content": "\"\"\" Helper Functions \"\"\"\n\n# Copyright (c) 2020. Lightly AG and its affiliates.\n# All Rights Reserved\n\nimport os\nfrom torchvision import datasets\n\nfrom lightly.data._image import DatasetFolder\n\ntry:\n    from lightly.data._video import VideoDataset\n    VIDEO_DATASET_AVAILABLE = True\nexcept Exception as e:\n    VIDEO_DATASET_AVAILABLE = False\n    VIDEO_DATASET_ERRORMSG = e\n\n\nIMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp',\n                  '.pgm', '.tif', '.tiff', '.webp')\n\nVIDEO_EXTENSIONS = ('.mp4', '.mov', '.avi', '.mpg',\n                    '.hevc', '.m4v', '.webm', '.mpeg')\n\n\ndef _contains_videos(root: str, extensions: tuple):\n    \"\"\"Checks whether directory contains video files.\n\n    Args:\n        root: Root directory path.\n\n    Returns:\n        True if root contains subdirectories else false.\n    \"\"\"\n    with os.scandir(root) as scan_dir:\n        return any(f.name.lower().endswith(extensions) for f in scan_dir)\n\n\ndef _is_lightly_output_dir(dirname: str):\n    \"\"\"Checks whether the directory is a lightly_output directory.\n\n    Args:\n        dirname: Directory to check.\n\n    Returns:\n        True if dirname is \"lightly_outputs\" else false.\n\n    \"\"\"\n    return 'lightly_outputs' in dirname\n\n\ndef _contains_subdirs(root: str):\n    \"\"\"Checks whether directory contains subdirectories.\n\n    Args:\n        root: Root directory path.\n\n    Returns:\n        True if root contains subdirectories else false.\n\n    \"\"\"\n    with os.scandir(root) as scan_dir:\n        return any(not _is_lightly_output_dir(f.name) for f in scan_dir \\\n            if f.is_dir())\n\n\ndef _load_dataset_from_folder(root: str, transform):\n    \"\"\"Initializes dataset from folder.\n\n    Args:\n        root: (str) Root directory path\n        transform: (torchvision.transforms.Compose) image transformations\n\n    Returns:\n        Dataset consisting of images in the root directory.\n\n    \"\"\"\n\n    # if there is a video in the input directory but we do not have\n    # the right dependencies, raise a ValueError\n    contains_videos = _contains_videos(root, VIDEO_EXTENSIONS)\n    if contains_videos and not VIDEO_DATASET_AVAILABLE:\n        raise ValueError(f'The input directory {root} contains videos '\n                         'but the VideoDataset is not available. \\n'\n                         'Make sure you have installed the right '\n                         'dependencies. The error from the imported '\n                         f'module was: {VIDEO_DATASET_ERRORMSG}')\n\n    if contains_videos:\n        # root contains videos -> create a video dataset\n        dataset = VideoDataset(root,\n                               extensions=VIDEO_EXTENSIONS,\n                               transform=transform)\n    elif _contains_subdirs(root):\n        # root contains subdirectories -> create an image folder dataset\n        dataset = datasets.ImageFolder(root,\n                                       transform=transform)\n    else:\n        # root contains plain images -> create a folder dataset\n        dataset = DatasetFolder(root,\n                                extensions=IMG_EXTENSIONS,\n                                transform=transform)\n\n    return dataset\n\n\ndef _load_dataset(input_dir: str,\n                  transform=None):\n    \"\"\"Initializes dataset from torchvision or from folder.\n\n    Args:\n        root: (str) Directory where dataset is stored\n        name: (str) Name of the dataset (e.g. cifar10, cifar100)\n        train: (bool) Use the training set\n        download: (bool) Download the dataset\n        transform: (torchvision.transforms.Compose) image transformations\n        from_folder: (str) Path to directory holding the images to load.\n\n    Returns:\n        A torchvision dataset\n\n    Raises:\n        ValueError: If the specified dataset doesn't exist\n\n    \"\"\"\n\n    if not os.path.exists(input_dir):\n        raise ValueError(f'The input directory {input_dir} does not exist!')\n\n    return _load_dataset_from_folder(input_dir, transform)\n", "path": "lightly/data/_helpers.py"}, {"content": "\"\"\" Image Dataset \"\"\"\n\n# Copyright (c) 2020. Lightly AG and its affiliates.\n# All Rights Reserved\n\nfrom typing import List, Tuple\n\nimport os\nimport torchvision.datasets as datasets\n\nfrom lightly.data._image_loaders import default_loader\n\n\ndef _make_dataset(directory, extensions=None, is_valid_file=None) -> List[Tuple[str, int]]:\n    \"\"\"Returns a list of all image files with targets in the directory.\n\n    Args:\n        directory:\n            Root directory path (should not contain subdirectories!).\n        extensions:\n            Tuple of valid extensions.\n        is_valid_file:\n            Used to find valid files.\n\n    Returns:\n        List of instance tuples: (path_i, target_i = 0).\n\n    \"\"\"\n\n    # handle is_valid_file and extensions the same way torchvision handles them:\n    # https://pytorch.org/docs/stable/_modules/torchvision/datasets/folder.html#ImageFolder\n    both_none = extensions is None and is_valid_file is None\n    both_something = extensions is not None and is_valid_file is not None\n    if both_none or both_something:\n        raise ValueError('Both extensions and is_valid_file cannot be None or '\n                         'not None at the same time')\n\n    if extensions is not None:\n        def _is_valid_file(filename):\n            return filename.lower().endswith(extensions)\n\n    if is_valid_file is not None:\n        _is_valid_file = is_valid_file\n\n    instances = []\n    for f in os.scandir(directory):\n\n        if not _is_valid_file(f.name):\n            continue\n\n        # convention: the label of all images is 0, based on the fact that\n        # they are all in the same directory\n        item = (f.path, 0)\n        instances.append(item)\n\n    return sorted(instances, key=lambda x: x[0]) # sort by path\n\n\nclass DatasetFolder(datasets.VisionDataset):\n    \"\"\"Implements a dataset folder.\n    \n    DatasetFolder based on torchvisions implementation.\n    (https://pytorch.org/docs/stable/torchvision/datasets.html#datasetfolder)\n\n    Attributes:\n        root:\n            Root directory path\n        loader:\n            Function that loads file at path\n        extensions:\n            Tuple of allowed extensions\n        transform:\n            Function that takes a PIL image and returns transformed version\n        target_transform:\n            As transform but for targets\n        is_valid_file:\n            Used to check corrupt files\n\n    Raises:\n        RuntimeError: If no supported files are found in root.\n\n    \"\"\"\n\n    def __init__(self,\n                 root: str,\n                 loader=default_loader,\n                 extensions=None,\n                 transform=None,\n                 target_transform=None,\n                 is_valid_file=None):\n\n        super(DatasetFolder, self).__init__(root,\n                                            transform=transform,\n                                            target_transform=target_transform)\n\n        samples = _make_dataset(self.root, extensions, is_valid_file)\n        if len(samples) == 0:\n            msg = 'Found 0 files in folder: {}\\n'.format(self.root)\n            if extensions is not None:\n                msg += 'Supported extensions are: {}'.format(\n                    ','.join(extensions))\n            raise RuntimeError(msg)\n\n        self.loader = loader\n        self.extensions = extensions\n\n        self.samples = samples\n        self.targets = [s[1] for s in samples]\n\n    def __getitem__(self, index: int):\n        \"\"\"Returns item at index.\n\n        Args:\n            index:\n                Index of the sample to retrieve.\n\n        Returns:\n            A tuple (sample, target) where target is 0.\n\n        \"\"\"\n\n        path, target = self.samples[index]\n        sample = self.loader(path)\n        if self.transform is not None:\n            sample = self.transform(sample)\n        if self.target_transform is not None:\n            target = self.target_transform(target)\n\n        return sample, target\n\n    def __len__(self):\n        \"\"\"Returns the number of samples in the dataset.\n\n        \"\"\"\n        return len(self.samples)\n", "path": "lightly/data/_image.py"}]}