We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
data.LightlyDataset.from_torch_dataset  does not work properly
Hello,

I am trying to use Lightly with a dataset of images in `.npz` format. I have created a torch dataset class that loads the sample images and use `data.LightlyDataset.from_torch_dataset ` to create a Lightly dataset from it. However, it seems that the transformations that are passed to the `data.LightlyDataset.from_torch_dataset ` function are not properly performed. and subsequently the `Dataloader` does not work as expected. Can you help me figure out how to handle this? thanks

```
dataset_test_pl = data.LightlyDataset.from_torch_dataset(dataset_test, transform=test_transform)
dataloader_test = torch.utils.data.DataLoader(
    dataset_test_pl,
    batch_size=batch_size,
    shuffle=False,
    drop_last=False,
    num_workers=num_workers
)
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `lightly/data/dataset.py`
Content:
```
1 """ Lightly Dataset """
2 
3 # Copyright (c) 2020. Lightly AG and its affiliates.
4 # All Rights Reserved
5 
6 import os
7 import bisect
8 import shutil
9 import tempfile
10 
11 from PIL import Image
12 from typing import List, Union, Callable
13 
14 import torchvision.datasets as datasets
15 from torchvision import transforms
16 
17 from lightly.data._helpers import _load_dataset_from_folder
18 from lightly.data._helpers import DatasetFolder
19 from lightly.data._video import VideoDataset
20 from lightly.utils.io import check_filenames
21 
22 
23 def _get_filename_by_index(dataset, index):
24     """Default function which maps the index of an image to a filename.
25 
26     """
27     if isinstance(dataset, datasets.ImageFolder):
28         # filename is the path of the image relative to the dataset root
29         full_path = dataset.imgs[index][0]
30         return os.path.relpath(full_path, dataset.root)
31     elif isinstance(dataset, DatasetFolder):
32         # filename is the path of the image relative to the dataset root
33         full_path = dataset.samples[index][0]
34         return os.path.relpath(full_path, dataset.root)
35     elif isinstance(dataset, VideoDataset):
36         # filename is constructed by the video dataset
37         return dataset.get_filename(index)
38     else:
39         # dummy to prevent crashes
40         return str(index)
41 
42 
43 def _ensure_dir(path):
44     """Makes sure that the directory at path exists.
45 
46     """
47     dirname = os.path.dirname(path)
48     os.makedirs(dirname, exist_ok=True)
49 
50 
51 def _copy_image(input_dir, output_dir, filename):
52     """Copies an image from the input directory to the output directory.
53 
54     """
55     source = os.path.join(input_dir, filename)
56     target = os.path.join(output_dir, filename)
57     _ensure_dir(target)
58     shutil.copyfile(source, target)
59 
60 
61 def _save_image(image, output_dir, filename, fmt):
62     """Saves an image in the output directory.
63 
64     """
65     target = os.path.join(output_dir, filename)
66     _ensure_dir(target)
67     try:
68         # try to save the image with the specified format or
69         # derive the format from the filename (if format=None)
70         image.save(target, format=fmt)
71     except ValueError:
72         # could not determine format from filename
73         image.save(target, format='png')
74 
75 
76 def _dump_image(dataset, output_dir, filename, index, fmt):
77     """Saves a single image to the output directory.
78 
79     Will copy the image from the input directory to the output directory
80     if possible. If not (e.g. for VideoDatasets), will load the image and
81     then save it to the output directory with the specified format.
82 
83     """
84 
85     if isinstance(dataset, datasets.ImageFolder):
86         # can safely copy the image from the input to the output directory
87         _copy_image(dataset.root, output_dir, filename)
88     elif isinstance(dataset, DatasetFolder):
89         # can safely copy the image from the input to the output directory
90         _copy_image(dataset.root, output_dir, filename)
91     else:
92         # need to load the image and save it to the output directory
93         image, _ = dataset[index]
94         _save_image(image, output_dir, filename, fmt)
95 
96 
97 class LightlyDataset:
98     """Provides a uniform data interface for the embedding models.
99 
100     Should be used for all models and functions in the lightly package.
101     Returns a tuple (sample, target, fname) when accessed using __getitem__.
102 
103     The LightlyDataset supports different input sources. You can use it
104     on a folder of images. You can also use it on a folder with subfolders
105     with images (ImageNet style). If the input_dir has subfolders,
106     each subfolder gets its own target label.
107     You can also work with videos (requires pyav).
108     If there are multiple videos in the input_dir each video gets a different
109     target label assigned. If input_dir contains images and videos
110     only the videos are used.
111 
112     Can also be used in combination with the `from_torch_dataset` method
113     to load a dataset offered by torchvision (e.g. cifar10).
114 
115     Args:
116         input_dir:
117             Path to directory holding the images or videos to load.
118         transform:
119             Image transforms (as in torchvision).
120         index_to_filename:
121             Function which takes the dataset and index as input and returns
122             the filename of the file at the index. If None, uses default.
123         filenames:
124             If not None, it filters the dataset in the input directory
125             by the given filenames.
126 
127     Examples:
128         >>> # load a dataset consisting of images from a local folder
129         >>> # mydata/
130         >>> # `- img1.png
131         >>> # `- img2.png
132         >>> # `- ...
133         >>> import lightly.data as data
134         >>> dataset = data.LightlyDataset(input_dir='path/to/mydata/')
135         >>> sample, target, fname = dataset[0]
136         >>>
137         >>> # also works with subfolders
138         >>> # mydata/
139         >>> # `- subfolder1
140         >>> #     `- img1.png
141         >>> # `- subfolder2
142         >>> # ...
143         >>>
144         >>> # also works with videos
145         >>> # mydata/
146         >>> # `- video1.mp4
147         >>> # `- video2.mp4
148         >>> # `- ...
149     """
150 
151     def __init__(self,
152                  input_dir: Union[str, None],
153                  transform: transforms.Compose = None,
154                  index_to_filename:
155                  Callable[[datasets.VisionDataset, int], str] = None,
156                  filenames: List[str] = None,
157                  ):
158 
159         # can pass input_dir=None to create an "empty" dataset
160         self.input_dir = input_dir
161         if filenames is not None:
162             filepaths = [
163                 os.path.join(input_dir, filename)
164                 for filename in filenames
165             ]
166             filepaths = set(filepaths)
167 
168             def is_valid_file(filepath: str):
169                 return filepath in filepaths
170         else:
171             is_valid_file = None
172         if self.input_dir is not None:
173             self.dataset = _load_dataset_from_folder(
174                 self.input_dir, transform, is_valid_file=is_valid_file
175             )
176 
177         # initialize function to get filename of image
178         self.index_to_filename = _get_filename_by_index
179         if index_to_filename is not None:
180             self.index_to_filename = index_to_filename
181 
182         # if created from an input directory with filenames, check if they
183         # are valid
184         if input_dir:
185             check_filenames(self.get_filenames())
186 
187     @classmethod
188     def from_torch_dataset(cls,
189                            dataset,
190                            transform=None,
191                            index_to_filename=None):
192         """Builds a LightlyDataset from a PyTorch (or torchvision) dataset.
193 
194         Args:
195             dataset:
196                 PyTorch/torchvision dataset.
197             transform:
198                 Image transforms (as in torchvision).
199             index_to_filename:
200                 Function which takes the dataset and index as input and returns
201                 the filename of the file at the index. If None, uses default.
202 
203         Returns:
204             A LightlyDataset object.
205 
206         Examples:
207             >>> # load cifar10 from torchvision
208             >>> import torchvision
209             >>> import lightly.data as data
210             >>> base = torchvision.datasets.CIFAR10(root='./')
211             >>> dataset = data.LightlyDataset.from_torch_dataset(base)
212 
213         """
214         # create an "empty" dataset object
215         dataset_obj = cls(
216             None,
217             transform=transform,
218             index_to_filename=index_to_filename
219         )
220 
221         # populate it with the torch dataset
222         dataset_obj.dataset = dataset
223         return dataset_obj
224 
225     def __getitem__(self, index: int):
226         """Returns (sample, target, fname) of item at index.
227 
228         Args:
229             index:
230                 Index of the queried item.
231 
232         Returns:
233             The image, target, and filename of the item at index.
234 
235         """
236         fname = self.index_to_filename(self.dataset, index)
237         sample, target = self.dataset.__getitem__(index)
238 
239         return sample, target, fname
240 
241     def __len__(self):
242         """Returns the length of the dataset.
243 
244         """
245         return len(self.dataset)
246 
247     def __add__(self, other):
248         """Adds another item to the dataset.
249 
250         """
251         raise NotImplementedError()
252 
253     def get_filenames(self) -> List[str]:
254         """Returns all filenames in the dataset.
255 
256         """
257         list_of_filenames = []
258         for index in range(len(self)):
259             fname = self.index_to_filename(self.dataset, index)
260             list_of_filenames.append(fname)
261         return list_of_filenames
262 
263     def dump(self,
264              output_dir: str,
265              filenames: Union[List[str], None] = None,
266              format: Union[str, None] = None):
267         """Saves images in the dataset to the output directory.
268 
269         Will copy the images from the input directory to the output directory
270         if possible. If not (e.g. for VideoDatasets), will load the images and
271         then save them to the output directory with the specified format.
272 
273         Args:
274             output_dir:
275                 Output directory where the image is stored.
276             filenames:
277                 Filenames of the images to store. If None, stores all images.
278             format:
279                 Image format. Can be any pillow image format (png, jpg, ...).
280                 By default we try to use the same format as the input data. If
281                 not possible (e.g. for videos) we dump the image 
282                 as a png image to prevent compression artifacts.
283 
284         """
285 
286         if self.dataset.transform is not None:
287             raise RuntimeError('Cannot dump dataset which applies transforms!')
288 
289         # create directory if it doesn't exist yet
290         os.makedirs(output_dir, exist_ok=True)
291 
292         # dump all the files if no filenames were passed, otherwise dump only
293         # the ones referenced in the list
294         if filenames is None:
295             indices = [i for i in range(self.__len__())]
296             filenames = self.get_filenames()
297         else:
298             indices = []
299             filenames = sorted(filenames)
300             all_filenames = self.get_filenames()
301             for index, filename in enumerate(all_filenames):
302                 filename_index = bisect.bisect_left(filenames, filename)
303                 # make sure the filename exists in filenames
304                 if filename_index < len(filenames) and \
305                         filenames[filename_index] == filename:
306                     indices.append(index)
307 
308         # dump images
309         for i, filename in zip(indices, filenames):
310             _dump_image(self.dataset, output_dir, filename, i, fmt=format)
311 
312     def get_filepath_from_filename(self, filename: str, image: Image = None):
313         """Returns the filepath given the filename of the image
314 
315         There are three cases:
316         - The dataset is a regular dataset with the images in the input dir.
317         - The dataset is a video dataset, thus the images have to be saved in a
318         temporary folder.
319         - The dataset is a torch dataset, thus the images have to be saved in a
320         temporary folder.
321         Args:
322             filename:
323                 The filename of the image
324             image:
325                 The image corresponding to the filename
326 
327         Returns:
328             The filename to the image, either the existing one (case 1) or a
329             newly created jpg (case 2, 3)
330 
331         """
332 
333         has_input_dir = hasattr(self, 'input_dir') and \
334             isinstance(self.input_dir, str)
335         if has_input_dir:
336             path_to_image = os.path.join(self.input_dir, filename)
337             if os.path.isfile(path_to_image):
338                 # the file exists, return its filepath
339                 return path_to_image
340 
341         if image is None:
342             raise ValueError(
343                 'The parameter image must not be None for'
344                 'VideoDatasets and TorchDatasets'
345             )
346 
347         # the file doesn't exist, save it as a jpg and return filepath
348         folder_path = tempfile.mkdtemp()
349         filepath = os.path.join(folder_path, filename) + '.jpg'
350         
351         if os.path.dirname(filepath):
352             os.makedirs(os.path.dirname(filepath), exist_ok=True)
353 
354         image.save(filepath)
355         return filepath
356 
357     @property
358     def transform(self):
359         """Getter for the transform of the dataset.
360 
361         """
362         return self.dataset.transform
363 
364     @transform.setter
365     def transform(self, t):
366         """Setter for the transform of the dataset.
367 
368         """
369         self.dataset.transform = t
370
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/lightly/data/dataset.py b/lightly/data/dataset.py
--- a/lightly/data/dataset.py
+++ b/lightly/data/dataset.py
@@ -10,6 +10,7 @@
 
 from PIL import Image
 from typing import List, Union, Callable
+from torch._C import Value
 
 import torchvision.datasets as datasets
 from torchvision import transforms
@@ -153,8 +154,7 @@
                  transform: transforms.Compose = None,
                  index_to_filename:
                  Callable[[datasets.VisionDataset, int], str] = None,
-                 filenames: List[str] = None,
-                 ):
+                 filenames: List[str] = None):
 
         # can pass input_dir=None to create an "empty" dataset
         self.input_dir = input_dir
@@ -169,10 +169,16 @@
                 return filepath in filepaths
         else:
             is_valid_file = None
+
         if self.input_dir is not None:
             self.dataset = _load_dataset_from_folder(
                 self.input_dir, transform, is_valid_file=is_valid_file
             )
+        elif transform is not None:
+            raise ValueError(
+                'transform must be None when input_dir is None but is '
+                f'{transform}',
+            )
 
         # initialize function to get filename of image
         self.index_to_filename = _get_filename_by_index
@@ -214,12 +220,12 @@
         # create an "empty" dataset object
         dataset_obj = cls(
             None,
-            transform=transform,
-            index_to_filename=index_to_filename
+            index_to_filename=index_to_filename,
         )
 
         # populate it with the torch dataset
         dataset_obj.dataset = dataset
+        dataset_obj.transform = transform
         return dataset_obj
 
     def __getitem__(self, index: int):

{"golden_diff": "diff --git a/lightly/data/dataset.py b/lightly/data/dataset.py\n--- a/lightly/data/dataset.py\n+++ b/lightly/data/dataset.py\n@@ -10,6 +10,7 @@\n \n from PIL import Image\n from typing import List, Union, Callable\n+from torch._C import Value\n \n import torchvision.datasets as datasets\n from torchvision import transforms\n@@ -153,8 +154,7 @@\n                  transform: transforms.Compose = None,\n                  index_to_filename:\n                  Callable[[datasets.VisionDataset, int], str] = None,\n-                 filenames: List[str] = None,\n-                 ):\n+                 filenames: List[str] = None):\n \n         # can pass input_dir=None to create an \"empty\" dataset\n         self.input_dir = input_dir\n@@ -169,10 +169,16 @@\n                 return filepath in filepaths\n         else:\n             is_valid_file = None\n+\n         if self.input_dir is not None:\n             self.dataset = _load_dataset_from_folder(\n                 self.input_dir, transform, is_valid_file=is_valid_file\n             )\n+        elif transform is not None:\n+            raise ValueError(\n+                'transform must be None when input_dir is None but is '\n+                f'{transform}',\n+            )\n \n         # initialize function to get filename of image\n         self.index_to_filename = _get_filename_by_index\n@@ -214,12 +220,12 @@\n         # create an \"empty\" dataset object\n         dataset_obj = cls(\n             None,\n-            transform=transform,\n-            index_to_filename=index_to_filename\n+            index_to_filename=index_to_filename,\n         )\n \n         # populate it with the torch dataset\n         dataset_obj.dataset = dataset\n+        dataset_obj.transform = transform\n         return dataset_obj\n \n     def __getitem__(self, index: int):\n", "issue": "data.LightlyDataset.from_torch_dataset  does not work properly\nHello,\r\n\r\nI am trying to use Lightly with a dataset of images in `.npz` format. I have created a torch dataset class that loads the sample images and use `data.LightlyDataset.from_torch_dataset ` to create a Lightly dataset from it. However, it seems that the transformations that are passed to the `data.LightlyDataset.from_torch_dataset ` function are not properly performed. and subsequently the `Dataloader` does not work as expected. Can you help me figure out how to handle this? thanks\r\n\r\n```\r\ndataset_test_pl = data.LightlyDataset.from_torch_dataset(dataset_test, transform=test_transform)\r\ndataloader_test = torch.utils.data.DataLoader(\r\n    dataset_test_pl,\r\n    batch_size=batch_size,\r\n    shuffle=False,\r\n    drop_last=False,\r\n    num_workers=num_workers\r\n)\r\n```\n", "before_files": [{"content": "\"\"\" Lightly Dataset \"\"\"\n\n# Copyright (c) 2020. Lightly AG and its affiliates.\n# All Rights Reserved\n\nimport os\nimport bisect\nimport shutil\nimport tempfile\n\nfrom PIL import Image\nfrom typing import List, Union, Callable\n\nimport torchvision.datasets as datasets\nfrom torchvision import transforms\n\nfrom lightly.data._helpers import _load_dataset_from_folder\nfrom lightly.data._helpers import DatasetFolder\nfrom lightly.data._video import VideoDataset\nfrom lightly.utils.io import check_filenames\n\n\ndef _get_filename_by_index(dataset, index):\n    \"\"\"Default function which maps the index of an image to a filename.\n\n    \"\"\"\n    if isinstance(dataset, datasets.ImageFolder):\n        # filename is the path of the image relative to the dataset root\n        full_path = dataset.imgs[index][0]\n        return os.path.relpath(full_path, dataset.root)\n    elif isinstance(dataset, DatasetFolder):\n        # filename is the path of the image relative to the dataset root\n        full_path = dataset.samples[index][0]\n        return os.path.relpath(full_path, dataset.root)\n    elif isinstance(dataset, VideoDataset):\n        # filename is constructed by the video dataset\n        return dataset.get_filename(index)\n    else:\n        # dummy to prevent crashes\n        return str(index)\n\n\ndef _ensure_dir(path):\n    \"\"\"Makes sure that the directory at path exists.\n\n    \"\"\"\n    dirname = os.path.dirname(path)\n    os.makedirs(dirname, exist_ok=True)\n\n\ndef _copy_image(input_dir, output_dir, filename):\n    \"\"\"Copies an image from the input directory to the output directory.\n\n    \"\"\"\n    source = os.path.join(input_dir, filename)\n    target = os.path.join(output_dir, filename)\n    _ensure_dir(target)\n    shutil.copyfile(source, target)\n\n\ndef _save_image(image, output_dir, filename, fmt):\n    \"\"\"Saves an image in the output directory.\n\n    \"\"\"\n    target = os.path.join(output_dir, filename)\n    _ensure_dir(target)\n    try:\n        # try to save the image with the specified format or\n        # derive the format from the filename (if format=None)\n        image.save(target, format=fmt)\n    except ValueError:\n        # could not determine format from filename\n        image.save(target, format='png')\n\n\ndef _dump_image(dataset, output_dir, filename, index, fmt):\n    \"\"\"Saves a single image to the output directory.\n\n    Will copy the image from the input directory to the output directory\n    if possible. If not (e.g. for VideoDatasets), will load the image and\n    then save it to the output directory with the specified format.\n\n    \"\"\"\n\n    if isinstance(dataset, datasets.ImageFolder):\n        # can safely copy the image from the input to the output directory\n        _copy_image(dataset.root, output_dir, filename)\n    elif isinstance(dataset, DatasetFolder):\n        # can safely copy the image from the input to the output directory\n        _copy_image(dataset.root, output_dir, filename)\n    else:\n        # need to load the image and save it to the output directory\n        image, _ = dataset[index]\n        _save_image(image, output_dir, filename, fmt)\n\n\nclass LightlyDataset:\n    \"\"\"Provides a uniform data interface for the embedding models.\n\n    Should be used for all models and functions in the lightly package.\n    Returns a tuple (sample, target, fname) when accessed using __getitem__.\n\n    The LightlyDataset supports different input sources. You can use it\n    on a folder of images. You can also use it on a folder with subfolders\n    with images (ImageNet style). If the input_dir has subfolders,\n    each subfolder gets its own target label.\n    You can also work with videos (requires pyav).\n    If there are multiple videos in the input_dir each video gets a different\n    target label assigned. If input_dir contains images and videos\n    only the videos are used.\n\n    Can also be used in combination with the `from_torch_dataset` method\n    to load a dataset offered by torchvision (e.g. cifar10).\n\n    Args:\n        input_dir:\n            Path to directory holding the images or videos to load.\n        transform:\n            Image transforms (as in torchvision).\n        index_to_filename:\n            Function which takes the dataset and index as input and returns\n            the filename of the file at the index. If None, uses default.\n        filenames:\n            If not None, it filters the dataset in the input directory\n            by the given filenames.\n\n    Examples:\n        >>> # load a dataset consisting of images from a local folder\n        >>> # mydata/\n        >>> # `- img1.png\n        >>> # `- img2.png\n        >>> # `- ...\n        >>> import lightly.data as data\n        >>> dataset = data.LightlyDataset(input_dir='path/to/mydata/')\n        >>> sample, target, fname = dataset[0]\n        >>>\n        >>> # also works with subfolders\n        >>> # mydata/\n        >>> # `- subfolder1\n        >>> #     `- img1.png\n        >>> # `- subfolder2\n        >>> # ...\n        >>>\n        >>> # also works with videos\n        >>> # mydata/\n        >>> # `- video1.mp4\n        >>> # `- video2.mp4\n        >>> # `- ...\n    \"\"\"\n\n    def __init__(self,\n                 input_dir: Union[str, None],\n                 transform: transforms.Compose = None,\n                 index_to_filename:\n                 Callable[[datasets.VisionDataset, int], str] = None,\n                 filenames: List[str] = None,\n                 ):\n\n        # can pass input_dir=None to create an \"empty\" dataset\n        self.input_dir = input_dir\n        if filenames is not None:\n            filepaths = [\n                os.path.join(input_dir, filename)\n                for filename in filenames\n            ]\n            filepaths = set(filepaths)\n\n            def is_valid_file(filepath: str):\n                return filepath in filepaths\n        else:\n            is_valid_file = None\n        if self.input_dir is not None:\n            self.dataset = _load_dataset_from_folder(\n                self.input_dir, transform, is_valid_file=is_valid_file\n            )\n\n        # initialize function to get filename of image\n        self.index_to_filename = _get_filename_by_index\n        if index_to_filename is not None:\n            self.index_to_filename = index_to_filename\n\n        # if created from an input directory with filenames, check if they\n        # are valid\n        if input_dir:\n            check_filenames(self.get_filenames())\n\n    @classmethod\n    def from_torch_dataset(cls,\n                           dataset,\n                           transform=None,\n                           index_to_filename=None):\n        \"\"\"Builds a LightlyDataset from a PyTorch (or torchvision) dataset.\n\n        Args:\n            dataset:\n                PyTorch/torchvision dataset.\n            transform:\n                Image transforms (as in torchvision).\n            index_to_filename:\n                Function which takes the dataset and index as input and returns\n                the filename of the file at the index. If None, uses default.\n\n        Returns:\n            A LightlyDataset object.\n\n        Examples:\n            >>> # load cifar10 from torchvision\n            >>> import torchvision\n            >>> import lightly.data as data\n            >>> base = torchvision.datasets.CIFAR10(root='./')\n            >>> dataset = data.LightlyDataset.from_torch_dataset(base)\n\n        \"\"\"\n        # create an \"empty\" dataset object\n        dataset_obj = cls(\n            None,\n            transform=transform,\n            index_to_filename=index_to_filename\n        )\n\n        # populate it with the torch dataset\n        dataset_obj.dataset = dataset\n        return dataset_obj\n\n    def __getitem__(self, index: int):\n        \"\"\"Returns (sample, target, fname) of item at index.\n\n        Args:\n            index:\n                Index of the queried item.\n\n        Returns:\n            The image, target, and filename of the item at index.\n\n        \"\"\"\n        fname = self.index_to_filename(self.dataset, index)\n        sample, target = self.dataset.__getitem__(index)\n\n        return sample, target, fname\n\n    def __len__(self):\n        \"\"\"Returns the length of the dataset.\n\n        \"\"\"\n        return len(self.dataset)\n\n    def __add__(self, other):\n        \"\"\"Adds another item to the dataset.\n\n        \"\"\"\n        raise NotImplementedError()\n\n    def get_filenames(self) -> List[str]:\n        \"\"\"Returns all filenames in the dataset.\n\n        \"\"\"\n        list_of_filenames = []\n        for index in range(len(self)):\n            fname = self.index_to_filename(self.dataset, index)\n            list_of_filenames.append(fname)\n        return list_of_filenames\n\n    def dump(self,\n             output_dir: str,\n             filenames: Union[List[str], None] = None,\n             format: Union[str, None] = None):\n        \"\"\"Saves images in the dataset to the output directory.\n\n        Will copy the images from the input directory to the output directory\n        if possible. If not (e.g. for VideoDatasets), will load the images and\n        then save them to the output directory with the specified format.\n\n        Args:\n            output_dir:\n                Output directory where the image is stored.\n            filenames:\n                Filenames of the images to store. If None, stores all images.\n            format:\n                Image format. Can be any pillow image format (png, jpg, ...).\n                By default we try to use the same format as the input data. If\n                not possible (e.g. for videos) we dump the image \n                as a png image to prevent compression artifacts.\n\n        \"\"\"\n\n        if self.dataset.transform is not None:\n            raise RuntimeError('Cannot dump dataset which applies transforms!')\n\n        # create directory if it doesn't exist yet\n        os.makedirs(output_dir, exist_ok=True)\n\n        # dump all the files if no filenames were passed, otherwise dump only\n        # the ones referenced in the list\n        if filenames is None:\n            indices = [i for i in range(self.__len__())]\n            filenames = self.get_filenames()\n        else:\n            indices = []\n            filenames = sorted(filenames)\n            all_filenames = self.get_filenames()\n            for index, filename in enumerate(all_filenames):\n                filename_index = bisect.bisect_left(filenames, filename)\n                # make sure the filename exists in filenames\n                if filename_index < len(filenames) and \\\n                        filenames[filename_index] == filename:\n                    indices.append(index)\n\n        # dump images\n        for i, filename in zip(indices, filenames):\n            _dump_image(self.dataset, output_dir, filename, i, fmt=format)\n\n    def get_filepath_from_filename(self, filename: str, image: Image = None):\n        \"\"\"Returns the filepath given the filename of the image\n\n        There are three cases:\n        - The dataset is a regular dataset with the images in the input dir.\n        - The dataset is a video dataset, thus the images have to be saved in a\n        temporary folder.\n        - The dataset is a torch dataset, thus the images have to be saved in a\n        temporary folder.\n        Args:\n            filename:\n                The filename of the image\n            image:\n                The image corresponding to the filename\n\n        Returns:\n            The filename to the image, either the existing one (case 1) or a\n            newly created jpg (case 2, 3)\n\n        \"\"\"\n\n        has_input_dir = hasattr(self, 'input_dir') and \\\n            isinstance(self.input_dir, str)\n        if has_input_dir:\n            path_to_image = os.path.join(self.input_dir, filename)\n            if os.path.isfile(path_to_image):\n                # the file exists, return its filepath\n                return path_to_image\n\n        if image is None:\n            raise ValueError(\n                'The parameter image must not be None for'\n                'VideoDatasets and TorchDatasets'\n            )\n\n        # the file doesn't exist, save it as a jpg and return filepath\n        folder_path = tempfile.mkdtemp()\n        filepath = os.path.join(folder_path, filename) + '.jpg'\n        \n        if os.path.dirname(filepath):\n            os.makedirs(os.path.dirname(filepath), exist_ok=True)\n\n        image.save(filepath)\n        return filepath\n\n    @property\n    def transform(self):\n        \"\"\"Getter for the transform of the dataset.\n\n        \"\"\"\n        return self.dataset.transform\n\n    @transform.setter\n    def transform(self, t):\n        \"\"\"Setter for the transform of the dataset.\n\n        \"\"\"\n        self.dataset.transform = t\n", "path": "lightly/data/dataset.py"}], "after_files": [{"content": "\"\"\" Lightly Dataset \"\"\"\n\n# Copyright (c) 2020. Lightly AG and its affiliates.\n# All Rights Reserved\n\nimport os\nimport bisect\nimport shutil\nimport tempfile\n\nfrom PIL import Image\nfrom typing import List, Union, Callable\nfrom torch._C import Value\n\nimport torchvision.datasets as datasets\nfrom torchvision import transforms\n\nfrom lightly.data._helpers import _load_dataset_from_folder\nfrom lightly.data._helpers import DatasetFolder\nfrom lightly.data._video import VideoDataset\nfrom lightly.utils.io import check_filenames\n\n\ndef _get_filename_by_index(dataset, index):\n    \"\"\"Default function which maps the index of an image to a filename.\n\n    \"\"\"\n    if isinstance(dataset, datasets.ImageFolder):\n        # filename is the path of the image relative to the dataset root\n        full_path = dataset.imgs[index][0]\n        return os.path.relpath(full_path, dataset.root)\n    elif isinstance(dataset, DatasetFolder):\n        # filename is the path of the image relative to the dataset root\n        full_path = dataset.samples[index][0]\n        return os.path.relpath(full_path, dataset.root)\n    elif isinstance(dataset, VideoDataset):\n        # filename is constructed by the video dataset\n        return dataset.get_filename(index)\n    else:\n        # dummy to prevent crashes\n        return str(index)\n\n\ndef _ensure_dir(path):\n    \"\"\"Makes sure that the directory at path exists.\n\n    \"\"\"\n    dirname = os.path.dirname(path)\n    os.makedirs(dirname, exist_ok=True)\n\n\ndef _copy_image(input_dir, output_dir, filename):\n    \"\"\"Copies an image from the input directory to the output directory.\n\n    \"\"\"\n    source = os.path.join(input_dir, filename)\n    target = os.path.join(output_dir, filename)\n    _ensure_dir(target)\n    shutil.copyfile(source, target)\n\n\ndef _save_image(image, output_dir, filename, fmt):\n    \"\"\"Saves an image in the output directory.\n\n    \"\"\"\n    target = os.path.join(output_dir, filename)\n    _ensure_dir(target)\n    try:\n        # try to save the image with the specified format or\n        # derive the format from the filename (if format=None)\n        image.save(target, format=fmt)\n    except ValueError:\n        # could not determine format from filename\n        image.save(target, format='png')\n\n\ndef _dump_image(dataset, output_dir, filename, index, fmt):\n    \"\"\"Saves a single image to the output directory.\n\n    Will copy the image from the input directory to the output directory\n    if possible. If not (e.g. for VideoDatasets), will load the image and\n    then save it to the output directory with the specified format.\n\n    \"\"\"\n\n    if isinstance(dataset, datasets.ImageFolder):\n        # can safely copy the image from the input to the output directory\n        _copy_image(dataset.root, output_dir, filename)\n    elif isinstance(dataset, DatasetFolder):\n        # can safely copy the image from the input to the output directory\n        _copy_image(dataset.root, output_dir, filename)\n    else:\n        # need to load the image and save it to the output directory\n        image, _ = dataset[index]\n        _save_image(image, output_dir, filename, fmt)\n\n\nclass LightlyDataset:\n    \"\"\"Provides a uniform data interface for the embedding models.\n\n    Should be used for all models and functions in the lightly package.\n    Returns a tuple (sample, target, fname) when accessed using __getitem__.\n\n    The LightlyDataset supports different input sources. You can use it\n    on a folder of images. You can also use it on a folder with subfolders\n    with images (ImageNet style). If the input_dir has subfolders,\n    each subfolder gets its own target label.\n    You can also work with videos (requires pyav).\n    If there are multiple videos in the input_dir each video gets a different\n    target label assigned. If input_dir contains images and videos\n    only the videos are used.\n\n    Can also be used in combination with the `from_torch_dataset` method\n    to load a dataset offered by torchvision (e.g. cifar10).\n\n    Args:\n        input_dir:\n            Path to directory holding the images or videos to load.\n        transform:\n            Image transforms (as in torchvision).\n        index_to_filename:\n            Function which takes the dataset and index as input and returns\n            the filename of the file at the index. If None, uses default.\n        filenames:\n            If not None, it filters the dataset in the input directory\n            by the given filenames.\n\n    Examples:\n        >>> # load a dataset consisting of images from a local folder\n        >>> # mydata/\n        >>> # `- img1.png\n        >>> # `- img2.png\n        >>> # `- ...\n        >>> import lightly.data as data\n        >>> dataset = data.LightlyDataset(input_dir='path/to/mydata/')\n        >>> sample, target, fname = dataset[0]\n        >>>\n        >>> # also works with subfolders\n        >>> # mydata/\n        >>> # `- subfolder1\n        >>> #     `- img1.png\n        >>> # `- subfolder2\n        >>> # ...\n        >>>\n        >>> # also works with videos\n        >>> # mydata/\n        >>> # `- video1.mp4\n        >>> # `- video2.mp4\n        >>> # `- ...\n    \"\"\"\n\n    def __init__(self,\n                 input_dir: Union[str, None],\n                 transform: transforms.Compose = None,\n                 index_to_filename:\n                 Callable[[datasets.VisionDataset, int], str] = None,\n                 filenames: List[str] = None):\n\n        # can pass input_dir=None to create an \"empty\" dataset\n        self.input_dir = input_dir\n        if filenames is not None:\n            filepaths = [\n                os.path.join(input_dir, filename)\n                for filename in filenames\n            ]\n            filepaths = set(filepaths)\n\n            def is_valid_file(filepath: str):\n                return filepath in filepaths\n        else:\n            is_valid_file = None\n\n        if self.input_dir is not None:\n            self.dataset = _load_dataset_from_folder(\n                self.input_dir, transform, is_valid_file=is_valid_file\n            )\n        elif transform is not None:\n            raise ValueError(\n                'transform must be None when input_dir is None but is '\n                f'{transform}',\n            )\n\n        # initialize function to get filename of image\n        self.index_to_filename = _get_filename_by_index\n        if index_to_filename is not None:\n            self.index_to_filename = index_to_filename\n\n        # if created from an input directory with filenames, check if they\n        # are valid\n        if input_dir:\n            check_filenames(self.get_filenames())\n\n    @classmethod\n    def from_torch_dataset(cls,\n                           dataset,\n                           transform=None,\n                           index_to_filename=None):\n        \"\"\"Builds a LightlyDataset from a PyTorch (or torchvision) dataset.\n\n        Args:\n            dataset:\n                PyTorch/torchvision dataset.\n            transform:\n                Image transforms (as in torchvision).\n            index_to_filename:\n                Function which takes the dataset and index as input and returns\n                the filename of the file at the index. If None, uses default.\n\n        Returns:\n            A LightlyDataset object.\n\n        Examples:\n            >>> # load cifar10 from torchvision\n            >>> import torchvision\n            >>> import lightly.data as data\n            >>> base = torchvision.datasets.CIFAR10(root='./')\n            >>> dataset = data.LightlyDataset.from_torch_dataset(base)\n\n        \"\"\"\n        # create an \"empty\" dataset object\n        dataset_obj = cls(\n            None,\n            index_to_filename=index_to_filename,\n        )\n\n        # populate it with the torch dataset\n        dataset_obj.dataset = dataset\n        dataset_obj.transform = transform\n        return dataset_obj\n\n    def __getitem__(self, index: int):\n        \"\"\"Returns (sample, target, fname) of item at index.\n\n        Args:\n            index:\n                Index of the queried item.\n\n        Returns:\n            The image, target, and filename of the item at index.\n\n        \"\"\"\n        fname = self.index_to_filename(self.dataset, index)\n        sample, target = self.dataset.__getitem__(index)\n\n        return sample, target, fname\n\n    def __len__(self):\n        \"\"\"Returns the length of the dataset.\n\n        \"\"\"\n        return len(self.dataset)\n\n    def __add__(self, other):\n        \"\"\"Adds another item to the dataset.\n\n        \"\"\"\n        raise NotImplementedError()\n\n    def get_filenames(self) -> List[str]:\n        \"\"\"Returns all filenames in the dataset.\n\n        \"\"\"\n        list_of_filenames = []\n        for index in range(len(self)):\n            fname = self.index_to_filename(self.dataset, index)\n            list_of_filenames.append(fname)\n        return list_of_filenames\n\n    def dump(self,\n             output_dir: str,\n             filenames: Union[List[str], None] = None,\n             format: Union[str, None] = None):\n        \"\"\"Saves images in the dataset to the output directory.\n\n        Will copy the images from the input directory to the output directory\n        if possible. If not (e.g. for VideoDatasets), will load the images and\n        then save them to the output directory with the specified format.\n\n        Args:\n            output_dir:\n                Output directory where the image is stored.\n            filenames:\n                Filenames of the images to store. If None, stores all images.\n            format:\n                Image format. Can be any pillow image format (png, jpg, ...).\n                By default we try to use the same format as the input data. If\n                not possible (e.g. for videos) we dump the image \n                as a png image to prevent compression artifacts.\n\n        \"\"\"\n\n        if self.dataset.transform is not None:\n            raise RuntimeError('Cannot dump dataset which applies transforms!')\n\n        # create directory if it doesn't exist yet\n        os.makedirs(output_dir, exist_ok=True)\n\n        # dump all the files if no filenames were passed, otherwise dump only\n        # the ones referenced in the list\n        if filenames is None:\n            indices = [i for i in range(self.__len__())]\n            filenames = self.get_filenames()\n        else:\n            indices = []\n            filenames = sorted(filenames)\n            all_filenames = self.get_filenames()\n            for index, filename in enumerate(all_filenames):\n                filename_index = bisect.bisect_left(filenames, filename)\n                # make sure the filename exists in filenames\n                if filename_index < len(filenames) and \\\n                        filenames[filename_index] == filename:\n                    indices.append(index)\n\n        # dump images\n        for i, filename in zip(indices, filenames):\n            _dump_image(self.dataset, output_dir, filename, i, fmt=format)\n\n    def get_filepath_from_filename(self, filename: str, image: Image = None):\n        \"\"\"Returns the filepath given the filename of the image\n\n        There are three cases:\n        - The dataset is a regular dataset with the images in the input dir.\n        - The dataset is a video dataset, thus the images have to be saved in a\n        temporary folder.\n        - The dataset is a torch dataset, thus the images have to be saved in a\n        temporary folder.\n        Args:\n            filename:\n                The filename of the image\n            image:\n                The image corresponding to the filename\n\n        Returns:\n            The filename to the image, either the existing one (case 1) or a\n            newly created jpg (case 2, 3)\n\n        \"\"\"\n\n        has_input_dir = hasattr(self, 'input_dir') and \\\n            isinstance(self.input_dir, str)\n        if has_input_dir:\n            path_to_image = os.path.join(self.input_dir, filename)\n            if os.path.isfile(path_to_image):\n                # the file exists, return its filepath\n                return path_to_image\n\n        if image is None:\n            raise ValueError(\n                'The parameter image must not be None for'\n                'VideoDatasets and TorchDatasets'\n            )\n\n        # the file doesn't exist, save it as a jpg and return filepath\n        folder_path = tempfile.mkdtemp()\n        filepath = os.path.join(folder_path, filename) + '.jpg'\n        \n        if os.path.dirname(filepath):\n            os.makedirs(os.path.dirname(filepath), exist_ok=True)\n\n        image.save(filepath)\n        return filepath\n\n    @property\n    def transform(self):\n        \"\"\"Getter for the transform of the dataset.\n\n        \"\"\"\n        return self.dataset.transform\n\n    @transform.setter\n    def transform(self, t):\n        \"\"\"Setter for the transform of the dataset.\n\n        \"\"\"\n        self.dataset.transform = t\n", "path": "lightly/data/dataset.py"}]}