We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
loader.num_workers=12 doesn't actually spawn multiple workers
I've noticed that when trying to create an embedding with the param `loader.num_workers` there seems to be no difference in CPU consumption;
`lightly-embed input_dir='./75000' trainer.max_epochs=1 loader.num_workers=12`

This could be that my images are ~8kb and there is some other bottleneck preventing the CPUs to max-out? in My ActivityMonitor (macOSX) I see that I have 80% idle CPU

System: MacBook Pro (16-inch, 2019), 2.6 GHz 6-Core Intel Core i7
OS: MacOSX 11.2.3




--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `lightly/cli/upload_cli.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 """**Lightly Upload:** Upload images to the Lightly platform.
3 
4 This module contains the entrypoint for the **lightly-upload**
5 command-line interface.
6 """
7 
8 # Copyright (c) 2020. Lightly AG and its affiliates.
9 # All Rights Reserved
10 
11 import hydra
12 
13 import torchvision
14 
15 from lightly.cli._helpers import fix_input_path
16 
17 from lightly.api.utils import getenv
18 from lightly.api.api_workflow_client import ApiWorkflowClient
19 from lightly.data import LightlyDataset
20 
21 
22 def _upload_cli(cfg, is_cli_call=True):
23 
24     input_dir = cfg['input_dir']
25     if input_dir and is_cli_call:
26         input_dir = fix_input_path(input_dir)
27 
28     path_to_embeddings = cfg['embeddings']
29     if path_to_embeddings and is_cli_call:
30         path_to_embeddings = fix_input_path(path_to_embeddings)
31 
32     dataset_id = cfg['dataset_id']
33     token = cfg['token']
34 
35     size = cfg['resize']
36     if not isinstance(size, int):
37         size = tuple(size)
38     transform = None
39     if isinstance(size, tuple) or size > 0:
40         transform = torchvision.transforms.Resize(size)
41 
42     if not token or not dataset_id:
43         print('Please specify your access token and dataset id.')
44         print('For help, try: lightly-upload --help')
45         return
46 
47     api_workflow_client = ApiWorkflowClient(
48         token=token, dataset_id=dataset_id
49     )
50 
51     if input_dir:
52         mode = cfg['upload']
53         dataset = LightlyDataset(input_dir=input_dir, transform=transform)
54         api_workflow_client.upload_dataset(
55             input=dataset, mode=mode
56         )
57 
58     if path_to_embeddings:
59         name = cfg['embedding_name']
60         api_workflow_client.upload_embeddings(
61             path_to_embeddings_csv=path_to_embeddings, name=name
62         )
63 
64 
65 @hydra.main(config_path='config', config_name='config')
66 def upload_cli(cfg):
67     """Upload images/embeddings from the command-line to the Lightly platform.
68 
69     Args:
70         cfg:
71             The default configs are loaded from the config file.
72             To overwrite them please see the section on the config file 
73             (.config.config.yaml).
74     
75     Command-Line Args:
76         input_dir:
77             Path to the input directory where images are stored.
78         embeddings:
79             Path to the csv file storing the embeddings generated by
80             lightly.
81         token:
82             User access token to the Lightly platform. If dataset_id
83             and token are specified, the images and embeddings are 
84             uploaded to the platform.
85         dataset_id:
86             Identifier of the dataset on the Lightly platform. If 
87             dataset_id and token are specified, the images and 
88             embeddings are uploaded to the platform.
89         upload:
90             String to determine whether to upload the full images, 
91             thumbnails only, or metadata only.
92 
93             Must be one of ['full', 'thumbnails', 'metadata']
94         embedding_name:
95             Assign the embedding a name in order to identify it on the 
96             Lightly platform.
97         resize:
98             Desired size of the uploaded images. If negative, default size is used.
99             If size is a sequence like (h, w), output size will be matched to 
100             this. If size is an int, smaller edge of the image will be matched 
101             to this number. i.e, if height > width, then image will be rescaled
102             to (size * height / width, size).
103 
104     Examples:
105         >>> # upload thumbnails to the Lightly platform
106         >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ'
107         >>> 
108         >>> # upload full images to the Lightly platform
109         >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ' upload='full'
110         >>>
111         >>> # upload metadata to the Lightly platform
112         >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ' upload='metadata'
113         >>>
114         >>> # upload embeddings to the Lightly platform (must have uploaded images beforehand)
115         >>> lightly-upload embeddings=embeddings.csv token='123' dataset_id='XYZ'
116         >>>
117         >>> # upload both, images and embeddings in a single command
118         >>> lightly-upload input_dir=data/ embeddings=embeddings.csv upload='full' \\
119         >>>     token='123' dataset_id='XYZ'
120 
121     """
122     _upload_cli(cfg)
123 
124 
125 def entry():
126     upload_cli()
127
```

Path: `lightly/api/api_workflow_upload_dataset.py`
Content:
```
1 import warnings
2 from concurrent.futures.thread import ThreadPoolExecutor
3 from typing import Union
4 import tqdm
5 
6 from lightly.openapi_generated.swagger_client import TagCreator
7 from lightly.openapi_generated.swagger_client.models.sample_create_request import SampleCreateRequest
8 from lightly.api.utils import check_filename, check_image, get_thumbnail_from_img, PIL_to_bytes
9 from lightly.openapi_generated.swagger_client.models.initial_tag_create_request import InitialTagCreateRequest
10 from lightly.data.dataset import LightlyDataset
11 
12 from lightly.api.utils import retry
13 
14 
15 class _UploadDatasetMixin:
16 
17     def upload_dataset(self, input: Union[str, LightlyDataset], max_workers: int = 8,
18                        mode: str = 'thumbnails', verbose: bool = True):
19         """Uploads a dataset to to the Lightly cloud solution.
20 
21         Args:
22             input:
23                 one of the following:
24                     - the path to the dataset, e.g. "path/to/dataset"
25                     - the dataset in form of a LightlyDataset
26             max_workers:
27                 Maximum number of workers uploading images in parallel.
28             max_requests:
29                 Maximum number of requests a single worker can do before he has
30                 to wait for the others.
31             mode:
32                 One of [full, thumbnails, metadata]. Whether to upload thumbnails,
33                 full images, or metadata only.
34 
35         Raises:
36             ValueError if dataset is too large or input has the wrong type
37             RuntimeError if the connection to the server failed.
38 
39         """
40         no_tags_on_server = len(self._get_all_tags())
41         if no_tags_on_server > 0:
42             warnings.warn(f"Dataset with id {self.dataset_id} has already been completely uploaded to the platform. Skipping upload.")
43             return
44 
45         # Check input variable 'input'
46         if isinstance(input, str):
47             dataset = LightlyDataset(input_dir=input)
48         elif isinstance(input, LightlyDataset):
49             dataset = input
50         else:
51             raise ValueError(f"input must either be a LightlyDataset or the path to the dataset as str, "
52                              f"but is of type {type(input)}")
53 
54         # check the allowed dataset size
55         max_dataset_size_str = self.quota_api.get_quota_maximum_dataset_size()
56         max_dataset_size = int(max_dataset_size_str)
57         if len(dataset) > max_dataset_size:
58             msg = f'Your dataset has {len(dataset)} samples which'
59             msg += f' is more than the allowed maximum of {max_dataset_size}'
60             raise ValueError(msg)
61 
62         # handle the case where len(dataset) < max_workers
63         max_workers = min(len(dataset), max_workers)
64 
65         # upload the samples
66         if verbose:
67             print(f'Uploading images (with {max_workers} workers).', flush=True)
68 
69         pbar = tqdm.tqdm(unit='imgs', total=len(dataset))
70         tqdm_lock = tqdm.tqdm.get_lock()
71 
72         # define lambda function for concurrent upload
73         def lambda_(i):
74             # load image
75             image, label, filename = dataset[i]
76             # try to upload image
77             try:
78                 self._upload_single_image(
79                     image=image,
80                     label=label,
81                     filename=filename,
82                     mode=mode,
83                 )
84                 success = True
85             except Exception as e:
86                 warnings.warn(f"Upload of image {filename} failed with error {e}")
87                 success = False
88 
89             # update the progress bar
90             tqdm_lock.acquire()  # lock
91             pbar.update(1)  # update
92             tqdm_lock.release()  # unlock
93             # return whether the upload was successful
94             return success
95 
96         with ThreadPoolExecutor(max_workers=max_workers) as executor:
97             results = list(executor.map(
98                 lambda_, [i for i in range(len(dataset))], chunksize=1))
99 
100         if not all(results):
101             msg = 'Warning: Unsuccessful upload(s)! '
102             msg += 'This could cause problems when uploading embeddings.'
103             msg += 'Failed at image: {}'.format(results.index(False))
104             warnings.warn(msg)
105 
106         # set image type of data and create initial tag
107         if mode == 'full':
108             img_type = 'full'
109         elif mode == 'thumbnails':
110             img_type = 'thumbnail'
111         else:
112             img_type = 'meta'
113 
114         initial_tag_create_request = InitialTagCreateRequest(img_type=img_type, creator=TagCreator.USER_PIP)
115         self.tags_api.create_initial_tag_by_dataset_id(body=initial_tag_create_request, dataset_id=self.dataset_id)
116 
117     def _upload_single_image(self, image, label, filename: str, mode):
118         """Uploads a single image to the Lightly platform.
119 
120         """
121 
122         # check whether the filename is too long
123         basename = filename
124         if not check_filename(basename):
125             msg = (f'Filename {basename} is longer than the allowed maximum of '
126                    'characters and will be skipped.')
127             warnings.warn(msg)
128             return False
129 
130         # calculate metadata, and check if corrupted
131         metadata = check_image(image)
132 
133         # generate thumbnail if necessary
134         thumbname = None
135         if not metadata['is_corrupted'] and mode in ["thumbnails", "full"]:
136             thumbname = '.'.join(basename.split('.')[:-1]) + '_thumb.webp'
137 
138         body = SampleCreateRequest(file_name=basename, thumb_name=thumbname, meta_data=metadata)
139         sample_id = retry(
140             self.samples_api.create_sample_by_dataset_id,
141             body=body,
142             dataset_id=self.dataset_id
143         ).id
144 
145         if not metadata['is_corrupted'] and mode in ["thumbnails", "full"]:
146             thumbnail = get_thumbnail_from_img(image)
147             image_to_upload = PIL_to_bytes(thumbnail, ext='webp', quality=90)
148 
149             signed_url = retry(
150                 self.samples_api.get_sample_image_write_url_by_id,
151                 dataset_id=self.dataset_id,
152                 sample_id=sample_id,
153                 is_thumbnail=True
154             )
155 
156             # try to upload thumbnail
157             retry(
158                 self.upload_file_with_signed_url,
159                 image_to_upload,
160                 signed_url
161             )
162 
163             thumbnail.close()
164 
165         if not metadata['is_corrupted'] and mode == "full":
166             image_to_upload = PIL_to_bytes(image)
167 
168             signed_url = retry(
169                 self.samples_api.get_sample_image_write_url_by_id,
170                 dataset_id=self.dataset_id,
171                 sample_id=sample_id,
172                 is_thumbnail=False
173             )
174 
175             # try to upload thumbnail
176             retry(
177                 self.upload_file_with_signed_url,
178                 image_to_upload,
179                 signed_url
180             )
181 
182             image.close()
183 
184 
185
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/lightly/api/api_workflow_upload_dataset.py b/lightly/api/api_workflow_upload_dataset.py
--- a/lightly/api/api_workflow_upload_dataset.py
+++ b/lightly/api/api_workflow_upload_dataset.py
@@ -61,6 +61,7 @@
 
         # handle the case where len(dataset) < max_workers
         max_workers = min(len(dataset), max_workers)
+        max_workers = max(max_workers, 1)
 
         # upload the samples
         if verbose:
diff --git a/lightly/cli/upload_cli.py b/lightly/cli/upload_cli.py
--- a/lightly/cli/upload_cli.py
+++ b/lightly/cli/upload_cli.py
@@ -52,7 +52,7 @@
         mode = cfg['upload']
         dataset = LightlyDataset(input_dir=input_dir, transform=transform)
         api_workflow_client.upload_dataset(
-            input=dataset, mode=mode
+            input=dataset, mode=mode, max_workers=cfg['loader']['num_workers']
         )
 
     if path_to_embeddings:

{"golden_diff": "diff --git a/lightly/api/api_workflow_upload_dataset.py b/lightly/api/api_workflow_upload_dataset.py\n--- a/lightly/api/api_workflow_upload_dataset.py\n+++ b/lightly/api/api_workflow_upload_dataset.py\n@@ -61,6 +61,7 @@\n \n         # handle the case where len(dataset) < max_workers\n         max_workers = min(len(dataset), max_workers)\n+        max_workers = max(max_workers, 1)\n \n         # upload the samples\n         if verbose:\ndiff --git a/lightly/cli/upload_cli.py b/lightly/cli/upload_cli.py\n--- a/lightly/cli/upload_cli.py\n+++ b/lightly/cli/upload_cli.py\n@@ -52,7 +52,7 @@\n         mode = cfg['upload']\n         dataset = LightlyDataset(input_dir=input_dir, transform=transform)\n         api_workflow_client.upload_dataset(\n-            input=dataset, mode=mode\n+            input=dataset, mode=mode, max_workers=cfg['loader']['num_workers']\n         )\n \n     if path_to_embeddings:\n", "issue": "loader.num_workers=12 doesn't actually spawn multiple workers\nI've noticed that when trying to create an embedding with the param `loader.num_workers` there seems to be no difference in CPU consumption;\r\n`lightly-embed input_dir='./75000' trainer.max_epochs=1 loader.num_workers=12`\r\n\r\nThis could be that my images are ~8kb and there is some other bottleneck preventing the CPUs to max-out? in My ActivityMonitor (macOSX) I see that I have 80% idle CPU\r\n\r\nSystem: MacBook Pro (16-inch, 2019), 2.6 GHz 6-Core Intel Core i7\r\nOS: MacOSX 11.2.3\r\n\r\n\r\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n\"\"\"**Lightly Upload:** Upload images to the Lightly platform.\n\nThis module contains the entrypoint for the **lightly-upload**\ncommand-line interface.\n\"\"\"\n\n# Copyright (c) 2020. Lightly AG and its affiliates.\n# All Rights Reserved\n\nimport hydra\n\nimport torchvision\n\nfrom lightly.cli._helpers import fix_input_path\n\nfrom lightly.api.utils import getenv\nfrom lightly.api.api_workflow_client import ApiWorkflowClient\nfrom lightly.data import LightlyDataset\n\n\ndef _upload_cli(cfg, is_cli_call=True):\n\n    input_dir = cfg['input_dir']\n    if input_dir and is_cli_call:\n        input_dir = fix_input_path(input_dir)\n\n    path_to_embeddings = cfg['embeddings']\n    if path_to_embeddings and is_cli_call:\n        path_to_embeddings = fix_input_path(path_to_embeddings)\n\n    dataset_id = cfg['dataset_id']\n    token = cfg['token']\n\n    size = cfg['resize']\n    if not isinstance(size, int):\n        size = tuple(size)\n    transform = None\n    if isinstance(size, tuple) or size > 0:\n        transform = torchvision.transforms.Resize(size)\n\n    if not token or not dataset_id:\n        print('Please specify your access token and dataset id.')\n        print('For help, try: lightly-upload --help')\n        return\n\n    api_workflow_client = ApiWorkflowClient(\n        token=token, dataset_id=dataset_id\n    )\n\n    if input_dir:\n        mode = cfg['upload']\n        dataset = LightlyDataset(input_dir=input_dir, transform=transform)\n        api_workflow_client.upload_dataset(\n            input=dataset, mode=mode\n        )\n\n    if path_to_embeddings:\n        name = cfg['embedding_name']\n        api_workflow_client.upload_embeddings(\n            path_to_embeddings_csv=path_to_embeddings, name=name\n        )\n\n\n@hydra.main(config_path='config', config_name='config')\ndef upload_cli(cfg):\n    \"\"\"Upload images/embeddings from the command-line to the Lightly platform.\n\n    Args:\n        cfg:\n            The default configs are loaded from the config file.\n            To overwrite them please see the section on the config file \n            (.config.config.yaml).\n    \n    Command-Line Args:\n        input_dir:\n            Path to the input directory where images are stored.\n        embeddings:\n            Path to the csv file storing the embeddings generated by\n            lightly.\n        token:\n            User access token to the Lightly platform. If dataset_id\n            and token are specified, the images and embeddings are \n            uploaded to the platform.\n        dataset_id:\n            Identifier of the dataset on the Lightly platform. If \n            dataset_id and token are specified, the images and \n            embeddings are uploaded to the platform.\n        upload:\n            String to determine whether to upload the full images, \n            thumbnails only, or metadata only.\n\n            Must be one of ['full', 'thumbnails', 'metadata']\n        embedding_name:\n            Assign the embedding a name in order to identify it on the \n            Lightly platform.\n        resize:\n            Desired size of the uploaded images. If negative, default size is used.\n            If size is a sequence like (h, w), output size will be matched to \n            this. If size is an int, smaller edge of the image will be matched \n            to this number. i.e, if height > width, then image will be rescaled\n            to (size * height / width, size).\n\n    Examples:\n        >>> #\u00a0upload thumbnails to the Lightly platform\n        >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ'\n        >>> \n        >>> # upload full images to the Lightly platform\n        >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ' upload='full'\n        >>>\n        >>> # upload metadata to the Lightly platform\n        >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ' upload='metadata'\n        >>>\n        >>> # upload embeddings to the Lightly platform (must have uploaded images beforehand)\n        >>> lightly-upload embeddings=embeddings.csv token='123' dataset_id='XYZ'\n        >>>\n        >>> # upload both, images and embeddings in a single command\n        >>> lightly-upload input_dir=data/ embeddings=embeddings.csv upload='full' \\\\\n        >>>     token='123' dataset_id='XYZ'\n\n    \"\"\"\n    _upload_cli(cfg)\n\n\ndef entry():\n    upload_cli()\n", "path": "lightly/cli/upload_cli.py"}, {"content": "import warnings\nfrom concurrent.futures.thread import ThreadPoolExecutor\nfrom typing import Union\nimport tqdm\n\nfrom lightly.openapi_generated.swagger_client import TagCreator\nfrom lightly.openapi_generated.swagger_client.models.sample_create_request import SampleCreateRequest\nfrom lightly.api.utils import check_filename, check_image, get_thumbnail_from_img, PIL_to_bytes\nfrom lightly.openapi_generated.swagger_client.models.initial_tag_create_request import InitialTagCreateRequest\nfrom lightly.data.dataset import LightlyDataset\n\nfrom lightly.api.utils import retry\n\n\nclass _UploadDatasetMixin:\n\n    def upload_dataset(self, input: Union[str, LightlyDataset], max_workers: int = 8,\n                       mode: str = 'thumbnails', verbose: bool = True):\n        \"\"\"Uploads a dataset to to the Lightly cloud solution.\n\n        Args:\n            input:\n                one of the following:\n                    - the path to the dataset, e.g. \"path/to/dataset\"\n                    - the dataset in form of a LightlyDataset\n            max_workers:\n                Maximum number of workers uploading images in parallel.\n            max_requests:\n                Maximum number of requests a single worker can do before he has\n                to wait for the others.\n            mode:\n                One of [full, thumbnails, metadata]. Whether to upload thumbnails,\n                full images, or metadata only.\n\n        Raises:\n            ValueError if dataset is too large or input has the wrong type\n            RuntimeError if the connection to the server failed.\n\n        \"\"\"\n        no_tags_on_server = len(self._get_all_tags())\n        if no_tags_on_server > 0:\n            warnings.warn(f\"Dataset with id {self.dataset_id} has already been completely uploaded to the platform. Skipping upload.\")\n            return\n\n        # Check input variable 'input'\n        if isinstance(input, str):\n            dataset = LightlyDataset(input_dir=input)\n        elif isinstance(input, LightlyDataset):\n            dataset = input\n        else:\n            raise ValueError(f\"input must either be a LightlyDataset or the path to the dataset as str, \"\n                             f\"but is of type {type(input)}\")\n\n        # check the allowed dataset size\n        max_dataset_size_str = self.quota_api.get_quota_maximum_dataset_size()\n        max_dataset_size = int(max_dataset_size_str)\n        if len(dataset) > max_dataset_size:\n            msg = f'Your dataset has {len(dataset)} samples which'\n            msg += f' is more than the allowed maximum of {max_dataset_size}'\n            raise ValueError(msg)\n\n        # handle the case where len(dataset) < max_workers\n        max_workers = min(len(dataset), max_workers)\n\n        # upload the samples\n        if verbose:\n            print(f'Uploading images (with {max_workers} workers).', flush=True)\n\n        pbar = tqdm.tqdm(unit='imgs', total=len(dataset))\n        tqdm_lock = tqdm.tqdm.get_lock()\n\n        # define lambda function for concurrent upload\n        def lambda_(i):\n            # load image\n            image, label, filename = dataset[i]\n            # try to upload image\n            try:\n                self._upload_single_image(\n                    image=image,\n                    label=label,\n                    filename=filename,\n                    mode=mode,\n                )\n                success = True\n            except Exception as e:\n                warnings.warn(f\"Upload of image {filename} failed with error {e}\")\n                success = False\n\n            # update the progress bar\n            tqdm_lock.acquire()  # lock\n            pbar.update(1)  # update\n            tqdm_lock.release()  # unlock\n            # return whether the upload was successful\n            return success\n\n        with ThreadPoolExecutor(max_workers=max_workers) as executor:\n            results = list(executor.map(\n                lambda_, [i for i in range(len(dataset))], chunksize=1))\n\n        if not all(results):\n            msg = 'Warning: Unsuccessful upload(s)! '\n            msg += 'This could cause problems when uploading embeddings.'\n            msg += 'Failed at image: {}'.format(results.index(False))\n            warnings.warn(msg)\n\n        # set image type of data and create initial tag\n        if mode == 'full':\n            img_type = 'full'\n        elif mode == 'thumbnails':\n            img_type = 'thumbnail'\n        else:\n            img_type = 'meta'\n\n        initial_tag_create_request = InitialTagCreateRequest(img_type=img_type, creator=TagCreator.USER_PIP)\n        self.tags_api.create_initial_tag_by_dataset_id(body=initial_tag_create_request, dataset_id=self.dataset_id)\n\n    def _upload_single_image(self, image, label, filename: str, mode):\n        \"\"\"Uploads a single image to the Lightly platform.\n\n        \"\"\"\n\n        # check whether the filename is too long\n        basename = filename\n        if not check_filename(basename):\n            msg = (f'Filename {basename} is longer than the allowed maximum of '\n                   'characters and will be skipped.')\n            warnings.warn(msg)\n            return False\n\n        # calculate metadata, and check if corrupted\n        metadata = check_image(image)\n\n        # generate thumbnail if necessary\n        thumbname = None\n        if not metadata['is_corrupted'] and mode in [\"thumbnails\", \"full\"]:\n            thumbname = '.'.join(basename.split('.')[:-1]) + '_thumb.webp'\n\n        body = SampleCreateRequest(file_name=basename, thumb_name=thumbname, meta_data=metadata)\n        sample_id = retry(\n            self.samples_api.create_sample_by_dataset_id,\n            body=body,\n            dataset_id=self.dataset_id\n        ).id\n\n        if not metadata['is_corrupted'] and mode in [\"thumbnails\", \"full\"]:\n            thumbnail = get_thumbnail_from_img(image)\n            image_to_upload = PIL_to_bytes(thumbnail, ext='webp', quality=90)\n\n            signed_url = retry(\n                self.samples_api.get_sample_image_write_url_by_id,\n                dataset_id=self.dataset_id,\n                sample_id=sample_id,\n                is_thumbnail=True\n            )\n\n            # try to upload thumbnail\n            retry(\n                self.upload_file_with_signed_url,\n                image_to_upload,\n                signed_url\n            )\n\n            thumbnail.close()\n\n        if not metadata['is_corrupted'] and mode == \"full\":\n            image_to_upload = PIL_to_bytes(image)\n\n            signed_url = retry(\n                self.samples_api.get_sample_image_write_url_by_id,\n                dataset_id=self.dataset_id,\n                sample_id=sample_id,\n                is_thumbnail=False\n            )\n\n            # try to upload thumbnail\n            retry(\n                self.upload_file_with_signed_url,\n                image_to_upload,\n                signed_url\n            )\n\n            image.close()\n\n\n", "path": "lightly/api/api_workflow_upload_dataset.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n\"\"\"**Lightly Upload:** Upload images to the Lightly platform.\n\nThis module contains the entrypoint for the **lightly-upload**\ncommand-line interface.\n\"\"\"\n\n# Copyright (c) 2020. Lightly AG and its affiliates.\n# All Rights Reserved\n\nimport hydra\n\nimport torchvision\n\nfrom lightly.cli._helpers import fix_input_path\n\nfrom lightly.api.utils import getenv\nfrom lightly.api.api_workflow_client import ApiWorkflowClient\nfrom lightly.data import LightlyDataset\n\n\ndef _upload_cli(cfg, is_cli_call=True):\n\n    input_dir = cfg['input_dir']\n    if input_dir and is_cli_call:\n        input_dir = fix_input_path(input_dir)\n\n    path_to_embeddings = cfg['embeddings']\n    if path_to_embeddings and is_cli_call:\n        path_to_embeddings = fix_input_path(path_to_embeddings)\n\n    dataset_id = cfg['dataset_id']\n    token = cfg['token']\n\n    size = cfg['resize']\n    if not isinstance(size, int):\n        size = tuple(size)\n    transform = None\n    if isinstance(size, tuple) or size > 0:\n        transform = torchvision.transforms.Resize(size)\n\n    if not token or not dataset_id:\n        print('Please specify your access token and dataset id.')\n        print('For help, try: lightly-upload --help')\n        return\n\n    api_workflow_client = ApiWorkflowClient(\n        token=token, dataset_id=dataset_id\n    )\n\n    if input_dir:\n        mode = cfg['upload']\n        dataset = LightlyDataset(input_dir=input_dir, transform=transform)\n        api_workflow_client.upload_dataset(\n            input=dataset, mode=mode, max_workers=cfg['loader']['num_workers']\n        )\n\n    if path_to_embeddings:\n        name = cfg['embedding_name']\n        api_workflow_client.upload_embeddings(\n            path_to_embeddings_csv=path_to_embeddings, name=name\n        )\n\n\n@hydra.main(config_path='config', config_name='config')\ndef upload_cli(cfg):\n    \"\"\"Upload images/embeddings from the command-line to the Lightly platform.\n\n    Args:\n        cfg:\n            The default configs are loaded from the config file.\n            To overwrite them please see the section on the config file \n            (.config.config.yaml).\n    \n    Command-Line Args:\n        input_dir:\n            Path to the input directory where images are stored.\n        embeddings:\n            Path to the csv file storing the embeddings generated by\n            lightly.\n        token:\n            User access token to the Lightly platform. If dataset_id\n            and token are specified, the images and embeddings are \n            uploaded to the platform.\n        dataset_id:\n            Identifier of the dataset on the Lightly platform. If \n            dataset_id and token are specified, the images and \n            embeddings are uploaded to the platform.\n        upload:\n            String to determine whether to upload the full images, \n            thumbnails only, or metadata only.\n\n            Must be one of ['full', 'thumbnails', 'metadata']\n        embedding_name:\n            Assign the embedding a name in order to identify it on the \n            Lightly platform.\n        resize:\n            Desired size of the uploaded images. If negative, default size is used.\n            If size is a sequence like (h, w), output size will be matched to \n            this. If size is an int, smaller edge of the image will be matched \n            to this number. i.e, if height > width, then image will be rescaled\n            to (size * height / width, size).\n\n    Examples:\n        >>> #\u00a0upload thumbnails to the Lightly platform\n        >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ'\n        >>> \n        >>> # upload full images to the Lightly platform\n        >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ' upload='full'\n        >>>\n        >>> # upload metadata to the Lightly platform\n        >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ' upload='metadata'\n        >>>\n        >>> # upload embeddings to the Lightly platform (must have uploaded images beforehand)\n        >>> lightly-upload embeddings=embeddings.csv token='123' dataset_id='XYZ'\n        >>>\n        >>> # upload both, images and embeddings in a single command\n        >>> lightly-upload input_dir=data/ embeddings=embeddings.csv upload='full' \\\\\n        >>>     token='123' dataset_id='XYZ'\n\n    \"\"\"\n    _upload_cli(cfg)\n\n\ndef entry():\n    upload_cli()\n", "path": "lightly/cli/upload_cli.py"}, {"content": "import warnings\nfrom concurrent.futures.thread import ThreadPoolExecutor\nfrom typing import Union\nimport tqdm\n\nfrom lightly.openapi_generated.swagger_client import TagCreator\nfrom lightly.openapi_generated.swagger_client.models.sample_create_request import SampleCreateRequest\nfrom lightly.api.utils import check_filename, check_image, get_thumbnail_from_img, PIL_to_bytes\nfrom lightly.openapi_generated.swagger_client.models.initial_tag_create_request import InitialTagCreateRequest\nfrom lightly.data.dataset import LightlyDataset\n\nfrom lightly.api.utils import retry\n\n\nclass _UploadDatasetMixin:\n\n    def upload_dataset(self, input: Union[str, LightlyDataset], max_workers: int = 8,\n                       mode: str = 'thumbnails', verbose: bool = True):\n        \"\"\"Uploads a dataset to to the Lightly cloud solution.\n\n        Args:\n            input:\n                one of the following:\n                    - the path to the dataset, e.g. \"path/to/dataset\"\n                    - the dataset in form of a LightlyDataset\n            max_workers:\n                Maximum number of workers uploading images in parallel.\n            max_requests:\n                Maximum number of requests a single worker can do before he has\n                to wait for the others.\n            mode:\n                One of [full, thumbnails, metadata]. Whether to upload thumbnails,\n                full images, or metadata only.\n\n        Raises:\n            ValueError if dataset is too large or input has the wrong type\n            RuntimeError if the connection to the server failed.\n\n        \"\"\"\n        no_tags_on_server = len(self._get_all_tags())\n        if no_tags_on_server > 0:\n            warnings.warn(f\"Dataset with id {self.dataset_id} has already been completely uploaded to the platform. Skipping upload.\")\n            return\n\n        # Check input variable 'input'\n        if isinstance(input, str):\n            dataset = LightlyDataset(input_dir=input)\n        elif isinstance(input, LightlyDataset):\n            dataset = input\n        else:\n            raise ValueError(f\"input must either be a LightlyDataset or the path to the dataset as str, \"\n                             f\"but is of type {type(input)}\")\n\n        # check the allowed dataset size\n        max_dataset_size_str = self.quota_api.get_quota_maximum_dataset_size()\n        max_dataset_size = int(max_dataset_size_str)\n        if len(dataset) > max_dataset_size:\n            msg = f'Your dataset has {len(dataset)} samples which'\n            msg += f' is more than the allowed maximum of {max_dataset_size}'\n            raise ValueError(msg)\n\n        # handle the case where len(dataset) < max_workers\n        max_workers = min(len(dataset), max_workers)\n        max_workers = max(max_workers, 1)\n\n        # upload the samples\n        if verbose:\n            print(f'Uploading images (with {max_workers} workers).', flush=True)\n\n        pbar = tqdm.tqdm(unit='imgs', total=len(dataset))\n        tqdm_lock = tqdm.tqdm.get_lock()\n\n        # define lambda function for concurrent upload\n        def lambda_(i):\n            # load image\n            image, label, filename = dataset[i]\n            # try to upload image\n            try:\n                self._upload_single_image(\n                    image=image,\n                    label=label,\n                    filename=filename,\n                    mode=mode,\n                )\n                success = True\n            except Exception as e:\n                warnings.warn(f\"Upload of image {filename} failed with error {e}\")\n                success = False\n\n            # update the progress bar\n            tqdm_lock.acquire()  # lock\n            pbar.update(1)  # update\n            tqdm_lock.release()  # unlock\n            # return whether the upload was successful\n            return success\n\n        with ThreadPoolExecutor(max_workers=max_workers) as executor:\n            results = list(executor.map(\n                lambda_, [i for i in range(len(dataset))], chunksize=1))\n\n        if not all(results):\n            msg = 'Warning: Unsuccessful upload(s)! '\n            msg += 'This could cause problems when uploading embeddings.'\n            msg += 'Failed at image: {}'.format(results.index(False))\n            warnings.warn(msg)\n\n        # set image type of data and create initial tag\n        if mode == 'full':\n            img_type = 'full'\n        elif mode == 'thumbnails':\n            img_type = 'thumbnail'\n        else:\n            img_type = 'meta'\n\n        initial_tag_create_request = InitialTagCreateRequest(img_type=img_type, creator=TagCreator.USER_PIP)\n        self.tags_api.create_initial_tag_by_dataset_id(body=initial_tag_create_request, dataset_id=self.dataset_id)\n\n    def _upload_single_image(self, image, label, filename: str, mode):\n        \"\"\"Uploads a single image to the Lightly platform.\n\n        \"\"\"\n\n        # check whether the filename is too long\n        basename = filename\n        if not check_filename(basename):\n            msg = (f'Filename {basename} is longer than the allowed maximum of '\n                   'characters and will be skipped.')\n            warnings.warn(msg)\n            return False\n\n        # calculate metadata, and check if corrupted\n        metadata = check_image(image)\n\n        # generate thumbnail if necessary\n        thumbname = None\n        if not metadata['is_corrupted'] and mode in [\"thumbnails\", \"full\"]:\n            thumbname = '.'.join(basename.split('.')[:-1]) + '_thumb.webp'\n\n        body = SampleCreateRequest(file_name=basename, thumb_name=thumbname, meta_data=metadata)\n        sample_id = retry(\n            self.samples_api.create_sample_by_dataset_id,\n            body=body,\n            dataset_id=self.dataset_id\n        ).id\n\n        if not metadata['is_corrupted'] and mode in [\"thumbnails\", \"full\"]:\n            thumbnail = get_thumbnail_from_img(image)\n            image_to_upload = PIL_to_bytes(thumbnail, ext='webp', quality=90)\n\n            signed_url = retry(\n                self.samples_api.get_sample_image_write_url_by_id,\n                dataset_id=self.dataset_id,\n                sample_id=sample_id,\n                is_thumbnail=True\n            )\n\n            # try to upload thumbnail\n            retry(\n                self.upload_file_with_signed_url,\n                image_to_upload,\n                signed_url\n            )\n\n            thumbnail.close()\n\n        if not metadata['is_corrupted'] and mode == \"full\":\n            image_to_upload = PIL_to_bytes(image)\n\n            signed_url = retry(\n                self.samples_api.get_sample_image_write_url_by_id,\n                dataset_id=self.dataset_id,\n                sample_id=sample_id,\n                is_thumbnail=False\n            )\n\n            # try to upload thumbnail\n            retry(\n                self.upload_file_with_signed_url,\n                image_to_upload,\n                signed_url\n            )\n\n            image.close()\n\n\n", "path": "lightly/api/api_workflow_upload_dataset.py"}]}