We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Upload.__init__ relies too much on makedirs
**CKAN version**

2.8, 2.9

**Describe the bug**

`ckan.lib.uploader:Upload.__init__` relies on `os.makedirs` to create the storage directory. However, this is inadequate if, for example, the storage directory has been created by another process, with sufficient permissions for CKAN to use it, yet with insufficient permissions in the parent directory for CKAN to have created it.

It also adds extra OS calls on every creation of an Upload object.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `ckan/lib/uploader.py`
Content:
```
1 # encoding: utf-8
2 
3 import os
4 import cgi
5 import datetime
6 import logging
7 import magic
8 import mimetypes
9 from urllib.parse import urlparse
10 
11 from werkzeug.datastructures import FileStorage as FlaskFileStorage
12 
13 import ckan.lib.munge as munge
14 import ckan.logic as logic
15 import ckan.plugins as plugins
16 from ckan.common import config
17 
18 ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage)
19 MB = 1 << 20
20 
21 log = logging.getLogger(__name__)
22 
23 _storage_path = None
24 _max_resource_size = None
25 _max_image_size = None
26 
27 
28 def _copy_file(input_file, output_file, max_size):
29     input_file.seek(0)
30     current_size = 0
31     while True:
32         current_size = current_size + 1
33         # MB chunks
34         data = input_file.read(MB)
35 
36         if not data:
37             break
38         output_file.write(data)
39         if current_size > max_size:
40             raise logic.ValidationError({'upload': ['File upload too large']})
41 
42 
43 def _get_underlying_file(wrapper):
44     if isinstance(wrapper, FlaskFileStorage):
45         return wrapper.stream
46     return wrapper.file
47 
48 
49 def get_uploader(upload_to, old_filename=None):
50     '''Query IUploader plugins and return an uploader instance for general
51     files.'''
52     upload = None
53     for plugin in plugins.PluginImplementations(plugins.IUploader):
54         upload = plugin.get_uploader(upload_to, old_filename)
55         if upload:
56             break
57 
58     # default uploader
59     if upload is None:
60         upload = Upload(upload_to, old_filename)
61 
62     return upload
63 
64 
65 def get_resource_uploader(data_dict):
66     '''Query IUploader plugins and return a resource uploader instance.'''
67     upload = None
68     for plugin in plugins.PluginImplementations(plugins.IUploader):
69         upload = plugin.get_resource_uploader(data_dict)
70         if upload:
71             break
72 
73     # default uploader
74     if upload is None:
75         upload = ResourceUpload(data_dict)
76 
77     return upload
78 
79 
80 def get_storage_path():
81     '''Function to cache storage path'''
82     global _storage_path
83 
84     # None means it has not been set. False means not in config.
85     if _storage_path is None:
86         storage_path = config.get_value('ckan.storage_path')
87         if storage_path:
88             _storage_path = storage_path
89         else:
90             log.critical('''Please specify a ckan.storage_path in your config
91                          for your uploads''')
92             _storage_path = False
93 
94     return _storage_path
95 
96 
97 def get_max_image_size():
98     global _max_image_size
99     if _max_image_size is None:
100         _max_image_size = config.get_value('ckan.max_image_size')
101     return _max_image_size
102 
103 
104 def get_max_resource_size():
105     global _max_resource_size
106     if _max_resource_size is None:
107         _max_resource_size = config.get_value('ckan.max_resource_size')
108     return _max_resource_size
109 
110 
111 class Upload(object):
112     def __init__(self, object_type, old_filename=None):
113         ''' Setup upload by creating a subdirectory of the storage directory
114         of name object_type. old_filename is the name of the file in the url
115         field last time'''
116 
117         self.storage_path = None
118         self.filename = None
119         self.filepath = None
120         path = get_storage_path()
121         if not path:
122             return
123         self.storage_path = os.path.join(path, 'storage',
124                                          'uploads', object_type)
125         try:
126             os.makedirs(self.storage_path)
127         except OSError as e:
128             # errno 17 is file already exists
129             if e.errno != 17:
130                 raise
131         self.object_type = object_type
132         self.old_filename = old_filename
133         if old_filename:
134             self.old_filepath = os.path.join(self.storage_path, old_filename)
135 
136     def update_data_dict(self, data_dict, url_field, file_field, clear_field):
137         ''' Manipulate data from the data_dict.  url_field is the name of the
138         field where the upload is going to be. file_field is name of the key
139         where the FieldStorage is kept (i.e the field where the file data
140         actually is). clear_field is the name of a boolean field which
141         requests the upload to be deleted.  This needs to be called before
142         it reaches any validators'''
143 
144         self.url = data_dict.get(url_field, '')
145         self.clear = data_dict.pop(clear_field, None)
146         self.file_field = file_field
147         self.upload_field_storage = data_dict.pop(file_field, None)
148 
149         if not self.storage_path:
150             return
151 
152         if isinstance(self.upload_field_storage, ALLOWED_UPLOAD_TYPES):
153             if self.upload_field_storage.filename:
154                 self.filename = self.upload_field_storage.filename
155                 self.filename = str(datetime.datetime.utcnow()) + self.filename
156                 self.filename = munge.munge_filename_legacy(self.filename)
157                 self.filepath = os.path.join(self.storage_path, self.filename)
158                 data_dict[url_field] = self.filename
159                 self.upload_file = _get_underlying_file(
160                     self.upload_field_storage)
161                 self.tmp_filepath = self.filepath + '~'
162         # keep the file if there has been no change
163         elif self.old_filename and not self.old_filename.startswith('http'):
164             if not self.clear:
165                 data_dict[url_field] = self.old_filename
166             if self.clear and self.url == self.old_filename:
167                 data_dict[url_field] = ''
168 
169     def upload(self, max_size=2):
170         ''' Actually upload the file.
171         This should happen just before a commit but after the data has
172         been validated and flushed to the db. This is so we do not store
173         anything unless the request is actually good.
174         max_size is size in MB maximum of the file'''
175 
176         self.verify_type()
177 
178         if self.filename:
179             with open(self.tmp_filepath, 'wb+') as output_file:
180                 try:
181                     _copy_file(self.upload_file, output_file, max_size)
182                 except logic.ValidationError:
183                     os.remove(self.tmp_filepath)
184                     raise
185                 finally:
186                     self.upload_file.close()
187             os.rename(self.tmp_filepath, self.filepath)
188             self.clear = True
189 
190         if (self.clear and self.old_filename
191                 and not self.old_filename.startswith('http')):
192             try:
193                 os.remove(self.old_filepath)
194             except OSError:
195                 pass
196 
197     def verify_type(self):
198         if not self.filename:
199             return
200 
201         mimetypes = config.get_value(
202             f"ckan.upload.{self.object_type}.mimetypes")
203         types = config.get_value(f"ckan.upload.{self.object_type}.types")
204         if not mimetypes and not types:
205             return
206 
207         actual = magic.from_buffer(self.upload_file.read(1024), mime=True)
208         self.upload_file.seek(0, os.SEEK_SET)
209         err = {self.file_field: [f"Unsupported upload type: {actual}"]}
210 
211         if mimetypes and actual not in mimetypes:
212             raise logic.ValidationError(err)
213 
214         type_ = actual.split("/")[0]
215         if types and type_ not in types:
216             raise logic.ValidationError(err)
217 
218 
219 class ResourceUpload(object):
220     def __init__(self, resource):
221         path = get_storage_path()
222         config_mimetype_guess = config.get_value('ckan.mimetype_guess')
223 
224         if not path:
225             self.storage_path = None
226             return
227         self.storage_path = os.path.join(path, 'resources')
228         try:
229             os.makedirs(self.storage_path)
230         except OSError as e:
231             # errno 17 is file already exists
232             if e.errno != 17:
233                 raise
234         self.filename = None
235         self.mimetype = None
236 
237         url = resource.get('url')
238 
239         upload_field_storage = resource.pop('upload', None)
240         self.clear = resource.pop('clear_upload', None)
241 
242         if url and config_mimetype_guess == 'file_ext' and urlparse(url).path:
243             self.mimetype = mimetypes.guess_type(url)[0]
244 
245         if bool(upload_field_storage) and \
246                 isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):
247             self.filesize = 0  # bytes
248 
249             self.filename = upload_field_storage.filename
250             self.filename = munge.munge_filename(self.filename)
251             resource['url'] = self.filename
252             resource['url_type'] = 'upload'
253             resource['last_modified'] = datetime.datetime.utcnow()
254             self.upload_file = _get_underlying_file(upload_field_storage)
255             self.upload_file.seek(0, os.SEEK_END)
256             self.filesize = self.upload_file.tell()
257             # go back to the beginning of the file buffer
258             self.upload_file.seek(0, os.SEEK_SET)
259 
260             # check if the mimetype failed from guessing with the url
261             if not self.mimetype and config_mimetype_guess == 'file_ext':
262                 self.mimetype = mimetypes.guess_type(self.filename)[0]
263 
264             if not self.mimetype and config_mimetype_guess == 'file_contents':
265                 try:
266                     self.mimetype = magic.from_buffer(self.upload_file.read(),
267                                                       mime=True)
268                     self.upload_file.seek(0, os.SEEK_SET)
269                 except IOError:
270                     # Not that important if call above fails
271                     self.mimetype = None
272 
273         elif self.clear:
274             resource['url_type'] = ''
275 
276     def get_directory(self, id):
277         directory = os.path.join(self.storage_path,
278                                  id[0:3], id[3:6])
279         return directory
280 
281     def get_path(self, id):
282         directory = self.get_directory(id)
283         filepath = os.path.join(directory, id[6:])
284         return filepath
285 
286     def upload(self, id, max_size=10):
287         '''Actually upload the file.
288 
289         :returns: ``'file uploaded'`` if a new file was successfully uploaded
290             (whether it overwrote a previously uploaded file or not),
291             ``'file deleted'`` if an existing uploaded file was deleted,
292             or ``None`` if nothing changed
293         :rtype: ``string`` or ``None``
294 
295         '''
296         if not self.storage_path:
297             return
298 
299         # Get directory and filepath on the system
300         # where the file for this resource will be stored
301         directory = self.get_directory(id)
302         filepath = self.get_path(id)
303 
304         # If a filename has been provided (a file is being uploaded)
305         # we write it to the filepath (and overwrite it if it already
306         # exists). This way the uploaded file will always be stored
307         # in the same location
308         if self.filename:
309             try:
310                 os.makedirs(directory)
311             except OSError as e:
312                 # errno 17 is file already exists
313                 if e.errno != 17:
314                     raise
315             tmp_filepath = filepath + '~'
316             with open(tmp_filepath, 'wb+') as output_file:
317                 try:
318                     _copy_file(self.upload_file, output_file, max_size)
319                 except logic.ValidationError:
320                     os.remove(tmp_filepath)
321                     raise
322                 finally:
323                     self.upload_file.close()
324             os.rename(tmp_filepath, filepath)
325             return
326 
327         # The resource form only sets self.clear (via the input clear_upload)
328         # to True when an uploaded file is not replaced by another uploaded
329         # file, only if it is replaced by a link to file.
330         # If the uploaded file is replaced by a link, we should remove the
331         # previously uploaded file to clean up the file system.
332         if self.clear:
333             try:
334                 os.remove(filepath)
335             except OSError:
336                 pass
337
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -122,12 +122,17 @@
             return
         self.storage_path = os.path.join(path, 'storage',
                                          'uploads', object_type)
-        try:
-            os.makedirs(self.storage_path)
-        except OSError as e:
-            # errno 17 is file already exists
-            if e.errno != 17:
-                raise
+        # check if the storage directory is already created by
+        # the user or third-party
+        if os.path.isdir(self.storage_path):
+            pass
+        else:
+            try:
+                os.makedirs(self.storage_path)
+            except OSError as e:
+                # errno 17 is file already exists
+                if e.errno != 17:
+                    raise
         self.object_type = object_type
         self.old_filename = old_filename
         if old_filename:

{"golden_diff": "diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py\n--- a/ckan/lib/uploader.py\n+++ b/ckan/lib/uploader.py\n@@ -122,12 +122,17 @@\n             return\n         self.storage_path = os.path.join(path, 'storage',\n                                          'uploads', object_type)\n-        try:\n-            os.makedirs(self.storage_path)\n-        except OSError as e:\n-            # errno 17 is file already exists\n-            if e.errno != 17:\n-                raise\n+        # check if the storage directory is already created by\n+        # the user or third-party\n+        if os.path.isdir(self.storage_path):\n+            pass\n+        else:\n+            try:\n+                os.makedirs(self.storage_path)\n+            except OSError as e:\n+                # errno 17 is file already exists\n+                if e.errno != 17:\n+                    raise\n         self.object_type = object_type\n         self.old_filename = old_filename\n         if old_filename:\n", "issue": "Upload.__init__ relies too much on makedirs\n**CKAN version**\r\n\r\n2.8, 2.9\r\n\r\n**Describe the bug**\r\n\r\n`ckan.lib.uploader:Upload.__init__` relies on `os.makedirs` to create the storage directory. However, this is inadequate if, for example, the storage directory has been created by another process, with sufficient permissions for CKAN to use it, yet with insufficient permissions in the parent directory for CKAN to have created it.\r\n\r\nIt also adds extra OS calls on every creation of an Upload object.\r\n\n", "before_files": [{"content": "# encoding: utf-8\n\nimport os\nimport cgi\nimport datetime\nimport logging\nimport magic\nimport mimetypes\nfrom urllib.parse import urlparse\n\nfrom werkzeug.datastructures import FileStorage as FlaskFileStorage\n\nimport ckan.lib.munge as munge\nimport ckan.logic as logic\nimport ckan.plugins as plugins\nfrom ckan.common import config\n\nALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage)\nMB = 1 << 20\n\nlog = logging.getLogger(__name__)\n\n_storage_path = None\n_max_resource_size = None\n_max_image_size = None\n\n\ndef _copy_file(input_file, output_file, max_size):\n    input_file.seek(0)\n    current_size = 0\n    while True:\n        current_size = current_size + 1\n        # MB chunks\n        data = input_file.read(MB)\n\n        if not data:\n            break\n        output_file.write(data)\n        if current_size > max_size:\n            raise logic.ValidationError({'upload': ['File upload too large']})\n\n\ndef _get_underlying_file(wrapper):\n    if isinstance(wrapper, FlaskFileStorage):\n        return wrapper.stream\n    return wrapper.file\n\n\ndef get_uploader(upload_to, old_filename=None):\n    '''Query IUploader plugins and return an uploader instance for general\n    files.'''\n    upload = None\n    for plugin in plugins.PluginImplementations(plugins.IUploader):\n        upload = plugin.get_uploader(upload_to, old_filename)\n        if upload:\n            break\n\n    # default uploader\n    if upload is None:\n        upload = Upload(upload_to, old_filename)\n\n    return upload\n\n\ndef get_resource_uploader(data_dict):\n    '''Query IUploader plugins and return a resource uploader instance.'''\n    upload = None\n    for plugin in plugins.PluginImplementations(plugins.IUploader):\n        upload = plugin.get_resource_uploader(data_dict)\n        if upload:\n            break\n\n    # default uploader\n    if upload is None:\n        upload = ResourceUpload(data_dict)\n\n    return upload\n\n\ndef get_storage_path():\n    '''Function to cache storage path'''\n    global _storage_path\n\n    # None means it has not been set. False means not in config.\n    if _storage_path is None:\n        storage_path = config.get_value('ckan.storage_path')\n        if storage_path:\n            _storage_path = storage_path\n        else:\n            log.critical('''Please specify a ckan.storage_path in your config\n                         for your uploads''')\n            _storage_path = False\n\n    return _storage_path\n\n\ndef get_max_image_size():\n    global _max_image_size\n    if _max_image_size is None:\n        _max_image_size = config.get_value('ckan.max_image_size')\n    return _max_image_size\n\n\ndef get_max_resource_size():\n    global _max_resource_size\n    if _max_resource_size is None:\n        _max_resource_size = config.get_value('ckan.max_resource_size')\n    return _max_resource_size\n\n\nclass Upload(object):\n    def __init__(self, object_type, old_filename=None):\n        ''' Setup upload by creating a subdirectory of the storage directory\n        of name object_type. old_filename is the name of the file in the url\n        field last time'''\n\n        self.storage_path = None\n        self.filename = None\n        self.filepath = None\n        path = get_storage_path()\n        if not path:\n            return\n        self.storage_path = os.path.join(path, 'storage',\n                                         'uploads', object_type)\n        try:\n            os.makedirs(self.storage_path)\n        except OSError as e:\n            # errno 17 is file already exists\n            if e.errno != 17:\n                raise\n        self.object_type = object_type\n        self.old_filename = old_filename\n        if old_filename:\n            self.old_filepath = os.path.join(self.storage_path, old_filename)\n\n    def update_data_dict(self, data_dict, url_field, file_field, clear_field):\n        ''' Manipulate data from the data_dict.  url_field is the name of the\n        field where the upload is going to be. file_field is name of the key\n        where the FieldStorage is kept (i.e the field where the file data\n        actually is). clear_field is the name of a boolean field which\n        requests the upload to be deleted.  This needs to be called before\n        it reaches any validators'''\n\n        self.url = data_dict.get(url_field, '')\n        self.clear = data_dict.pop(clear_field, None)\n        self.file_field = file_field\n        self.upload_field_storage = data_dict.pop(file_field, None)\n\n        if not self.storage_path:\n            return\n\n        if isinstance(self.upload_field_storage, ALLOWED_UPLOAD_TYPES):\n            if self.upload_field_storage.filename:\n                self.filename = self.upload_field_storage.filename\n                self.filename = str(datetime.datetime.utcnow()) + self.filename\n                self.filename = munge.munge_filename_legacy(self.filename)\n                self.filepath = os.path.join(self.storage_path, self.filename)\n                data_dict[url_field] = self.filename\n                self.upload_file = _get_underlying_file(\n                    self.upload_field_storage)\n                self.tmp_filepath = self.filepath + '~'\n        # keep the file if there has been no change\n        elif self.old_filename and not self.old_filename.startswith('http'):\n            if not self.clear:\n                data_dict[url_field] = self.old_filename\n            if self.clear and self.url == self.old_filename:\n                data_dict[url_field] = ''\n\n    def upload(self, max_size=2):\n        ''' Actually upload the file.\n        This should happen just before a commit but after the data has\n        been validated and flushed to the db. This is so we do not store\n        anything unless the request is actually good.\n        max_size is size in MB maximum of the file'''\n\n        self.verify_type()\n\n        if self.filename:\n            with open(self.tmp_filepath, 'wb+') as output_file:\n                try:\n                    _copy_file(self.upload_file, output_file, max_size)\n                except logic.ValidationError:\n                    os.remove(self.tmp_filepath)\n                    raise\n                finally:\n                    self.upload_file.close()\n            os.rename(self.tmp_filepath, self.filepath)\n            self.clear = True\n\n        if (self.clear and self.old_filename\n                and not self.old_filename.startswith('http')):\n            try:\n                os.remove(self.old_filepath)\n            except OSError:\n                pass\n\n    def verify_type(self):\n        if not self.filename:\n            return\n\n        mimetypes = config.get_value(\n            f\"ckan.upload.{self.object_type}.mimetypes\")\n        types = config.get_value(f\"ckan.upload.{self.object_type}.types\")\n        if not mimetypes and not types:\n            return\n\n        actual = magic.from_buffer(self.upload_file.read(1024), mime=True)\n        self.upload_file.seek(0, os.SEEK_SET)\n        err = {self.file_field: [f\"Unsupported upload type: {actual}\"]}\n\n        if mimetypes and actual not in mimetypes:\n            raise logic.ValidationError(err)\n\n        type_ = actual.split(\"/\")[0]\n        if types and type_ not in types:\n            raise logic.ValidationError(err)\n\n\nclass ResourceUpload(object):\n    def __init__(self, resource):\n        path = get_storage_path()\n        config_mimetype_guess = config.get_value('ckan.mimetype_guess')\n\n        if not path:\n            self.storage_path = None\n            return\n        self.storage_path = os.path.join(path, 'resources')\n        try:\n            os.makedirs(self.storage_path)\n        except OSError as e:\n            # errno 17 is file already exists\n            if e.errno != 17:\n                raise\n        self.filename = None\n        self.mimetype = None\n\n        url = resource.get('url')\n\n        upload_field_storage = resource.pop('upload', None)\n        self.clear = resource.pop('clear_upload', None)\n\n        if url and config_mimetype_guess == 'file_ext' and urlparse(url).path:\n            self.mimetype = mimetypes.guess_type(url)[0]\n\n        if bool(upload_field_storage) and \\\n                isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):\n            self.filesize = 0  # bytes\n\n            self.filename = upload_field_storage.filename\n            self.filename = munge.munge_filename(self.filename)\n            resource['url'] = self.filename\n            resource['url_type'] = 'upload'\n            resource['last_modified'] = datetime.datetime.utcnow()\n            self.upload_file = _get_underlying_file(upload_field_storage)\n            self.upload_file.seek(0, os.SEEK_END)\n            self.filesize = self.upload_file.tell()\n            # go back to the beginning of the file buffer\n            self.upload_file.seek(0, os.SEEK_SET)\n\n            # check if the mimetype failed from guessing with the url\n            if not self.mimetype and config_mimetype_guess == 'file_ext':\n                self.mimetype = mimetypes.guess_type(self.filename)[0]\n\n            if not self.mimetype and config_mimetype_guess == 'file_contents':\n                try:\n                    self.mimetype = magic.from_buffer(self.upload_file.read(),\n                                                      mime=True)\n                    self.upload_file.seek(0, os.SEEK_SET)\n                except IOError:\n                    # Not that important if call above fails\n                    self.mimetype = None\n\n        elif self.clear:\n            resource['url_type'] = ''\n\n    def get_directory(self, id):\n        directory = os.path.join(self.storage_path,\n                                 id[0:3], id[3:6])\n        return directory\n\n    def get_path(self, id):\n        directory = self.get_directory(id)\n        filepath = os.path.join(directory, id[6:])\n        return filepath\n\n    def upload(self, id, max_size=10):\n        '''Actually upload the file.\n\n        :returns: ``'file uploaded'`` if a new file was successfully uploaded\n            (whether it overwrote a previously uploaded file or not),\n            ``'file deleted'`` if an existing uploaded file was deleted,\n            or ``None`` if nothing changed\n        :rtype: ``string`` or ``None``\n\n        '''\n        if not self.storage_path:\n            return\n\n        # Get directory and filepath on the system\n        # where the file for this resource will be stored\n        directory = self.get_directory(id)\n        filepath = self.get_path(id)\n\n        # If a filename has been provided (a file is being uploaded)\n        # we write it to the filepath (and overwrite it if it already\n        # exists). This way the uploaded file will always be stored\n        # in the same location\n        if self.filename:\n            try:\n                os.makedirs(directory)\n            except OSError as e:\n                # errno 17 is file already exists\n                if e.errno != 17:\n                    raise\n            tmp_filepath = filepath + '~'\n            with open(tmp_filepath, 'wb+') as output_file:\n                try:\n                    _copy_file(self.upload_file, output_file, max_size)\n                except logic.ValidationError:\n                    os.remove(tmp_filepath)\n                    raise\n                finally:\n                    self.upload_file.close()\n            os.rename(tmp_filepath, filepath)\n            return\n\n        # The resource form only sets self.clear (via the input clear_upload)\n        # to True when an uploaded file is not replaced by another uploaded\n        # file, only if it is replaced by a link to file.\n        # If the uploaded file is replaced by a link, we should remove the\n        # previously uploaded file to clean up the file system.\n        if self.clear:\n            try:\n                os.remove(filepath)\n            except OSError:\n                pass\n", "path": "ckan/lib/uploader.py"}], "after_files": [{"content": "# encoding: utf-8\n\nimport os\nimport cgi\nimport datetime\nimport logging\nimport magic\nimport mimetypes\nfrom urllib.parse import urlparse\n\nfrom werkzeug.datastructures import FileStorage as FlaskFileStorage\n\nimport ckan.lib.munge as munge\nimport ckan.logic as logic\nimport ckan.plugins as plugins\nfrom ckan.common import config\n\nALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage)\nMB = 1 << 20\n\nlog = logging.getLogger(__name__)\n\n_storage_path = None\n_max_resource_size = None\n_max_image_size = None\n\n\ndef _copy_file(input_file, output_file, max_size):\n    input_file.seek(0)\n    current_size = 0\n    while True:\n        current_size = current_size + 1\n        # MB chunks\n        data = input_file.read(MB)\n\n        if not data:\n            break\n        output_file.write(data)\n        if current_size > max_size:\n            raise logic.ValidationError({'upload': ['File upload too large']})\n\n\ndef _get_underlying_file(wrapper):\n    if isinstance(wrapper, FlaskFileStorage):\n        return wrapper.stream\n    return wrapper.file\n\n\ndef get_uploader(upload_to, old_filename=None):\n    '''Query IUploader plugins and return an uploader instance for general\n    files.'''\n    upload = None\n    for plugin in plugins.PluginImplementations(plugins.IUploader):\n        upload = plugin.get_uploader(upload_to, old_filename)\n        if upload:\n            break\n\n    # default uploader\n    if upload is None:\n        upload = Upload(upload_to, old_filename)\n\n    return upload\n\n\ndef get_resource_uploader(data_dict):\n    '''Query IUploader plugins and return a resource uploader instance.'''\n    upload = None\n    for plugin in plugins.PluginImplementations(plugins.IUploader):\n        upload = plugin.get_resource_uploader(data_dict)\n        if upload:\n            break\n\n    # default uploader\n    if upload is None:\n        upload = ResourceUpload(data_dict)\n\n    return upload\n\n\ndef get_storage_path():\n    '''Function to cache storage path'''\n    global _storage_path\n\n    # None means it has not been set. False means not in config.\n    if _storage_path is None:\n        storage_path = config.get_value('ckan.storage_path')\n        if storage_path:\n            _storage_path = storage_path\n        else:\n            log.critical('''Please specify a ckan.storage_path in your config\n                         for your uploads''')\n            _storage_path = False\n\n    return _storage_path\n\n\ndef get_max_image_size():\n    global _max_image_size\n    if _max_image_size is None:\n        _max_image_size = config.get_value('ckan.max_image_size')\n    return _max_image_size\n\n\ndef get_max_resource_size():\n    global _max_resource_size\n    if _max_resource_size is None:\n        _max_resource_size = config.get_value('ckan.max_resource_size')\n    return _max_resource_size\n\n\nclass Upload(object):\n    def __init__(self, object_type, old_filename=None):\n        ''' Setup upload by creating a subdirectory of the storage directory\n        of name object_type. old_filename is the name of the file in the url\n        field last time'''\n\n        self.storage_path = None\n        self.filename = None\n        self.filepath = None\n        path = get_storage_path()\n        if not path:\n            return\n        self.storage_path = os.path.join(path, 'storage',\n                                         'uploads', object_type)\n        # check if the storage directory is already created by\n        # the user or third-party\n        if os.path.isdir(self.storage_path):\n            pass\n        else:\n            try:\n                os.makedirs(self.storage_path)\n            except OSError as e:\n                # errno 17 is file already exists\n                if e.errno != 17:\n                    raise\n        self.object_type = object_type\n        self.old_filename = old_filename\n        if old_filename:\n            self.old_filepath = os.path.join(self.storage_path, old_filename)\n\n    def update_data_dict(self, data_dict, url_field, file_field, clear_field):\n        ''' Manipulate data from the data_dict.  url_field is the name of the\n        field where the upload is going to be. file_field is name of the key\n        where the FieldStorage is kept (i.e the field where the file data\n        actually is). clear_field is the name of a boolean field which\n        requests the upload to be deleted.  This needs to be called before\n        it reaches any validators'''\n\n        self.url = data_dict.get(url_field, '')\n        self.clear = data_dict.pop(clear_field, None)\n        self.file_field = file_field\n        self.upload_field_storage = data_dict.pop(file_field, None)\n\n        if not self.storage_path:\n            return\n\n        if isinstance(self.upload_field_storage, ALLOWED_UPLOAD_TYPES):\n            if self.upload_field_storage.filename:\n                self.filename = self.upload_field_storage.filename\n                self.filename = str(datetime.datetime.utcnow()) + self.filename\n                self.filename = munge.munge_filename_legacy(self.filename)\n                self.filepath = os.path.join(self.storage_path, self.filename)\n                data_dict[url_field] = self.filename\n                self.upload_file = _get_underlying_file(\n                    self.upload_field_storage)\n                self.tmp_filepath = self.filepath + '~'\n        # keep the file if there has been no change\n        elif self.old_filename and not self.old_filename.startswith('http'):\n            if not self.clear:\n                data_dict[url_field] = self.old_filename\n            if self.clear and self.url == self.old_filename:\n                data_dict[url_field] = ''\n\n    def upload(self, max_size=2):\n        ''' Actually upload the file.\n        This should happen just before a commit but after the data has\n        been validated and flushed to the db. This is so we do not store\n        anything unless the request is actually good.\n        max_size is size in MB maximum of the file'''\n\n        self.verify_type()\n\n        if self.filename:\n            with open(self.tmp_filepath, 'wb+') as output_file:\n                try:\n                    _copy_file(self.upload_file, output_file, max_size)\n                except logic.ValidationError:\n                    os.remove(self.tmp_filepath)\n                    raise\n                finally:\n                    self.upload_file.close()\n            os.rename(self.tmp_filepath, self.filepath)\n            self.clear = True\n\n        if (self.clear and self.old_filename\n                and not self.old_filename.startswith('http')):\n            try:\n                os.remove(self.old_filepath)\n            except OSError:\n                pass\n\n    def verify_type(self):\n        if not self.filename:\n            return\n\n        mimetypes = config.get_value(\n            f\"ckan.upload.{self.object_type}.mimetypes\")\n        types = config.get_value(f\"ckan.upload.{self.object_type}.types\")\n        if not mimetypes and not types:\n            return\n\n        actual = magic.from_buffer(self.upload_file.read(1024), mime=True)\n        self.upload_file.seek(0, os.SEEK_SET)\n        err = {self.file_field: [f\"Unsupported upload type: {actual}\"]}\n\n        if mimetypes and actual not in mimetypes:\n            raise logic.ValidationError(err)\n\n        type_ = actual.split(\"/\")[0]\n        if types and type_ not in types:\n            raise logic.ValidationError(err)\n\n\nclass ResourceUpload(object):\n    def __init__(self, resource):\n        path = get_storage_path()\n        config_mimetype_guess = config.get_value('ckan.mimetype_guess')\n\n        if not path:\n            self.storage_path = None\n            return\n        self.storage_path = os.path.join(path, 'resources')\n        try:\n            os.makedirs(self.storage_path)\n        except OSError as e:\n            # errno 17 is file already exists\n            if e.errno != 17:\n                raise\n        self.filename = None\n        self.mimetype = None\n\n        url = resource.get('url')\n\n        upload_field_storage = resource.pop('upload', None)\n        self.clear = resource.pop('clear_upload', None)\n\n        if url and config_mimetype_guess == 'file_ext' and urlparse(url).path:\n            self.mimetype = mimetypes.guess_type(url)[0]\n\n        if bool(upload_field_storage) and \\\n                isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):\n            self.filesize = 0  # bytes\n\n            self.filename = upload_field_storage.filename\n            self.filename = munge.munge_filename(self.filename)\n            resource['url'] = self.filename\n            resource['url_type'] = 'upload'\n            resource['last_modified'] = datetime.datetime.utcnow()\n            self.upload_file = _get_underlying_file(upload_field_storage)\n            self.upload_file.seek(0, os.SEEK_END)\n            self.filesize = self.upload_file.tell()\n            # go back to the beginning of the file buffer\n            self.upload_file.seek(0, os.SEEK_SET)\n\n            # check if the mimetype failed from guessing with the url\n            if not self.mimetype and config_mimetype_guess == 'file_ext':\n                self.mimetype = mimetypes.guess_type(self.filename)[0]\n\n            if not self.mimetype and config_mimetype_guess == 'file_contents':\n                try:\n                    self.mimetype = magic.from_buffer(self.upload_file.read(),\n                                                      mime=True)\n                    self.upload_file.seek(0, os.SEEK_SET)\n                except IOError:\n                    # Not that important if call above fails\n                    self.mimetype = None\n\n        elif self.clear:\n            resource['url_type'] = ''\n\n    def get_directory(self, id):\n        directory = os.path.join(self.storage_path,\n                                 id[0:3], id[3:6])\n        return directory\n\n    def get_path(self, id):\n        directory = self.get_directory(id)\n        filepath = os.path.join(directory, id[6:])\n        return filepath\n\n    def upload(self, id, max_size=10):\n        '''Actually upload the file.\n\n        :returns: ``'file uploaded'`` if a new file was successfully uploaded\n            (whether it overwrote a previously uploaded file or not),\n            ``'file deleted'`` if an existing uploaded file was deleted,\n            or ``None`` if nothing changed\n        :rtype: ``string`` or ``None``\n\n        '''\n        if not self.storage_path:\n            return\n\n        # Get directory and filepath on the system\n        # where the file for this resource will be stored\n        directory = self.get_directory(id)\n        filepath = self.get_path(id)\n\n        # If a filename has been provided (a file is being uploaded)\n        # we write it to the filepath (and overwrite it if it already\n        # exists). This way the uploaded file will always be stored\n        # in the same location\n        if self.filename:\n            try:\n                os.makedirs(directory)\n            except OSError as e:\n                # errno 17 is file already exists\n                if e.errno != 17:\n                    raise\n            tmp_filepath = filepath + '~'\n            with open(tmp_filepath, 'wb+') as output_file:\n                try:\n                    _copy_file(self.upload_file, output_file, max_size)\n                except logic.ValidationError:\n                    os.remove(tmp_filepath)\n                    raise\n                finally:\n                    self.upload_file.close()\n            os.rename(tmp_filepath, filepath)\n            return\n\n        # The resource form only sets self.clear (via the input clear_upload)\n        # to True when an uploaded file is not replaced by another uploaded\n        # file, only if it is replaced by a link to file.\n        # If the uploaded file is replaced by a link, we should remove the\n        # previously uploaded file to clean up the file system.\n        if self.clear:\n            try:\n                os.remove(filepath)\n            except OSError:\n                pass\n", "path": "ckan/lib/uploader.py"}]}