We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Uploading large files fails on the "Too many open files" error
Author: @lubosmj (lmjachky)


Redmine Issue: 9634, https://pulp.plan.io/issues/9634

---

Steps to reproduce the behaviour:
```
(pulp) [vagrant@pulp3-source-fedora34 pulp_ostree]$  truncate -s 3G gentoo_root.img

(pulp) [vagrant@pulp3-source-fedora34 pulp_ostree]$ pulp artifact upload --file gentoo_root.img
Uploading file gentoo_root.img
................[truncated the number of dots]...................Upload complete. Creating artifact.
Started background task /pulp/api/v3/tasks/2d3cf569-2d5c-449d-9ac6-7a53ab5f3a30/
........Error: Task /pulp/api/v3/tasks/2d3cf569-2d5c-449d-9ac6-7a53ab5f3a30/ failed: '[Errno 24] Too many open files: '/var/lib/pulp/media/upload/ec2d5c92-57e9-4191-ae95-ff4b9ec94353''

(pulp) [vagrant@pulp3-source-fedora34 pulp_ostree]$ ls -la gentoo_root.img
-rw-r--r--. 1 vagrant vagrant 3221225472 Dec 13 11:32 gentoo_root.img
```

Traceback:
```
Task 2d3cf569-2d5c-449d-9ac6-7a53ab5f3a30 failed ([Errno 24] Too many open files: '/var/lib/pulp/media/upload/ec2d5c92-57e9-4191-ae95-ff4b9ec94353')
pulp [3a3a9ea662994f609eea7d43ac8f30aa]: pulpcore.tasking.pulpcore_worker:INFO:   File "/home/vagrant/devel/pulpcore/pulpcore/tasking/pulpcore_worker.py", line 362, in _perform_task
    result = func(*args, **kwargs)
  File "/home/vagrant/devel/pulpcore/pulpcore/app/tasks/upload.py", line 31, in commit
    temp_file.write(chunk.file.read())
  File "/usr/local/lib/pulp/lib64/python3.9/site-packages/django/core/files/utils.py", line 42, in <lambda>
    read = property(lambda self: self.file.read)
  File "/usr/local/lib/pulp/lib64/python3.9/site-packages/django/db/models/fields/files.py", line 45, in _get_file
    self._file = self.storage.open(self.name, 'rb')
  File "/usr/local/lib/pulp/lib64/python3.9/site-packages/django/core/files/storage.py", line 38, in open
    return self._open(name, mode)
  File "/usr/local/lib/pulp/lib64/python3.9/site-packages/django/core/files/storage.py", line 238, in _open
    return File(open(self.path(name), mode))
```




--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pulpcore/app/importexport.py`
Content:
```
1 import os
2 import io
3 import json
4 import tarfile
5 import tempfile
6 
7 from django.conf import settings
8 
9 from pulpcore.app.apps import get_plugin_config
10 from pulpcore.app.models.progress import ProgressReport
11 from pulpcore.app.models.repository import Repository
12 from pulpcore.app.modelresource import (
13     ArtifactResource,
14     ContentArtifactResource,
15     RepositoryResource,
16 )
17 from pulpcore.constants import TASK_STATES
18 
19 
20 def _write_export(the_tarfile, resource, dest_dir=None):
21     """
22     Write the JSON export for the specified resource to the specified tarfile.
23 
24     The resulting file will be found at <dest_dir>/<resource.__class__.__name__>.json. If dest_dir
25     is None, the file will be added at the 'top level' of the_tarfile.
26 
27     Export-files are UTF-8 encoded.
28 
29     Args:
30         the_tarfile (tarfile.Tarfile): tarfile we are writing into
31         resource (import_export.resources.ModelResource): ModelResource to be exported
32         dest_dir str(directory-path): directory 'inside' the tarfile to write to
33     """
34     filename = "{}.{}.json".format(resource.__module__, type(resource).__name__)
35     dataset = resource.export(resource.queryset)
36     if dest_dir:
37         dest_filename = os.path.join(dest_dir, filename)
38     else:
39         dest_filename = filename
40 
41     data = dataset.json.encode("utf8")
42     info = tarfile.TarInfo(name=dest_filename)
43     info.size = len(data)
44     the_tarfile.addfile(info, io.BytesIO(data))
45 
46 
47 def export_versions(export, version_info):
48     """
49     Write a JSON list of plugins and their versions as 'versions.json' to export.tarfile
50 
51     Output format is [{"component": "<pluginname>", "version": "<pluginversion>"},...]
52 
53     Args:
54         export (django.db.models.PulpExport): export instance that's doing the export
55         version_info (set): set of (distribution-label,version) tuples for repos in this export
56     """
57     # build the version-list from the distributions for each component
58     versions = [{"component": label, "version": version} for (label, version) in version_info]
59 
60     version_json = json.dumps(versions).encode("utf8")
61     info = tarfile.TarInfo(name="versions.json")
62     info.size = len(version_json)
63     export.tarfile.addfile(info, io.BytesIO(version_json))
64 
65 
66 def export_artifacts(export, artifacts):
67     """
68     Export a set of Artifacts, ArtifactResources, and RepositoryResources
69 
70     Args:
71         export (django.db.models.PulpExport): export instance that's doing the export
72         artifacts (django.db.models.Artifacts): list of artifacts in all repos being exported
73 
74     Raises:
75         ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting
76     """
77     data = dict(message="Exporting Artifacts", code="export.artifacts", total=len(artifacts))
78     with ProgressReport(**data) as pb:
79         for artifact in pb.iter(artifacts):
80             dest = artifact.file.name
81             if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
82                 with tempfile.TemporaryDirectory() as temp_dir:
83                     with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:
84                         temp_file.write(artifact.file.read())
85                         temp_file.flush()
86                         export.tarfile.add(temp_file.name, dest)
87             else:
88                 export.tarfile.add(artifact.file.path, dest)
89 
90     resource = ArtifactResource()
91     resource.queryset = artifacts
92     _write_export(export.tarfile, resource)
93 
94     resource = RepositoryResource()
95     resource.queryset = Repository.objects.filter(pk__in=export.exporter.repositories.all())
96     _write_export(export.tarfile, resource)
97 
98 
99 def export_content(export, repository_version):
100     """
101     Export db-content, and the db-content of the owning repositories
102 
103     Args:
104         export (django.db.models.PulpExport): export instance that's doing the export
105         repository_version (django.db.models.RepositoryVersion): RepositoryVersion being exported
106     """
107 
108     def _combine_content_mappings(map1, map2):
109         """Combine two content mapping dicts into one by combining ids for for each key."""
110         result = {}
111         for key in map1.keys() | map2.keys():
112             result[key] = list(set(map1.get(key, []) + map2.get(key, [])))
113         return result
114 
115     dest_dir = os.path.join(
116         "repository-{}_{}".format(
117             str(repository_version.repository.name), repository_version.number
118         )
119     )
120 
121     # content mapping is used by repo versions with subrepos (eg distribution tree repos)
122     content_mapping = {}
123 
124     # find and export any ModelResource found in pulp_<repo-type>.app.modelresource
125     plugin_name = repository_version.repository.pulp_type.split(".")[0]
126     cfg = get_plugin_config(plugin_name)
127     if cfg.exportable_classes:
128         for cls in cfg.exportable_classes:
129             resource = cls(repository_version)
130             _write_export(export.tarfile, resource, dest_dir)
131 
132             if hasattr(resource, "content_mapping") and resource.content_mapping:
133                 content_mapping = _combine_content_mappings(
134                     content_mapping, resource.content_mapping
135                 )
136 
137     # Export the connection between content and artifacts
138     resource = ContentArtifactResource(repository_version, content_mapping)
139     _write_export(export.tarfile, resource, dest_dir)
140 
141     msg = (
142         f"Exporting content for {plugin_name} "
143         f"repository-version {repository_version.repository.name}/{repository_version.number}"
144     )
145     content_count = repository_version.content.count()
146     data = dict(
147         message=msg,
148         code="export.repo.version.content",
149         total=content_count,
150         done=content_count,
151         state=TASK_STATES.COMPLETED,
152     )
153     pb = ProgressReport(**data)
154     pb.save()
155 
156     if content_mapping:
157         # write the content mapping to tarfile
158         cm_json = json.dumps(content_mapping).encode("utf8")
159         info = tarfile.TarInfo(name=f"{dest_dir}/content_mapping.json")
160         info.size = len(cm_json)
161         export.tarfile.addfile(info, io.BytesIO(cm_json))
162
```

Path: `pulpcore/app/tasks/upload.py`
Content:
```
1 from gettext import gettext as _
2 from logging import getLogger
3 from tempfile import NamedTemporaryFile
4 
5 from django.core.files import File
6 
7 from pulpcore.app import files, models
8 from pulpcore.app.models import CreatedResource
9 from pulpcore.app.serializers import ArtifactSerializer
10 
11 log = getLogger(__name__)
12 
13 
14 def commit(upload_id, sha256):
15     """
16     Commit the upload and turn it into an artifact.
17 
18     Args:
19         upload_id (int): The upload primary key
20         sha256 (str): The checksum for the uploaded file
21     """
22     try:
23         upload = models.Upload.objects.get(pk=upload_id)
24     except models.Upload.DoesNotExist:
25         log.info(_("The upload was not found. Nothing to do."))
26         return
27 
28     chunks = models.UploadChunk.objects.filter(upload=upload).order_by("offset")
29     with NamedTemporaryFile("ab") as temp_file:
30         for chunk in chunks:
31             temp_file.write(chunk.file.read())
32         temp_file.flush()
33 
34         file = files.PulpTemporaryUploadedFile.from_file(File(open(temp_file.name, "rb")))
35 
36     data = {"file": file, "sha256": sha256}
37     serializer = ArtifactSerializer(data=data)
38     serializer.is_valid(raise_exception=True)
39     artifact = serializer.save()
40 
41     resource = CreatedResource(content_object=artifact)
42     resource.save()
43 
44     # delete the upload since it can't be reused to create another artifact
45     upload.delete()
46
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py
--- a/pulpcore/app/importexport.py
+++ b/pulpcore/app/importexport.py
@@ -83,6 +83,7 @@
                     with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:
                         temp_file.write(artifact.file.read())
                         temp_file.flush()
+                        artifact.file.close()
                         export.tarfile.add(temp_file.name, dest)
             else:
                 export.tarfile.add(artifact.file.path, dest)
diff --git a/pulpcore/app/tasks/upload.py b/pulpcore/app/tasks/upload.py
--- a/pulpcore/app/tasks/upload.py
+++ b/pulpcore/app/tasks/upload.py
@@ -29,6 +29,7 @@
     with NamedTemporaryFile("ab") as temp_file:
         for chunk in chunks:
             temp_file.write(chunk.file.read())
+            chunk.file.close()
         temp_file.flush()
 
         file = files.PulpTemporaryUploadedFile.from_file(File(open(temp_file.name, "rb")))

{"golden_diff": "diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py\n--- a/pulpcore/app/importexport.py\n+++ b/pulpcore/app/importexport.py\n@@ -83,6 +83,7 @@\n                     with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:\n                         temp_file.write(artifact.file.read())\n                         temp_file.flush()\n+                        artifact.file.close()\n                         export.tarfile.add(temp_file.name, dest)\n             else:\n                 export.tarfile.add(artifact.file.path, dest)\ndiff --git a/pulpcore/app/tasks/upload.py b/pulpcore/app/tasks/upload.py\n--- a/pulpcore/app/tasks/upload.py\n+++ b/pulpcore/app/tasks/upload.py\n@@ -29,6 +29,7 @@\n     with NamedTemporaryFile(\"ab\") as temp_file:\n         for chunk in chunks:\n             temp_file.write(chunk.file.read())\n+            chunk.file.close()\n         temp_file.flush()\n \n         file = files.PulpTemporaryUploadedFile.from_file(File(open(temp_file.name, \"rb\")))\n", "issue": "Uploading large files fails on the \"Too many open files\" error\nAuthor: @lubosmj (lmjachky)\n\n\nRedmine Issue: 9634, https://pulp.plan.io/issues/9634\n\n---\n\nSteps to reproduce the behaviour:\r\n```\r\n(pulp) [vagrant@pulp3-source-fedora34 pulp_ostree]$  truncate -s 3G gentoo_root.img\r\n\r\n(pulp) [vagrant@pulp3-source-fedora34 pulp_ostree]$ pulp artifact upload --file gentoo_root.img\r\nUploading file gentoo_root.img\r\n................[truncated the number of dots]...................Upload complete. Creating artifact.\r\nStarted background task /pulp/api/v3/tasks/2d3cf569-2d5c-449d-9ac6-7a53ab5f3a30/\r\n........Error: Task /pulp/api/v3/tasks/2d3cf569-2d5c-449d-9ac6-7a53ab5f3a30/ failed: '[Errno 24] Too many open files: '/var/lib/pulp/media/upload/ec2d5c92-57e9-4191-ae95-ff4b9ec94353''\r\n\r\n(pulp) [vagrant@pulp3-source-fedora34 pulp_ostree]$ ls -la gentoo_root.img\r\n-rw-r--r--. 1 vagrant vagrant 3221225472 Dec 13 11:32 gentoo_root.img\r\n```\r\n\r\nTraceback:\r\n```\r\nTask 2d3cf569-2d5c-449d-9ac6-7a53ab5f3a30 failed ([Errno 24] Too many open files: '/var/lib/pulp/media/upload/ec2d5c92-57e9-4191-ae95-ff4b9ec94353')\r\npulp [3a3a9ea662994f609eea7d43ac8f30aa]: pulpcore.tasking.pulpcore_worker:INFO:   File \"/home/vagrant/devel/pulpcore/pulpcore/tasking/pulpcore_worker.py\", line 362, in _perform_task\r\n    result = func(*args, **kwargs)\r\n  File \"/home/vagrant/devel/pulpcore/pulpcore/app/tasks/upload.py\", line 31, in commit\r\n    temp_file.write(chunk.file.read())\r\n  File \"/usr/local/lib/pulp/lib64/python3.9/site-packages/django/core/files/utils.py\", line 42, in <lambda>\r\n    read = property(lambda self: self.file.read)\r\n  File \"/usr/local/lib/pulp/lib64/python3.9/site-packages/django/db/models/fields/files.py\", line 45, in _get_file\r\n    self._file = self.storage.open(self.name, 'rb')\r\n  File \"/usr/local/lib/pulp/lib64/python3.9/site-packages/django/core/files/storage.py\", line 38, in open\r\n    return self._open(name, mode)\r\n  File \"/usr/local/lib/pulp/lib64/python3.9/site-packages/django/core/files/storage.py\", line 238, in _open\r\n    return File(open(self.path(name), mode))\r\n```\n\n\n\n", "before_files": [{"content": "import os\nimport io\nimport json\nimport tarfile\nimport tempfile\n\nfrom django.conf import settings\n\nfrom pulpcore.app.apps import get_plugin_config\nfrom pulpcore.app.models.progress import ProgressReport\nfrom pulpcore.app.models.repository import Repository\nfrom pulpcore.app.modelresource import (\n    ArtifactResource,\n    ContentArtifactResource,\n    RepositoryResource,\n)\nfrom pulpcore.constants import TASK_STATES\n\n\ndef _write_export(the_tarfile, resource, dest_dir=None):\n    \"\"\"\n    Write the JSON export for the specified resource to the specified tarfile.\n\n    The resulting file will be found at <dest_dir>/<resource.__class__.__name__>.json. If dest_dir\n    is None, the file will be added at the 'top level' of the_tarfile.\n\n    Export-files are UTF-8 encoded.\n\n    Args:\n        the_tarfile (tarfile.Tarfile): tarfile we are writing into\n        resource (import_export.resources.ModelResource): ModelResource to be exported\n        dest_dir str(directory-path): directory 'inside' the tarfile to write to\n    \"\"\"\n    filename = \"{}.{}.json\".format(resource.__module__, type(resource).__name__)\n    dataset = resource.export(resource.queryset)\n    if dest_dir:\n        dest_filename = os.path.join(dest_dir, filename)\n    else:\n        dest_filename = filename\n\n    data = dataset.json.encode(\"utf8\")\n    info = tarfile.TarInfo(name=dest_filename)\n    info.size = len(data)\n    the_tarfile.addfile(info, io.BytesIO(data))\n\n\ndef export_versions(export, version_info):\n    \"\"\"\n    Write a JSON list of plugins and their versions as 'versions.json' to export.tarfile\n\n    Output format is [{\"component\": \"<pluginname>\", \"version\": \"<pluginversion>\"},...]\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        version_info (set): set of (distribution-label,version) tuples for repos in this export\n    \"\"\"\n    # build the version-list from the distributions for each component\n    versions = [{\"component\": label, \"version\": version} for (label, version) in version_info]\n\n    version_json = json.dumps(versions).encode(\"utf8\")\n    info = tarfile.TarInfo(name=\"versions.json\")\n    info.size = len(version_json)\n    export.tarfile.addfile(info, io.BytesIO(version_json))\n\n\ndef export_artifacts(export, artifacts):\n    \"\"\"\n    Export a set of Artifacts, ArtifactResources, and RepositoryResources\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        artifacts (django.db.models.Artifacts): list of artifacts in all repos being exported\n\n    Raises:\n        ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting\n    \"\"\"\n    data = dict(message=\"Exporting Artifacts\", code=\"export.artifacts\", total=len(artifacts))\n    with ProgressReport(**data) as pb:\n        for artifact in pb.iter(artifacts):\n            dest = artifact.file.name\n            if settings.DEFAULT_FILE_STORAGE != \"pulpcore.app.models.storage.FileSystem\":\n                with tempfile.TemporaryDirectory() as temp_dir:\n                    with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:\n                        temp_file.write(artifact.file.read())\n                        temp_file.flush()\n                        export.tarfile.add(temp_file.name, dest)\n            else:\n                export.tarfile.add(artifact.file.path, dest)\n\n    resource = ArtifactResource()\n    resource.queryset = artifacts\n    _write_export(export.tarfile, resource)\n\n    resource = RepositoryResource()\n    resource.queryset = Repository.objects.filter(pk__in=export.exporter.repositories.all())\n    _write_export(export.tarfile, resource)\n\n\ndef export_content(export, repository_version):\n    \"\"\"\n    Export db-content, and the db-content of the owning repositories\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        repository_version (django.db.models.RepositoryVersion): RepositoryVersion being exported\n    \"\"\"\n\n    def _combine_content_mappings(map1, map2):\n        \"\"\"Combine two content mapping dicts into one by combining ids for for each key.\"\"\"\n        result = {}\n        for key in map1.keys() | map2.keys():\n            result[key] = list(set(map1.get(key, []) + map2.get(key, [])))\n        return result\n\n    dest_dir = os.path.join(\n        \"repository-{}_{}\".format(\n            str(repository_version.repository.name), repository_version.number\n        )\n    )\n\n    # content mapping is used by repo versions with subrepos (eg distribution tree repos)\n    content_mapping = {}\n\n    # find and export any ModelResource found in pulp_<repo-type>.app.modelresource\n    plugin_name = repository_version.repository.pulp_type.split(\".\")[0]\n    cfg = get_plugin_config(plugin_name)\n    if cfg.exportable_classes:\n        for cls in cfg.exportable_classes:\n            resource = cls(repository_version)\n            _write_export(export.tarfile, resource, dest_dir)\n\n            if hasattr(resource, \"content_mapping\") and resource.content_mapping:\n                content_mapping = _combine_content_mappings(\n                    content_mapping, resource.content_mapping\n                )\n\n    # Export the connection between content and artifacts\n    resource = ContentArtifactResource(repository_version, content_mapping)\n    _write_export(export.tarfile, resource, dest_dir)\n\n    msg = (\n        f\"Exporting content for {plugin_name} \"\n        f\"repository-version {repository_version.repository.name}/{repository_version.number}\"\n    )\n    content_count = repository_version.content.count()\n    data = dict(\n        message=msg,\n        code=\"export.repo.version.content\",\n        total=content_count,\n        done=content_count,\n        state=TASK_STATES.COMPLETED,\n    )\n    pb = ProgressReport(**data)\n    pb.save()\n\n    if content_mapping:\n        # write the content mapping to tarfile\n        cm_json = json.dumps(content_mapping).encode(\"utf8\")\n        info = tarfile.TarInfo(name=f\"{dest_dir}/content_mapping.json\")\n        info.size = len(cm_json)\n        export.tarfile.addfile(info, io.BytesIO(cm_json))\n", "path": "pulpcore/app/importexport.py"}, {"content": "from gettext import gettext as _\nfrom logging import getLogger\nfrom tempfile import NamedTemporaryFile\n\nfrom django.core.files import File\n\nfrom pulpcore.app import files, models\nfrom pulpcore.app.models import CreatedResource\nfrom pulpcore.app.serializers import ArtifactSerializer\n\nlog = getLogger(__name__)\n\n\ndef commit(upload_id, sha256):\n    \"\"\"\n    Commit the upload and turn it into an artifact.\n\n    Args:\n        upload_id (int): The upload primary key\n        sha256 (str): The checksum for the uploaded file\n    \"\"\"\n    try:\n        upload = models.Upload.objects.get(pk=upload_id)\n    except models.Upload.DoesNotExist:\n        log.info(_(\"The upload was not found. Nothing to do.\"))\n        return\n\n    chunks = models.UploadChunk.objects.filter(upload=upload).order_by(\"offset\")\n    with NamedTemporaryFile(\"ab\") as temp_file:\n        for chunk in chunks:\n            temp_file.write(chunk.file.read())\n        temp_file.flush()\n\n        file = files.PulpTemporaryUploadedFile.from_file(File(open(temp_file.name, \"rb\")))\n\n    data = {\"file\": file, \"sha256\": sha256}\n    serializer = ArtifactSerializer(data=data)\n    serializer.is_valid(raise_exception=True)\n    artifact = serializer.save()\n\n    resource = CreatedResource(content_object=artifact)\n    resource.save()\n\n    # delete the upload since it can't be reused to create another artifact\n    upload.delete()\n", "path": "pulpcore/app/tasks/upload.py"}], "after_files": [{"content": "import os\nimport io\nimport json\nimport tarfile\nimport tempfile\n\nfrom django.conf import settings\n\nfrom pulpcore.app.apps import get_plugin_config\nfrom pulpcore.app.models.progress import ProgressReport\nfrom pulpcore.app.models.repository import Repository\nfrom pulpcore.app.modelresource import (\n    ArtifactResource,\n    ContentArtifactResource,\n    RepositoryResource,\n)\nfrom pulpcore.constants import TASK_STATES\n\n\ndef _write_export(the_tarfile, resource, dest_dir=None):\n    \"\"\"\n    Write the JSON export for the specified resource to the specified tarfile.\n\n    The resulting file will be found at <dest_dir>/<resource.__class__.__name__>.json. If dest_dir\n    is None, the file will be added at the 'top level' of the_tarfile.\n\n    Export-files are UTF-8 encoded.\n\n    Args:\n        the_tarfile (tarfile.Tarfile): tarfile we are writing into\n        resource (import_export.resources.ModelResource): ModelResource to be exported\n        dest_dir str(directory-path): directory 'inside' the tarfile to write to\n    \"\"\"\n    filename = \"{}.{}.json\".format(resource.__module__, type(resource).__name__)\n    dataset = resource.export(resource.queryset)\n    if dest_dir:\n        dest_filename = os.path.join(dest_dir, filename)\n    else:\n        dest_filename = filename\n\n    data = dataset.json.encode(\"utf8\")\n    info = tarfile.TarInfo(name=dest_filename)\n    info.size = len(data)\n    the_tarfile.addfile(info, io.BytesIO(data))\n\n\ndef export_versions(export, version_info):\n    \"\"\"\n    Write a JSON list of plugins and their versions as 'versions.json' to export.tarfile\n\n    Output format is [{\"component\": \"<pluginname>\", \"version\": \"<pluginversion>\"},...]\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        version_info (set): set of (distribution-label,version) tuples for repos in this export\n    \"\"\"\n    # build the version-list from the distributions for each component\n    versions = [{\"component\": label, \"version\": version} for (label, version) in version_info]\n\n    version_json = json.dumps(versions).encode(\"utf8\")\n    info = tarfile.TarInfo(name=\"versions.json\")\n    info.size = len(version_json)\n    export.tarfile.addfile(info, io.BytesIO(version_json))\n\n\ndef export_artifacts(export, artifacts):\n    \"\"\"\n    Export a set of Artifacts, ArtifactResources, and RepositoryResources\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        artifacts (django.db.models.Artifacts): list of artifacts in all repos being exported\n\n    Raises:\n        ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting\n    \"\"\"\n    data = dict(message=\"Exporting Artifacts\", code=\"export.artifacts\", total=len(artifacts))\n    with ProgressReport(**data) as pb:\n        for artifact in pb.iter(artifacts):\n            dest = artifact.file.name\n            if settings.DEFAULT_FILE_STORAGE != \"pulpcore.app.models.storage.FileSystem\":\n                with tempfile.TemporaryDirectory() as temp_dir:\n                    with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:\n                        temp_file.write(artifact.file.read())\n                        temp_file.flush()\n                        artifact.file.close()\n                        export.tarfile.add(temp_file.name, dest)\n            else:\n                export.tarfile.add(artifact.file.path, dest)\n\n    resource = ArtifactResource()\n    resource.queryset = artifacts\n    _write_export(export.tarfile, resource)\n\n    resource = RepositoryResource()\n    resource.queryset = Repository.objects.filter(pk__in=export.exporter.repositories.all())\n    _write_export(export.tarfile, resource)\n\n\ndef export_content(export, repository_version):\n    \"\"\"\n    Export db-content, and the db-content of the owning repositories\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        repository_version (django.db.models.RepositoryVersion): RepositoryVersion being exported\n    \"\"\"\n\n    def _combine_content_mappings(map1, map2):\n        \"\"\"Combine two content mapping dicts into one by combining ids for for each key.\"\"\"\n        result = {}\n        for key in map1.keys() | map2.keys():\n            result[key] = list(set(map1.get(key, []) + map2.get(key, [])))\n        return result\n\n    dest_dir = os.path.join(\n        \"repository-{}_{}\".format(\n            str(repository_version.repository.name), repository_version.number\n        )\n    )\n\n    # content mapping is used by repo versions with subrepos (eg distribution tree repos)\n    content_mapping = {}\n\n    # find and export any ModelResource found in pulp_<repo-type>.app.modelresource\n    plugin_name = repository_version.repository.pulp_type.split(\".\")[0]\n    cfg = get_plugin_config(plugin_name)\n    if cfg.exportable_classes:\n        for cls in cfg.exportable_classes:\n            resource = cls(repository_version)\n            _write_export(export.tarfile, resource, dest_dir)\n\n            if hasattr(resource, \"content_mapping\") and resource.content_mapping:\n                content_mapping = _combine_content_mappings(\n                    content_mapping, resource.content_mapping\n                )\n\n    # Export the connection between content and artifacts\n    resource = ContentArtifactResource(repository_version, content_mapping)\n    _write_export(export.tarfile, resource, dest_dir)\n\n    msg = (\n        f\"Exporting content for {plugin_name} \"\n        f\"repository-version {repository_version.repository.name}/{repository_version.number}\"\n    )\n    content_count = repository_version.content.count()\n    data = dict(\n        message=msg,\n        code=\"export.repo.version.content\",\n        total=content_count,\n        done=content_count,\n        state=TASK_STATES.COMPLETED,\n    )\n    pb = ProgressReport(**data)\n    pb.save()\n\n    if content_mapping:\n        # write the content mapping to tarfile\n        cm_json = json.dumps(content_mapping).encode(\"utf8\")\n        info = tarfile.TarInfo(name=f\"{dest_dir}/content_mapping.json\")\n        info.size = len(cm_json)\n        export.tarfile.addfile(info, io.BytesIO(cm_json))\n", "path": "pulpcore/app/importexport.py"}, {"content": "from gettext import gettext as _\nfrom logging import getLogger\nfrom tempfile import NamedTemporaryFile\n\nfrom django.core.files import File\n\nfrom pulpcore.app import files, models\nfrom pulpcore.app.models import CreatedResource\nfrom pulpcore.app.serializers import ArtifactSerializer\n\nlog = getLogger(__name__)\n\n\ndef commit(upload_id, sha256):\n    \"\"\"\n    Commit the upload and turn it into an artifact.\n\n    Args:\n        upload_id (int): The upload primary key\n        sha256 (str): The checksum for the uploaded file\n    \"\"\"\n    try:\n        upload = models.Upload.objects.get(pk=upload_id)\n    except models.Upload.DoesNotExist:\n        log.info(_(\"The upload was not found. Nothing to do.\"))\n        return\n\n    chunks = models.UploadChunk.objects.filter(upload=upload).order_by(\"offset\")\n    with NamedTemporaryFile(\"ab\") as temp_file:\n        for chunk in chunks:\n            temp_file.write(chunk.file.read())\n            chunk.file.close()\n        temp_file.flush()\n\n        file = files.PulpTemporaryUploadedFile.from_file(File(open(temp_file.name, \"rb\")))\n\n    data = {\"file\": file, \"sha256\": sha256}\n    serializer = ArtifactSerializer(data=data)\n    serializer.is_valid(raise_exception=True)\n    artifact = serializer.save()\n\n    resource = CreatedResource(content_object=artifact)\n    resource.save()\n\n    # delete the upload since it can't be reused to create another artifact\n    upload.delete()\n", "path": "pulpcore/app/tasks/upload.py"}]}