We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
PulpImport/Export of kickstart repos with subrepos broken
See https://bugzilla.redhat.com/show_bug.cgi?id=2040870 for details.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pulpcore/app/importexport.py`
Content:
```
1 import os
2 import io
3 import json
4 import tarfile
5 import tempfile
6 import logging
7 
8 from django.conf import settings
9 from django.db.models.query import QuerySet
10 
11 from pulpcore.app.apps import get_plugin_config
12 from pulpcore.app.models.progress import ProgressReport
13 from pulpcore.app.models.repository import Repository
14 from pulpcore.app.modelresource import (
15     ArtifactResource,
16     ContentArtifactResource,
17     RepositoryResource,
18 )
19 from pulpcore.constants import TASK_STATES, EXPORT_BATCH_SIZE
20 
21 log = logging.getLogger(__name__)
22 
23 
24 def _write_export(the_tarfile, resource, dest_dir=None):
25     """
26     Write the JSON export for the specified resource to the specified tarfile.
27 
28     The resulting file will be found at <dest_dir>/<resource.__class__.__name__>.json. If dest_dir
29     is None, the file will be added at the 'top level' of the_tarfile.
30 
31     Export-files are UTF-8 encoded.
32 
33     Args:
34         the_tarfile (tarfile.Tarfile): tarfile we are writing into
35         resource (import_export.resources.ModelResource): ModelResource to be exported
36         dest_dir str(directory-path): directory 'inside' the tarfile to write to
37     """
38     filename = "{}.{}.json".format(resource.__module__, type(resource).__name__)
39     if dest_dir:
40         dest_filename = os.path.join(dest_dir, filename)
41     else:
42         dest_filename = filename
43 
44     # If the resource is the type of QuerySet, then export the data in batch to save memory.
45     # Otherwise, export all data in oneshot. This is because the underlying libraries
46     # (json; django-import-export) do not support to stream the output to file, we export
47     # the data in batches to memory and concatenate the json lists via string manipulation.
48     with tempfile.NamedTemporaryFile(dir=os.getcwd(), mode="w", encoding="utf8") as temp_file:
49         if isinstance(resource.queryset, QuerySet):
50             temp_file.write("[")
51             total = resource.queryset.count()
52             for i in range(0, total, EXPORT_BATCH_SIZE):
53                 current_batch = i + EXPORT_BATCH_SIZE
54                 dataset = resource.export(resource.queryset[i:current_batch])
55                 # Strip "[" and "]" as we are writing the dataset in batch
56                 temp_file.write(dataset.json.lstrip("[").rstrip("]"))
57                 if current_batch < total:
58                     # Write "," if not last loop
59                     temp_file.write(", ")
60             temp_file.write("]")
61         else:
62             dataset = resource.export(resource.queryset)
63             temp_file.write(dataset.json)
64 
65         temp_file.flush()
66         info = tarfile.TarInfo(name=dest_filename)
67         info.size = os.path.getsize(temp_file.name)
68         with open(temp_file.name, "rb") as fd:
69             the_tarfile.addfile(info, fd)
70 
71 
72 def export_versions(export, version_info):
73     """
74     Write a JSON list of plugins and their versions as 'versions.json' to export.tarfile
75 
76     Output format is [{"component": "<pluginname>", "version": "<pluginversion>"},...]
77 
78     Args:
79         export (django.db.models.PulpExport): export instance that's doing the export
80         version_info (set): set of (distribution-label,version) tuples for repos in this export
81     """
82     # build the version-list from the distributions for each component
83     versions = [{"component": label, "version": version} for (label, version) in version_info]
84 
85     version_json = json.dumps(versions).encode("utf8")
86     info = tarfile.TarInfo(name="versions.json")
87     info.size = len(version_json)
88     export.tarfile.addfile(info, io.BytesIO(version_json))
89 
90 
91 def export_artifacts(export, artifacts):
92     """
93     Export a set of Artifacts, ArtifactResources, and RepositoryResources
94 
95     Args:
96         export (django.db.models.PulpExport): export instance that's doing the export
97         artifacts (django.db.models.Artifacts): list of artifacts in all repos being exported
98 
99     Raises:
100         ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting
101     """
102     data = dict(message="Exporting Artifacts", code="export.artifacts", total=len(artifacts))
103     with ProgressReport(**data) as pb:
104         for artifact in pb.iter(artifacts):
105             dest = artifact.file.name
106             if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
107                 with tempfile.TemporaryDirectory() as temp_dir:
108                     with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:
109                         temp_file.write(artifact.file.read())
110                         temp_file.flush()
111                         artifact.file.close()
112                         export.tarfile.add(temp_file.name, dest)
113             else:
114                 export.tarfile.add(artifact.file.path, dest)
115 
116     resource = ArtifactResource()
117     resource.queryset = artifacts
118     _write_export(export.tarfile, resource)
119 
120     resource = RepositoryResource()
121     resource.queryset = Repository.objects.filter(pk__in=export.exporter.repositories.all())
122     _write_export(export.tarfile, resource)
123 
124 
125 def export_content(export, repository_version):
126     """
127     Export db-content, and the db-content of the owning repositories
128 
129     Args:
130         export (django.db.models.PulpExport): export instance that's doing the export
131         repository_version (django.db.models.RepositoryVersion): RepositoryVersion being exported
132     """
133 
134     def _combine_content_mappings(map1, map2):
135         """Combine two content mapping dicts into one by combining ids for for each key."""
136         result = {}
137         for key in map1.keys() | map2.keys():
138             result[key] = list(set(map1.get(key, []) + map2.get(key, [])))
139         return result
140 
141     dest_dir = os.path.join(
142         "repository-{}_{}".format(
143             str(repository_version.repository.name), repository_version.number
144         )
145     )
146 
147     # Export the connection between content and artifacts
148     resource = ContentArtifactResource(repository_version)
149     _write_export(export.tarfile, resource, dest_dir)
150 
151     # content mapping is used by repo versions with subrepos (eg distribution tree repos)
152     content_mapping = {}
153 
154     # find and export any ModelResource found in pulp_<repo-type>.app.modelresource
155     plugin_name = repository_version.repository.pulp_type.split(".")[0]
156     cfg = get_plugin_config(plugin_name)
157     if cfg.exportable_classes:
158         for cls in cfg.exportable_classes:
159             resource = cls(repository_version)
160             _write_export(export.tarfile, resource, dest_dir)
161 
162             if hasattr(resource, "content_mapping") and resource.content_mapping:
163                 content_mapping = _combine_content_mappings(
164                     content_mapping, resource.content_mapping
165                 )
166 
167     msg = (
168         f"Exporting content for {plugin_name} "
169         f"repository-version {repository_version.repository.name}/{repository_version.number}"
170     )
171     content_count = repository_version.content.count()
172     data = dict(
173         message=msg,
174         code="export.repo.version.content",
175         total=content_count,
176         done=content_count,
177         state=TASK_STATES.COMPLETED,
178     )
179     pb = ProgressReport(**data)
180     pb.save()
181 
182     if content_mapping:
183         # write the content mapping to tarfile
184         cm_json = json.dumps(content_mapping).encode("utf8")
185         info = tarfile.TarInfo(name=f"{dest_dir}/content_mapping.json")
186         info.size = len(cm_json)
187         export.tarfile.addfile(info, io.BytesIO(cm_json))
188
```

Path: `pulpcore/app/modelresource.py`
Content:
```
1 from import_export import fields
2 from import_export.widgets import ForeignKeyWidget
3 from logging import getLogger
4 
5 from pulpcore.app.models.content import (
6     Artifact,
7     Content,
8     ContentArtifact,
9 )
10 from pulpcore.app.models.repository import Repository
11 from pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS
12 from pulpcore.plugin.importexport import QueryModelResource
13 
14 
15 log = getLogger(__name__)
16 
17 
18 #
19 # Artifact and Repository are different from other import-export entities, in that they are not
20 # repo-version-specific.
21 #
22 class ArtifactResource(QueryModelResource):
23     """Resource for import/export of artifacts."""
24 
25     def before_import_row(self, row, **kwargs):
26         """
27         Sets digests to None if they are blank strings.
28 
29         Args:
30             row (tablib.Dataset row): incoming import-row representing a single Variant.
31             kwargs: args passed along from the import() call.
32 
33         """
34         # the export converts None to blank strings but sha384 and sha512 have unique constraints
35         # that get triggered if they are blank. convert checksums back into None if they are blank.
36         for checksum in ALL_KNOWN_CONTENT_CHECKSUMS:
37             if row[checksum] == "":
38                 row[checksum] = None
39 
40     class Meta:
41         model = Artifact
42         exclude = (
43             "pulp_id",
44             "pulp_created",
45             "pulp_last_updated",
46         )
47         import_id_fields = ("sha256",)
48 
49 
50 class RepositoryResource(QueryModelResource):
51     class Meta:
52         model = Repository
53         import_id_fields = ("name",)
54         exclude = (
55             "pulp_id",
56             "pulp_created",
57             "pulp_last_updated",
58             "content",
59         )
60 
61 
62 class ContentArtifactResource(QueryModelResource):
63     """
64     Handles import/export of the ContentArtifact model.
65 
66     ContentArtifact is different from other import-export entities because it has no 'natural key'
67     other than a pulp_id, which aren't shared across instances. We do some magic to link up
68     ContentArtifacts to their matching (already-imported) Content.
69     """
70 
71     artifact = fields.Field(
72         column_name="artifact", attribute="artifact", widget=ForeignKeyWidget(Artifact, "sha256")
73     )
74 
75     def before_import_row(self, row, **kwargs):
76         """
77         Fixes the content-ptr of an incoming content-artifact row at import time.
78 
79         Finds the 'original uuid' of the Content for this row, looks it up as the
80         'upstream_id' of imported Content, and then replaces the Content-pk with its
81         (new) uuid.
82 
83         Args:
84             row (tablib.Dataset row): incoming import-row representing a single ContentArtifact.
85             kwargs: args passed along from the import() call.
86 
87         Returns:
88             (tablib.Dataset row): row that now points to the new downstream uuid for its content.
89         """
90 
91         linked_content = Content.objects.get(upstream_id=row["content"])
92         row["content"] = str(linked_content.pulp_id)
93 
94     def set_up_queryset(self):
95         return ContentArtifact.objects.filter(content__in=self.repo_version.content).order_by(
96             "content", "relative_path"
97         )
98 
99     class Meta:
100         model = ContentArtifact
101         import_id_fields = (
102             "content",
103             "relative_path",
104         )
105         exclude = (
106             "pulp_created",
107             "pulp_last_updated",
108             "_artifacts",
109             "pulp_id",
110         )
111
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py
--- a/pulpcore/app/importexport.py
+++ b/pulpcore/app/importexport.py
@@ -144,10 +144,6 @@
         )
     )
 
-    # Export the connection between content and artifacts
-    resource = ContentArtifactResource(repository_version)
-    _write_export(export.tarfile, resource, dest_dir)
-
     # content mapping is used by repo versions with subrepos (eg distribution tree repos)
     content_mapping = {}
 
@@ -164,6 +160,10 @@
                     content_mapping, resource.content_mapping
                 )
 
+    # Export the connection between content and artifacts
+    resource = ContentArtifactResource(repository_version, content_mapping)
+    _write_export(export.tarfile, resource, dest_dir)
+
     msg = (
         f"Exporting content for {plugin_name} "
         f"repository-version {repository_version.repository.name}/{repository_version.number}"
diff --git a/pulpcore/app/modelresource.py b/pulpcore/app/modelresource.py
--- a/pulpcore/app/modelresource.py
+++ b/pulpcore/app/modelresource.py
@@ -66,12 +66,19 @@
     ContentArtifact is different from other import-export entities because it has no 'natural key'
     other than a pulp_id, which aren't shared across instances. We do some magic to link up
     ContentArtifacts to their matching (already-imported) Content.
+
+    Some plugin-models have sub-repositories. We take advantage of the content-mapping
+    machinery to account for those contentartifacts as well.
     """
 
     artifact = fields.Field(
         column_name="artifact", attribute="artifact", widget=ForeignKeyWidget(Artifact, "sha256")
     )
 
+    def __init__(self, repo_version=None, content_mapping=None):
+        self.content_mapping = content_mapping
+        super().__init__(repo_version)
+
     def before_import_row(self, row, **kwargs):
         """
         Fixes the content-ptr of an incoming content-artifact row at import time.
@@ -92,9 +99,15 @@
         row["content"] = str(linked_content.pulp_id)
 
     def set_up_queryset(self):
-        return ContentArtifact.objects.filter(content__in=self.repo_version.content).order_by(
-            "content", "relative_path"
-        )
+        vers_content = ContentArtifact.objects.filter(content__in=self.repo_version.content)
+        if self.content_mapping:
+            all_content = []
+            for content_ids in self.content_mapping.values():
+                all_content.extend(content_ids)
+            vers_content = vers_content.union(
+                ContentArtifact.objects.filter(content__in=all_content)
+            )
+        return vers_content.order_by("content", "relative_path")
 
     class Meta:
         model = ContentArtifact

{"golden_diff": "diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py\n--- a/pulpcore/app/importexport.py\n+++ b/pulpcore/app/importexport.py\n@@ -144,10 +144,6 @@\n         )\n     )\n \n-    # Export the connection between content and artifacts\n-    resource = ContentArtifactResource(repository_version)\n-    _write_export(export.tarfile, resource, dest_dir)\n-\n     # content mapping is used by repo versions with subrepos (eg distribution tree repos)\n     content_mapping = {}\n \n@@ -164,6 +160,10 @@\n                     content_mapping, resource.content_mapping\n                 )\n \n+    # Export the connection between content and artifacts\n+    resource = ContentArtifactResource(repository_version, content_mapping)\n+    _write_export(export.tarfile, resource, dest_dir)\n+\n     msg = (\n         f\"Exporting content for {plugin_name} \"\n         f\"repository-version {repository_version.repository.name}/{repository_version.number}\"\ndiff --git a/pulpcore/app/modelresource.py b/pulpcore/app/modelresource.py\n--- a/pulpcore/app/modelresource.py\n+++ b/pulpcore/app/modelresource.py\n@@ -66,12 +66,19 @@\n     ContentArtifact is different from other import-export entities because it has no 'natural key'\n     other than a pulp_id, which aren't shared across instances. We do some magic to link up\n     ContentArtifacts to their matching (already-imported) Content.\n+\n+    Some plugin-models have sub-repositories. We take advantage of the content-mapping\n+    machinery to account for those contentartifacts as well.\n     \"\"\"\n \n     artifact = fields.Field(\n         column_name=\"artifact\", attribute=\"artifact\", widget=ForeignKeyWidget(Artifact, \"sha256\")\n     )\n \n+    def __init__(self, repo_version=None, content_mapping=None):\n+        self.content_mapping = content_mapping\n+        super().__init__(repo_version)\n+\n     def before_import_row(self, row, **kwargs):\n         \"\"\"\n         Fixes the content-ptr of an incoming content-artifact row at import time.\n@@ -92,9 +99,15 @@\n         row[\"content\"] = str(linked_content.pulp_id)\n \n     def set_up_queryset(self):\n-        return ContentArtifact.objects.filter(content__in=self.repo_version.content).order_by(\n-            \"content\", \"relative_path\"\n-        )\n+        vers_content = ContentArtifact.objects.filter(content__in=self.repo_version.content)\n+        if self.content_mapping:\n+            all_content = []\n+            for content_ids in self.content_mapping.values():\n+                all_content.extend(content_ids)\n+            vers_content = vers_content.union(\n+                ContentArtifact.objects.filter(content__in=all_content)\n+            )\n+        return vers_content.order_by(\"content\", \"relative_path\")\n \n     class Meta:\n         model = ContentArtifact\n", "issue": "PulpImport/Export of kickstart repos with subrepos broken\nSee https://bugzilla.redhat.com/show_bug.cgi?id=2040870 for details.\n", "before_files": [{"content": "import os\nimport io\nimport json\nimport tarfile\nimport tempfile\nimport logging\n\nfrom django.conf import settings\nfrom django.db.models.query import QuerySet\n\nfrom pulpcore.app.apps import get_plugin_config\nfrom pulpcore.app.models.progress import ProgressReport\nfrom pulpcore.app.models.repository import Repository\nfrom pulpcore.app.modelresource import (\n    ArtifactResource,\n    ContentArtifactResource,\n    RepositoryResource,\n)\nfrom pulpcore.constants import TASK_STATES, EXPORT_BATCH_SIZE\n\nlog = logging.getLogger(__name__)\n\n\ndef _write_export(the_tarfile, resource, dest_dir=None):\n    \"\"\"\n    Write the JSON export for the specified resource to the specified tarfile.\n\n    The resulting file will be found at <dest_dir>/<resource.__class__.__name__>.json. If dest_dir\n    is None, the file will be added at the 'top level' of the_tarfile.\n\n    Export-files are UTF-8 encoded.\n\n    Args:\n        the_tarfile (tarfile.Tarfile): tarfile we are writing into\n        resource (import_export.resources.ModelResource): ModelResource to be exported\n        dest_dir str(directory-path): directory 'inside' the tarfile to write to\n    \"\"\"\n    filename = \"{}.{}.json\".format(resource.__module__, type(resource).__name__)\n    if dest_dir:\n        dest_filename = os.path.join(dest_dir, filename)\n    else:\n        dest_filename = filename\n\n    # If the resource is the type of QuerySet, then export the data in batch to save memory.\n    # Otherwise, export all data in oneshot. This is because the underlying libraries\n    # (json; django-import-export) do not support to stream the output to file, we export\n    # the data in batches to memory and concatenate the json lists via string manipulation.\n    with tempfile.NamedTemporaryFile(dir=os.getcwd(), mode=\"w\", encoding=\"utf8\") as temp_file:\n        if isinstance(resource.queryset, QuerySet):\n            temp_file.write(\"[\")\n            total = resource.queryset.count()\n            for i in range(0, total, EXPORT_BATCH_SIZE):\n                current_batch = i + EXPORT_BATCH_SIZE\n                dataset = resource.export(resource.queryset[i:current_batch])\n                # Strip \"[\" and \"]\" as we are writing the dataset in batch\n                temp_file.write(dataset.json.lstrip(\"[\").rstrip(\"]\"))\n                if current_batch < total:\n                    # Write \",\" if not last loop\n                    temp_file.write(\", \")\n            temp_file.write(\"]\")\n        else:\n            dataset = resource.export(resource.queryset)\n            temp_file.write(dataset.json)\n\n        temp_file.flush()\n        info = tarfile.TarInfo(name=dest_filename)\n        info.size = os.path.getsize(temp_file.name)\n        with open(temp_file.name, \"rb\") as fd:\n            the_tarfile.addfile(info, fd)\n\n\ndef export_versions(export, version_info):\n    \"\"\"\n    Write a JSON list of plugins and their versions as 'versions.json' to export.tarfile\n\n    Output format is [{\"component\": \"<pluginname>\", \"version\": \"<pluginversion>\"},...]\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        version_info (set): set of (distribution-label,version) tuples for repos in this export\n    \"\"\"\n    # build the version-list from the distributions for each component\n    versions = [{\"component\": label, \"version\": version} for (label, version) in version_info]\n\n    version_json = json.dumps(versions).encode(\"utf8\")\n    info = tarfile.TarInfo(name=\"versions.json\")\n    info.size = len(version_json)\n    export.tarfile.addfile(info, io.BytesIO(version_json))\n\n\ndef export_artifacts(export, artifacts):\n    \"\"\"\n    Export a set of Artifacts, ArtifactResources, and RepositoryResources\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        artifacts (django.db.models.Artifacts): list of artifacts in all repos being exported\n\n    Raises:\n        ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting\n    \"\"\"\n    data = dict(message=\"Exporting Artifacts\", code=\"export.artifacts\", total=len(artifacts))\n    with ProgressReport(**data) as pb:\n        for artifact in pb.iter(artifacts):\n            dest = artifact.file.name\n            if settings.DEFAULT_FILE_STORAGE != \"pulpcore.app.models.storage.FileSystem\":\n                with tempfile.TemporaryDirectory() as temp_dir:\n                    with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:\n                        temp_file.write(artifact.file.read())\n                        temp_file.flush()\n                        artifact.file.close()\n                        export.tarfile.add(temp_file.name, dest)\n            else:\n                export.tarfile.add(artifact.file.path, dest)\n\n    resource = ArtifactResource()\n    resource.queryset = artifacts\n    _write_export(export.tarfile, resource)\n\n    resource = RepositoryResource()\n    resource.queryset = Repository.objects.filter(pk__in=export.exporter.repositories.all())\n    _write_export(export.tarfile, resource)\n\n\ndef export_content(export, repository_version):\n    \"\"\"\n    Export db-content, and the db-content of the owning repositories\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        repository_version (django.db.models.RepositoryVersion): RepositoryVersion being exported\n    \"\"\"\n\n    def _combine_content_mappings(map1, map2):\n        \"\"\"Combine two content mapping dicts into one by combining ids for for each key.\"\"\"\n        result = {}\n        for key in map1.keys() | map2.keys():\n            result[key] = list(set(map1.get(key, []) + map2.get(key, [])))\n        return result\n\n    dest_dir = os.path.join(\n        \"repository-{}_{}\".format(\n            str(repository_version.repository.name), repository_version.number\n        )\n    )\n\n    # Export the connection between content and artifacts\n    resource = ContentArtifactResource(repository_version)\n    _write_export(export.tarfile, resource, dest_dir)\n\n    # content mapping is used by repo versions with subrepos (eg distribution tree repos)\n    content_mapping = {}\n\n    # find and export any ModelResource found in pulp_<repo-type>.app.modelresource\n    plugin_name = repository_version.repository.pulp_type.split(\".\")[0]\n    cfg = get_plugin_config(plugin_name)\n    if cfg.exportable_classes:\n        for cls in cfg.exportable_classes:\n            resource = cls(repository_version)\n            _write_export(export.tarfile, resource, dest_dir)\n\n            if hasattr(resource, \"content_mapping\") and resource.content_mapping:\n                content_mapping = _combine_content_mappings(\n                    content_mapping, resource.content_mapping\n                )\n\n    msg = (\n        f\"Exporting content for {plugin_name} \"\n        f\"repository-version {repository_version.repository.name}/{repository_version.number}\"\n    )\n    content_count = repository_version.content.count()\n    data = dict(\n        message=msg,\n        code=\"export.repo.version.content\",\n        total=content_count,\n        done=content_count,\n        state=TASK_STATES.COMPLETED,\n    )\n    pb = ProgressReport(**data)\n    pb.save()\n\n    if content_mapping:\n        # write the content mapping to tarfile\n        cm_json = json.dumps(content_mapping).encode(\"utf8\")\n        info = tarfile.TarInfo(name=f\"{dest_dir}/content_mapping.json\")\n        info.size = len(cm_json)\n        export.tarfile.addfile(info, io.BytesIO(cm_json))\n", "path": "pulpcore/app/importexport.py"}, {"content": "from import_export import fields\nfrom import_export.widgets import ForeignKeyWidget\nfrom logging import getLogger\n\nfrom pulpcore.app.models.content import (\n    Artifact,\n    Content,\n    ContentArtifact,\n)\nfrom pulpcore.app.models.repository import Repository\nfrom pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS\nfrom pulpcore.plugin.importexport import QueryModelResource\n\n\nlog = getLogger(__name__)\n\n\n#\n# Artifact and Repository are different from other import-export entities, in that they are not\n# repo-version-specific.\n#\nclass ArtifactResource(QueryModelResource):\n    \"\"\"Resource for import/export of artifacts.\"\"\"\n\n    def before_import_row(self, row, **kwargs):\n        \"\"\"\n        Sets digests to None if they are blank strings.\n\n        Args:\n            row (tablib.Dataset row): incoming import-row representing a single Variant.\n            kwargs: args passed along from the import() call.\n\n        \"\"\"\n        # the export converts None to blank strings but sha384 and sha512 have unique constraints\n        # that get triggered if they are blank. convert checksums back into None if they are blank.\n        for checksum in ALL_KNOWN_CONTENT_CHECKSUMS:\n            if row[checksum] == \"\":\n                row[checksum] = None\n\n    class Meta:\n        model = Artifact\n        exclude = (\n            \"pulp_id\",\n            \"pulp_created\",\n            \"pulp_last_updated\",\n        )\n        import_id_fields = (\"sha256\",)\n\n\nclass RepositoryResource(QueryModelResource):\n    class Meta:\n        model = Repository\n        import_id_fields = (\"name\",)\n        exclude = (\n            \"pulp_id\",\n            \"pulp_created\",\n            \"pulp_last_updated\",\n            \"content\",\n        )\n\n\nclass ContentArtifactResource(QueryModelResource):\n    \"\"\"\n    Handles import/export of the ContentArtifact model.\n\n    ContentArtifact is different from other import-export entities because it has no 'natural key'\n    other than a pulp_id, which aren't shared across instances. We do some magic to link up\n    ContentArtifacts to their matching (already-imported) Content.\n    \"\"\"\n\n    artifact = fields.Field(\n        column_name=\"artifact\", attribute=\"artifact\", widget=ForeignKeyWidget(Artifact, \"sha256\")\n    )\n\n    def before_import_row(self, row, **kwargs):\n        \"\"\"\n        Fixes the content-ptr of an incoming content-artifact row at import time.\n\n        Finds the 'original uuid' of the Content for this row, looks it up as the\n        'upstream_id' of imported Content, and then replaces the Content-pk with its\n        (new) uuid.\n\n        Args:\n            row (tablib.Dataset row): incoming import-row representing a single ContentArtifact.\n            kwargs: args passed along from the import() call.\n\n        Returns:\n            (tablib.Dataset row): row that now points to the new downstream uuid for its content.\n        \"\"\"\n\n        linked_content = Content.objects.get(upstream_id=row[\"content\"])\n        row[\"content\"] = str(linked_content.pulp_id)\n\n    def set_up_queryset(self):\n        return ContentArtifact.objects.filter(content__in=self.repo_version.content).order_by(\n            \"content\", \"relative_path\"\n        )\n\n    class Meta:\n        model = ContentArtifact\n        import_id_fields = (\n            \"content\",\n            \"relative_path\",\n        )\n        exclude = (\n            \"pulp_created\",\n            \"pulp_last_updated\",\n            \"_artifacts\",\n            \"pulp_id\",\n        )\n", "path": "pulpcore/app/modelresource.py"}], "after_files": [{"content": "import os\nimport io\nimport json\nimport tarfile\nimport tempfile\nimport logging\n\nfrom django.conf import settings\nfrom django.db.models.query import QuerySet\n\nfrom pulpcore.app.apps import get_plugin_config\nfrom pulpcore.app.models.progress import ProgressReport\nfrom pulpcore.app.models.repository import Repository\nfrom pulpcore.app.modelresource import (\n    ArtifactResource,\n    ContentArtifactResource,\n    RepositoryResource,\n)\nfrom pulpcore.constants import TASK_STATES, EXPORT_BATCH_SIZE\n\nlog = logging.getLogger(__name__)\n\n\ndef _write_export(the_tarfile, resource, dest_dir=None):\n    \"\"\"\n    Write the JSON export for the specified resource to the specified tarfile.\n\n    The resulting file will be found at <dest_dir>/<resource.__class__.__name__>.json. If dest_dir\n    is None, the file will be added at the 'top level' of the_tarfile.\n\n    Export-files are UTF-8 encoded.\n\n    Args:\n        the_tarfile (tarfile.Tarfile): tarfile we are writing into\n        resource (import_export.resources.ModelResource): ModelResource to be exported\n        dest_dir str(directory-path): directory 'inside' the tarfile to write to\n    \"\"\"\n    filename = \"{}.{}.json\".format(resource.__module__, type(resource).__name__)\n    if dest_dir:\n        dest_filename = os.path.join(dest_dir, filename)\n    else:\n        dest_filename = filename\n\n    # If the resource is the type of QuerySet, then export the data in batch to save memory.\n    # Otherwise, export all data in oneshot. This is because the underlying libraries\n    # (json; django-import-export) do not support to stream the output to file, we export\n    # the data in batches to memory and concatenate the json lists via string manipulation.\n    with tempfile.NamedTemporaryFile(dir=os.getcwd(), mode=\"w\", encoding=\"utf8\") as temp_file:\n        if isinstance(resource.queryset, QuerySet):\n            temp_file.write(\"[\")\n            total = resource.queryset.count()\n            for i in range(0, total, EXPORT_BATCH_SIZE):\n                current_batch = i + EXPORT_BATCH_SIZE\n                dataset = resource.export(resource.queryset[i:current_batch])\n                # Strip \"[\" and \"]\" as we are writing the dataset in batch\n                temp_file.write(dataset.json.lstrip(\"[\").rstrip(\"]\"))\n                if current_batch < total:\n                    # Write \",\" if not last loop\n                    temp_file.write(\", \")\n            temp_file.write(\"]\")\n        else:\n            dataset = resource.export(resource.queryset)\n            temp_file.write(dataset.json)\n\n        temp_file.flush()\n        info = tarfile.TarInfo(name=dest_filename)\n        info.size = os.path.getsize(temp_file.name)\n        with open(temp_file.name, \"rb\") as fd:\n            the_tarfile.addfile(info, fd)\n\n\ndef export_versions(export, version_info):\n    \"\"\"\n    Write a JSON list of plugins and their versions as 'versions.json' to export.tarfile\n\n    Output format is [{\"component\": \"<pluginname>\", \"version\": \"<pluginversion>\"},...]\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        version_info (set): set of (distribution-label,version) tuples for repos in this export\n    \"\"\"\n    # build the version-list from the distributions for each component\n    versions = [{\"component\": label, \"version\": version} for (label, version) in version_info]\n\n    version_json = json.dumps(versions).encode(\"utf8\")\n    info = tarfile.TarInfo(name=\"versions.json\")\n    info.size = len(version_json)\n    export.tarfile.addfile(info, io.BytesIO(version_json))\n\n\ndef export_artifacts(export, artifacts):\n    \"\"\"\n    Export a set of Artifacts, ArtifactResources, and RepositoryResources\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        artifacts (django.db.models.Artifacts): list of artifacts in all repos being exported\n\n    Raises:\n        ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting\n    \"\"\"\n    data = dict(message=\"Exporting Artifacts\", code=\"export.artifacts\", total=len(artifacts))\n    with ProgressReport(**data) as pb:\n        for artifact in pb.iter(artifacts):\n            dest = artifact.file.name\n            if settings.DEFAULT_FILE_STORAGE != \"pulpcore.app.models.storage.FileSystem\":\n                with tempfile.TemporaryDirectory() as temp_dir:\n                    with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:\n                        temp_file.write(artifact.file.read())\n                        temp_file.flush()\n                        artifact.file.close()\n                        export.tarfile.add(temp_file.name, dest)\n            else:\n                export.tarfile.add(artifact.file.path, dest)\n\n    resource = ArtifactResource()\n    resource.queryset = artifacts\n    _write_export(export.tarfile, resource)\n\n    resource = RepositoryResource()\n    resource.queryset = Repository.objects.filter(pk__in=export.exporter.repositories.all())\n    _write_export(export.tarfile, resource)\n\n\ndef export_content(export, repository_version):\n    \"\"\"\n    Export db-content, and the db-content of the owning repositories\n\n    Args:\n        export (django.db.models.PulpExport): export instance that's doing the export\n        repository_version (django.db.models.RepositoryVersion): RepositoryVersion being exported\n    \"\"\"\n\n    def _combine_content_mappings(map1, map2):\n        \"\"\"Combine two content mapping dicts into one by combining ids for for each key.\"\"\"\n        result = {}\n        for key in map1.keys() | map2.keys():\n            result[key] = list(set(map1.get(key, []) + map2.get(key, [])))\n        return result\n\n    dest_dir = os.path.join(\n        \"repository-{}_{}\".format(\n            str(repository_version.repository.name), repository_version.number\n        )\n    )\n\n    # content mapping is used by repo versions with subrepos (eg distribution tree repos)\n    content_mapping = {}\n\n    # find and export any ModelResource found in pulp_<repo-type>.app.modelresource\n    plugin_name = repository_version.repository.pulp_type.split(\".\")[0]\n    cfg = get_plugin_config(plugin_name)\n    if cfg.exportable_classes:\n        for cls in cfg.exportable_classes:\n            resource = cls(repository_version)\n            _write_export(export.tarfile, resource, dest_dir)\n\n            if hasattr(resource, \"content_mapping\") and resource.content_mapping:\n                content_mapping = _combine_content_mappings(\n                    content_mapping, resource.content_mapping\n                )\n\n    # Export the connection between content and artifacts\n    resource = ContentArtifactResource(repository_version, content_mapping)\n    _write_export(export.tarfile, resource, dest_dir)\n\n    msg = (\n        f\"Exporting content for {plugin_name} \"\n        f\"repository-version {repository_version.repository.name}/{repository_version.number}\"\n    )\n    content_count = repository_version.content.count()\n    data = dict(\n        message=msg,\n        code=\"export.repo.version.content\",\n        total=content_count,\n        done=content_count,\n        state=TASK_STATES.COMPLETED,\n    )\n    pb = ProgressReport(**data)\n    pb.save()\n\n    if content_mapping:\n        # write the content mapping to tarfile\n        cm_json = json.dumps(content_mapping).encode(\"utf8\")\n        info = tarfile.TarInfo(name=f\"{dest_dir}/content_mapping.json\")\n        info.size = len(cm_json)\n        export.tarfile.addfile(info, io.BytesIO(cm_json))\n", "path": "pulpcore/app/importexport.py"}, {"content": "from import_export import fields\nfrom import_export.widgets import ForeignKeyWidget\nfrom logging import getLogger\n\nfrom pulpcore.app.models.content import (\n    Artifact,\n    Content,\n    ContentArtifact,\n)\nfrom pulpcore.app.models.repository import Repository\nfrom pulpcore.constants import ALL_KNOWN_CONTENT_CHECKSUMS\nfrom pulpcore.plugin.importexport import QueryModelResource\n\n\nlog = getLogger(__name__)\n\n\n#\n# Artifact and Repository are different from other import-export entities, in that they are not\n# repo-version-specific.\n#\nclass ArtifactResource(QueryModelResource):\n    \"\"\"Resource for import/export of artifacts.\"\"\"\n\n    def before_import_row(self, row, **kwargs):\n        \"\"\"\n        Sets digests to None if they are blank strings.\n\n        Args:\n            row (tablib.Dataset row): incoming import-row representing a single Variant.\n            kwargs: args passed along from the import() call.\n\n        \"\"\"\n        # the export converts None to blank strings but sha384 and sha512 have unique constraints\n        # that get triggered if they are blank. convert checksums back into None if they are blank.\n        for checksum in ALL_KNOWN_CONTENT_CHECKSUMS:\n            if row[checksum] == \"\":\n                row[checksum] = None\n\n    class Meta:\n        model = Artifact\n        exclude = (\n            \"pulp_id\",\n            \"pulp_created\",\n            \"pulp_last_updated\",\n        )\n        import_id_fields = (\"sha256\",)\n\n\nclass RepositoryResource(QueryModelResource):\n    class Meta:\n        model = Repository\n        import_id_fields = (\"name\",)\n        exclude = (\n            \"pulp_id\",\n            \"pulp_created\",\n            \"pulp_last_updated\",\n            \"content\",\n        )\n\n\nclass ContentArtifactResource(QueryModelResource):\n    \"\"\"\n    Handles import/export of the ContentArtifact model.\n\n    ContentArtifact is different from other import-export entities because it has no 'natural key'\n    other than a pulp_id, which aren't shared across instances. We do some magic to link up\n    ContentArtifacts to their matching (already-imported) Content.\n\n    Some plugin-models have sub-repositories. We take advantage of the content-mapping\n    machinery to account for those contentartifacts as well.\n    \"\"\"\n\n    artifact = fields.Field(\n        column_name=\"artifact\", attribute=\"artifact\", widget=ForeignKeyWidget(Artifact, \"sha256\")\n    )\n\n    def __init__(self, repo_version=None, content_mapping=None):\n        self.content_mapping = content_mapping\n        super().__init__(repo_version)\n\n    def before_import_row(self, row, **kwargs):\n        \"\"\"\n        Fixes the content-ptr of an incoming content-artifact row at import time.\n\n        Finds the 'original uuid' of the Content for this row, looks it up as the\n        'upstream_id' of imported Content, and then replaces the Content-pk with its\n        (new) uuid.\n\n        Args:\n            row (tablib.Dataset row): incoming import-row representing a single ContentArtifact.\n            kwargs: args passed along from the import() call.\n\n        Returns:\n            (tablib.Dataset row): row that now points to the new downstream uuid for its content.\n        \"\"\"\n\n        linked_content = Content.objects.get(upstream_id=row[\"content\"])\n        row[\"content\"] = str(linked_content.pulp_id)\n\n    def set_up_queryset(self):\n        vers_content = ContentArtifact.objects.filter(content__in=self.repo_version.content)\n        if self.content_mapping:\n            all_content = []\n            for content_ids in self.content_mapping.values():\n                all_content.extend(content_ids)\n            vers_content = vers_content.union(\n                ContentArtifact.objects.filter(content__in=all_content)\n            )\n        return vers_content.order_by(\"content\", \"relative_path\")\n\n    class Meta:\n        model = ContentArtifact\n        import_id_fields = (\n            \"content\",\n            \"relative_path\",\n        )\n        exclude = (\n            \"pulp_created\",\n            \"pulp_last_updated\",\n            \"_artifacts\",\n            \"pulp_id\",\n        )\n", "path": "pulpcore/app/modelresource.py"}]}