We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Change 1C export algo for Tags
В этой задаче речь идёт только о файле тегов:
shopelectro/management/commands/_update_catalog/update_tags.py

27.11.18 Произошла неприятная история.
Контентщик выгрузил нес-ко тегов для гирлянд, но они пропали от нашей выгрузки.
Сейчас выгрузка Тегов просто сносит всё что есть и заново накатывает из 1С. Мы немного изменим эту политику.

Реализуй такой алгоритм:
- все Теги, что зашли из 1С, заливаем в базу с перезаписью
- все Теги, что есть в базе, но нет в 1С, не трогаем



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `shopelectro/management/commands/_update_catalog/utils.py`
Content:
```
1 import glob
2 import logging
3 import os
4 import shutil
5 import subprocess
6 import time
7 from contextlib import contextmanager
8 from itertools import chain
9 from typing import Iterator, Dict
10 from uuid import UUID
11 from xml.etree import ElementTree
12 
13 import requests
14 from django.conf import settings
15 
16 
17 logger = logging.getLogger(__name__)
18 Data = Dict[str, str]
19 NOT_SAVE_TEMPLATE = '{entity} with name="{name}" has no {field}. It\'ll not be' \
20                     ' saved'
21 
22 
23 def is_correct_uuid(uuid_):
24     try:
25         val = UUID(uuid_)
26     except (ValueError, TypeError):
27         return False
28     return str(val) == uuid_
29 
30 
31 class XmlFile:
32 
33     namespace = '{urn:1C.ru:commerceml_2}'
34 
35     def __init__(self, fetch_callback, xml_path_pattern, xpath_queries,
36                  extra_options=None):
37         self.fetch_callback = fetch_callback
38         self.xml_path_pattern = xml_path_pattern
39         self.xpath_queries = xpath_queries
40         self.extra_options = extra_options or {}
41 
42     @property
43     def parsed_files(self):
44         """Get parsed xml files, that matched the path pattern."""
45         xml_files = glob.glob(os.path.join(
46             settings.ASSETS_DIR, self.xml_path_pattern
47         ))
48         assert xml_files, 'Files on path {} does not exist.'.format(
49             self.xml_path_pattern
50         )
51         return [ElementTree.parse(file) for file in xml_files]
52 
53     @property
54     def xpaths(self):
55         """Get xpath queries for xml."""
56         return {
57             name: query.format(self.namespace)
58             for name, query in self.xpath_queries.items()
59         }
60 
61     def get_data(self) -> Iterator:
62         """
63         Get data from xml files.
64 
65         Example files with products names or prices.
66         """
67         return chain.from_iterable(
68             self.fetch_callback(file, self)
69             for file in self.parsed_files
70         )
71 
72 
73 @contextmanager
74 def collect_errors(error_types: tuple):
75     errors = []
76 
77     @contextmanager
78     def collect():
79         try:
80             yield
81         except error_types as error:
82             errors.append(error)
83     yield collect
84     if errors:
85         raise errors[0]
86 
87 
88 @contextmanager
89 def download_catalog(destination):
90     """Download catalog's xml files and delete after handle them."""
91     wget_command = (
92         'wget -r -P {} ftp://{}:{}@{}/webdata'
93         ' 2>&1 | grep "время\|time\|Downloaded"'.format(
94             destination,
95             settings.FTP_USER,
96             settings.FTP_PASS,
97             settings.FTP_IP,
98         )
99     )
100 
101     subprocess.run(wget_command, shell=True)
102     assert os.path.exists(os.path.join(
103         destination, settings.FTP_IP)), 'Files do not downloaded...'
104     logger.info('Download catalog - completed...')
105 
106     try:
107         yield
108     finally:
109         # remove downloaded data
110         shutil.rmtree(os.path.join(destination, settings.FTP_IP))
111 
112 
113 def report(error):
114     report_url = getattr(settings, 'SLACK_REPORT_URL', None)
115     if report_url is not None:
116         requests.post(
117             url=report_url,
118             json={
119                 'text': '*Не удалось обновить каталог Shopelectro.*\n'
120                         '*Время*: {}\n'
121                         '*Ошибка*: {}'.format(time.ctime(), error),
122             }
123         )
124
```

Path: `shopelectro/management/commands/_update_catalog/update_tags.py`
Content:
```
1 import logging
2 from copy import deepcopy
3 from itertools import chain
4 from typing import Iterator, Dict
5 from xml.etree.ElementTree import Element
6 
7 from django.db import transaction
8 
9 from shopelectro.management.commands._update_catalog.utils import (
10     XmlFile, is_correct_uuid, UUID, Data,
11 )
12 from shopelectro.models import Tag, TagGroup
13 
14 
15 logger = logging.getLogger(__name__)
16 
17 
18 def fetch_tags(root: Element, config: XmlFile):
19     def get_uuid_name_pair(
20             element: Element,
21             uuid_xpath: str,
22             name_xpath: str,
23     ):
24         uuid = element.find(uuid_xpath).text
25         name = element.find(name_xpath).text
26 
27         return uuid, name
28 
29     tag_groups = root.findall(config.xpaths['tag_groups'])
30     for group in tag_groups:
31         group_uuid, group_name = get_uuid_name_pair(
32             group,
33             config.xpaths['tag_group_uuid'],
34             config.xpaths['tag_group_name'],
35         )
36 
37         tags = group.findall(config.xpaths['tags'])
38         tags_data = (
39             get_uuid_name_pair(
40                 tag,
41                 config.xpaths['tag_uuid'],
42                 config.xpaths['tag_name'],
43             ) for tag in tags
44         )
45 
46         yield group_uuid, {
47             'name': group_name,
48             'tags_data': tags_data,
49         }
50 
51 tag_file = XmlFile(
52     fetch_callback=fetch_tags,
53     xml_path_pattern='**/webdata/**/properties/**/import*.xml',
54     xpath_queries={
55         'tag_groups': './/{}Свойства/',
56         'tag_group_uuid': '.{}Ид',
57         'tag_group_name': '.{}Наименование',
58         'tags': '.{}ВариантыЗначений/',
59         'tag_name': '.{}Значение',
60         'tag_uuid': '.{}ИдЗначения',
61     },
62 )
63 
64 
65 @transaction.atomic
66 def create_or_update(data: Dict[UUID, Data]):
67     group_data = deepcopy(data)
68 
69     created_groups_count = 0
70     created_tags_count = 0
71 
72     for group_uuid, data_ in group_data.items():
73         tags = data_.pop('tags')
74 
75         group, group_created = TagGroup.objects.update_or_create(
76             uuid=group_uuid, defaults=data_
77         )
78 
79         created_groups_count += int(group_created)
80 
81         for tag_uuid, tag_data in tags.items():
82             _, tag_created = Tag.objects.update_or_create(
83                 uuid=tag_uuid,
84                 defaults={**tag_data, 'group': group}
85             )
86 
87             created_tags_count += int(tag_created)
88 
89     logger.info(f'{created_groups_count} tag groups were created.')
90     logger.info(f'{created_tags_count} tags were created.')
91 
92 
93 @transaction.atomic
94 def delete(group_data: Dict[UUID, Data]):
95     group_data = deepcopy(group_data)
96 
97     group_uuids = group_data.keys()
98     tag_uuids = set(chain.from_iterable(
99         data['tags'].keys()
100         for data in group_data.values()
101     ))
102 
103     if not (group_uuids and tag_uuids):
104         return
105 
106     group_count, _ = TagGroup.objects.exclude(uuid__in=group_uuids).delete()
107     tag_count, _ = Tag.objects.exclude(uuid__in=tag_uuids).delete()
108 
109     logger.info(f'{group_count} tag groups and {tag_count} tags were deleted.')
110 
111 
112 def prepare_data(group_data: Iterator) -> Dict[UUID, Data]:
113     def assembly_structure(group_uuid: str, group_data_: dict):
114         tags_data = group_data_.pop('tags_data', [])
115         tags = {
116             tag_uuid: {'name': tag_name}
117             for tag_uuid, tag_name in tags_data
118             if is_correct_uuid(tag_uuid)
119         }
120 
121         return (
122             group_uuid, {
123                 **group_data_,
124                 'tags': tags
125             }
126         )
127 
128     return dict(
129         assembly_structure(group_uuid, data)
130         for group_uuid, data in group_data
131         if is_correct_uuid(group_uuid)
132     )
133 
134 
135 def main(*args, **kwargs):
136     cleared_group_data = prepare_data(tag_file.get_data())
137     create_or_update(cleared_group_data)
138     delete(cleared_group_data)
139
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/shopelectro/management/commands/_update_catalog/update_tags.py b/shopelectro/management/commands/_update_catalog/update_tags.py
--- a/shopelectro/management/commands/_update_catalog/update_tags.py
+++ b/shopelectro/management/commands/_update_catalog/update_tags.py
@@ -1,13 +1,12 @@
 import logging
 from copy import deepcopy
-from itertools import chain
 from typing import Iterator, Dict
 from xml.etree.ElementTree import Element
 
 from django.db import transaction
 
 from shopelectro.management.commands._update_catalog.utils import (
-    XmlFile, is_correct_uuid, UUID, Data,
+    Data, is_correct_uuid, UUID_TYPE, XmlFile
 )
 from shopelectro.models import Tag, TagGroup
 
@@ -63,7 +62,7 @@
 
 
 @transaction.atomic
-def create_or_update(data: Dict[UUID, Data]):
+def create_or_update(data: Dict[UUID_TYPE, Data]):
     group_data = deepcopy(data)
 
     created_groups_count = 0
@@ -90,26 +89,7 @@
     logger.info(f'{created_tags_count} tags were created.')
 
 
-@transaction.atomic
-def delete(group_data: Dict[UUID, Data]):
-    group_data = deepcopy(group_data)
-
-    group_uuids = group_data.keys()
-    tag_uuids = set(chain.from_iterable(
-        data['tags'].keys()
-        for data in group_data.values()
-    ))
-
-    if not (group_uuids and tag_uuids):
-        return
-
-    group_count, _ = TagGroup.objects.exclude(uuid__in=group_uuids).delete()
-    tag_count, _ = Tag.objects.exclude(uuid__in=tag_uuids).delete()
-
-    logger.info(f'{group_count} tag groups and {tag_count} tags were deleted.')
-
-
-def prepare_data(group_data: Iterator) -> Dict[UUID, Data]:
+def prepare_data(group_data: Iterator) -> Dict[UUID_TYPE, Data]:
     def assembly_structure(group_uuid: str, group_data_: dict):
         tags_data = group_data_.pop('tags_data', [])
         tags = {
@@ -135,4 +115,3 @@
 def main(*args, **kwargs):
     cleared_group_data = prepare_data(tag_file.get_data())
     create_or_update(cleared_group_data)
-    delete(cleared_group_data)
diff --git a/shopelectro/management/commands/_update_catalog/utils.py b/shopelectro/management/commands/_update_catalog/utils.py
--- a/shopelectro/management/commands/_update_catalog/utils.py
+++ b/shopelectro/management/commands/_update_catalog/utils.py
@@ -15,7 +15,8 @@
 
 
 logger = logging.getLogger(__name__)
-Data = Dict[str, str]
+UUID_TYPE = str
+Data = Dict[str, Dict[str, dict]]
 NOT_SAVE_TEMPLATE = '{entity} with name="{name}" has no {field}. It\'ll not be' \
                     ' saved'

{"golden_diff": "diff --git a/shopelectro/management/commands/_update_catalog/update_tags.py b/shopelectro/management/commands/_update_catalog/update_tags.py\n--- a/shopelectro/management/commands/_update_catalog/update_tags.py\n+++ b/shopelectro/management/commands/_update_catalog/update_tags.py\n@@ -1,13 +1,12 @@\n import logging\n from copy import deepcopy\n-from itertools import chain\n from typing import Iterator, Dict\n from xml.etree.ElementTree import Element\n \n from django.db import transaction\n \n from shopelectro.management.commands._update_catalog.utils import (\n-    XmlFile, is_correct_uuid, UUID, Data,\n+    Data, is_correct_uuid, UUID_TYPE, XmlFile\n )\n from shopelectro.models import Tag, TagGroup\n \n@@ -63,7 +62,7 @@\n \n \n @transaction.atomic\n-def create_or_update(data: Dict[UUID, Data]):\n+def create_or_update(data: Dict[UUID_TYPE, Data]):\n     group_data = deepcopy(data)\n \n     created_groups_count = 0\n@@ -90,26 +89,7 @@\n     logger.info(f'{created_tags_count} tags were created.')\n \n \n-@transaction.atomic\n-def delete(group_data: Dict[UUID, Data]):\n-    group_data = deepcopy(group_data)\n-\n-    group_uuids = group_data.keys()\n-    tag_uuids = set(chain.from_iterable(\n-        data['tags'].keys()\n-        for data in group_data.values()\n-    ))\n-\n-    if not (group_uuids and tag_uuids):\n-        return\n-\n-    group_count, _ = TagGroup.objects.exclude(uuid__in=group_uuids).delete()\n-    tag_count, _ = Tag.objects.exclude(uuid__in=tag_uuids).delete()\n-\n-    logger.info(f'{group_count} tag groups and {tag_count} tags were deleted.')\n-\n-\n-def prepare_data(group_data: Iterator) -> Dict[UUID, Data]:\n+def prepare_data(group_data: Iterator) -> Dict[UUID_TYPE, Data]:\n     def assembly_structure(group_uuid: str, group_data_: dict):\n         tags_data = group_data_.pop('tags_data', [])\n         tags = {\n@@ -135,4 +115,3 @@\n def main(*args, **kwargs):\n     cleared_group_data = prepare_data(tag_file.get_data())\n     create_or_update(cleared_group_data)\n-    delete(cleared_group_data)\ndiff --git a/shopelectro/management/commands/_update_catalog/utils.py b/shopelectro/management/commands/_update_catalog/utils.py\n--- a/shopelectro/management/commands/_update_catalog/utils.py\n+++ b/shopelectro/management/commands/_update_catalog/utils.py\n@@ -15,7 +15,8 @@\n \n \n logger = logging.getLogger(__name__)\n-Data = Dict[str, str]\n+UUID_TYPE = str\n+Data = Dict[str, Dict[str, dict]]\n NOT_SAVE_TEMPLATE = '{entity} with name=\"{name}\" has no {field}. It\\'ll not be' \\\n                     ' saved'\n", "issue": "Change 1C export algo for Tags\n\u0412 \u044d\u0442\u043e\u0439 \u0437\u0430\u0434\u0430\u0447\u0435 \u0440\u0435\u0447\u044c \u0438\u0434\u0451\u0442 \u0442\u043e\u043b\u044c\u043a\u043e \u043e \u0444\u0430\u0439\u043b\u0435 \u0442\u0435\u0433\u043e\u0432:\r\nshopelectro/management/commands/_update_catalog/update_tags.py\r\n\r\n27.11.18 \u041f\u0440\u043e\u0438\u0437\u043e\u0448\u043b\u0430 \u043d\u0435\u043f\u0440\u0438\u044f\u0442\u043d\u0430\u044f \u0438\u0441\u0442\u043e\u0440\u0438\u044f.\r\n\u041a\u043e\u043d\u0442\u0435\u043d\u0442\u0449\u0438\u043a \u0432\u044b\u0433\u0440\u0443\u0437\u0438\u043b \u043d\u0435\u0441-\u043a\u043e \u0442\u0435\u0433\u043e\u0432 \u0434\u043b\u044f \u0433\u0438\u0440\u043b\u044f\u043d\u0434, \u043d\u043e \u043e\u043d\u0438 \u043f\u0440\u043e\u043f\u0430\u043b\u0438 \u043e\u0442 \u043d\u0430\u0448\u0435\u0439 \u0432\u044b\u0433\u0440\u0443\u0437\u043a\u0438.\r\n\u0421\u0435\u0439\u0447\u0430\u0441 \u0432\u044b\u0433\u0440\u0443\u0437\u043a\u0430 \u0422\u0435\u0433\u043e\u0432 \u043f\u0440\u043e\u0441\u0442\u043e \u0441\u043d\u043e\u0441\u0438\u0442 \u0432\u0441\u0451 \u0447\u0442\u043e \u0435\u0441\u0442\u044c \u0438 \u0437\u0430\u043d\u043e\u0432\u043e \u043d\u0430\u043a\u0430\u0442\u044b\u0432\u0430\u0435\u0442 \u0438\u0437 1\u0421. \u041c\u044b \u043d\u0435\u043c\u043d\u043e\u0433\u043e \u0438\u0437\u043c\u0435\u043d\u0438\u043c \u044d\u0442\u0443 \u043f\u043e\u043b\u0438\u0442\u0438\u043a\u0443.\r\n\r\n\u0420\u0435\u0430\u043b\u0438\u0437\u0443\u0439 \u0442\u0430\u043a\u043e\u0439 \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c:\r\n- \u0432\u0441\u0435 \u0422\u0435\u0433\u0438, \u0447\u0442\u043e \u0437\u0430\u0448\u043b\u0438 \u0438\u0437 1\u0421, \u0437\u0430\u043b\u0438\u0432\u0430\u0435\u043c \u0432 \u0431\u0430\u0437\u0443 \u0441 \u043f\u0435\u0440\u0435\u0437\u0430\u043f\u0438\u0441\u044c\u044e\r\n- \u0432\u0441\u0435 \u0422\u0435\u0433\u0438, \u0447\u0442\u043e \u0435\u0441\u0442\u044c \u0432 \u0431\u0430\u0437\u0435, \u043d\u043e \u043d\u0435\u0442 \u0432 1\u0421, \u043d\u0435 \u0442\u0440\u043e\u0433\u0430\u0435\u043c\r\n\r\n\n", "before_files": [{"content": "import glob\nimport logging\nimport os\nimport shutil\nimport subprocess\nimport time\nfrom contextlib import contextmanager\nfrom itertools import chain\nfrom typing import Iterator, Dict\nfrom uuid import UUID\nfrom xml.etree import ElementTree\n\nimport requests\nfrom django.conf import settings\n\n\nlogger = logging.getLogger(__name__)\nData = Dict[str, str]\nNOT_SAVE_TEMPLATE = '{entity} with name=\"{name}\" has no {field}. It\\'ll not be' \\\n                    ' saved'\n\n\ndef is_correct_uuid(uuid_):\n    try:\n        val = UUID(uuid_)\n    except (ValueError, TypeError):\n        return False\n    return str(val) == uuid_\n\n\nclass XmlFile:\n\n    namespace = '{urn:1C.ru:commerceml_2}'\n\n    def __init__(self, fetch_callback, xml_path_pattern, xpath_queries,\n                 extra_options=None):\n        self.fetch_callback = fetch_callback\n        self.xml_path_pattern = xml_path_pattern\n        self.xpath_queries = xpath_queries\n        self.extra_options = extra_options or {}\n\n    @property\n    def parsed_files(self):\n        \"\"\"Get parsed xml files, that matched the path pattern.\"\"\"\n        xml_files = glob.glob(os.path.join(\n            settings.ASSETS_DIR, self.xml_path_pattern\n        ))\n        assert xml_files, 'Files on path {} does not exist.'.format(\n            self.xml_path_pattern\n        )\n        return [ElementTree.parse(file) for file in xml_files]\n\n    @property\n    def xpaths(self):\n        \"\"\"Get xpath queries for xml.\"\"\"\n        return {\n            name: query.format(self.namespace)\n            for name, query in self.xpath_queries.items()\n        }\n\n    def get_data(self) -> Iterator:\n        \"\"\"\n        Get data from xml files.\n\n        Example files with products names or prices.\n        \"\"\"\n        return chain.from_iterable(\n            self.fetch_callback(file, self)\n            for file in self.parsed_files\n        )\n\n\n@contextmanager\ndef collect_errors(error_types: tuple):\n    errors = []\n\n    @contextmanager\n    def collect():\n        try:\n            yield\n        except error_types as error:\n            errors.append(error)\n    yield collect\n    if errors:\n        raise errors[0]\n\n\n@contextmanager\ndef download_catalog(destination):\n    \"\"\"Download catalog's xml files and delete after handle them.\"\"\"\n    wget_command = (\n        'wget -r -P {} ftp://{}:{}@{}/webdata'\n        ' 2>&1 | grep \"\u0432\u0440\u0435\u043c\u044f\\|time\\|Downloaded\"'.format(\n            destination,\n            settings.FTP_USER,\n            settings.FTP_PASS,\n            settings.FTP_IP,\n        )\n    )\n\n    subprocess.run(wget_command, shell=True)\n    assert os.path.exists(os.path.join(\n        destination, settings.FTP_IP)), 'Files do not downloaded...'\n    logger.info('Download catalog - completed...')\n\n    try:\n        yield\n    finally:\n        # remove downloaded data\n        shutil.rmtree(os.path.join(destination, settings.FTP_IP))\n\n\ndef report(error):\n    report_url = getattr(settings, 'SLACK_REPORT_URL', None)\n    if report_url is not None:\n        requests.post(\n            url=report_url,\n            json={\n                'text': '*\u041d\u0435 \u0443\u0434\u0430\u043b\u043e\u0441\u044c \u043e\u0431\u043d\u043e\u0432\u0438\u0442\u044c \u043a\u0430\u0442\u0430\u043b\u043e\u0433 Shopelectro.*\\n'\n                        '*\u0412\u0440\u0435\u043c\u044f*: {}\\n'\n                        '*\u041e\u0448\u0438\u0431\u043a\u0430*: {}'.format(time.ctime(), error),\n            }\n        )\n", "path": "shopelectro/management/commands/_update_catalog/utils.py"}, {"content": "import logging\nfrom copy import deepcopy\nfrom itertools import chain\nfrom typing import Iterator, Dict\nfrom xml.etree.ElementTree import Element\n\nfrom django.db import transaction\n\nfrom shopelectro.management.commands._update_catalog.utils import (\n    XmlFile, is_correct_uuid, UUID, Data,\n)\nfrom shopelectro.models import Tag, TagGroup\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef fetch_tags(root: Element, config: XmlFile):\n    def get_uuid_name_pair(\n            element: Element,\n            uuid_xpath: str,\n            name_xpath: str,\n    ):\n        uuid = element.find(uuid_xpath).text\n        name = element.find(name_xpath).text\n\n        return uuid, name\n\n    tag_groups = root.findall(config.xpaths['tag_groups'])\n    for group in tag_groups:\n        group_uuid, group_name = get_uuid_name_pair(\n            group,\n            config.xpaths['tag_group_uuid'],\n            config.xpaths['tag_group_name'],\n        )\n\n        tags = group.findall(config.xpaths['tags'])\n        tags_data = (\n            get_uuid_name_pair(\n                tag,\n                config.xpaths['tag_uuid'],\n                config.xpaths['tag_name'],\n            ) for tag in tags\n        )\n\n        yield group_uuid, {\n            'name': group_name,\n            'tags_data': tags_data,\n        }\n\ntag_file = XmlFile(\n    fetch_callback=fetch_tags,\n    xml_path_pattern='**/webdata/**/properties/**/import*.xml',\n    xpath_queries={\n        'tag_groups': './/{}\u0421\u0432\u043e\u0439\u0441\u0442\u0432\u0430/',\n        'tag_group_uuid': '.{}\u0418\u0434',\n        'tag_group_name': '.{}\u041d\u0430\u0438\u043c\u0435\u043d\u043e\u0432\u0430\u043d\u0438\u0435',\n        'tags': '.{}\u0412\u0430\u0440\u0438\u0430\u043d\u0442\u044b\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0439/',\n        'tag_name': '.{}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435',\n        'tag_uuid': '.{}\u0418\u0434\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u044f',\n    },\n)\n\n\n@transaction.atomic\ndef create_or_update(data: Dict[UUID, Data]):\n    group_data = deepcopy(data)\n\n    created_groups_count = 0\n    created_tags_count = 0\n\n    for group_uuid, data_ in group_data.items():\n        tags = data_.pop('tags')\n\n        group, group_created = TagGroup.objects.update_or_create(\n            uuid=group_uuid, defaults=data_\n        )\n\n        created_groups_count += int(group_created)\n\n        for tag_uuid, tag_data in tags.items():\n            _, tag_created = Tag.objects.update_or_create(\n                uuid=tag_uuid,\n                defaults={**tag_data, 'group': group}\n            )\n\n            created_tags_count += int(tag_created)\n\n    logger.info(f'{created_groups_count} tag groups were created.')\n    logger.info(f'{created_tags_count} tags were created.')\n\n\n@transaction.atomic\ndef delete(group_data: Dict[UUID, Data]):\n    group_data = deepcopy(group_data)\n\n    group_uuids = group_data.keys()\n    tag_uuids = set(chain.from_iterable(\n        data['tags'].keys()\n        for data in group_data.values()\n    ))\n\n    if not (group_uuids and tag_uuids):\n        return\n\n    group_count, _ = TagGroup.objects.exclude(uuid__in=group_uuids).delete()\n    tag_count, _ = Tag.objects.exclude(uuid__in=tag_uuids).delete()\n\n    logger.info(f'{group_count} tag groups and {tag_count} tags were deleted.')\n\n\ndef prepare_data(group_data: Iterator) -> Dict[UUID, Data]:\n    def assembly_structure(group_uuid: str, group_data_: dict):\n        tags_data = group_data_.pop('tags_data', [])\n        tags = {\n            tag_uuid: {'name': tag_name}\n            for tag_uuid, tag_name in tags_data\n            if is_correct_uuid(tag_uuid)\n        }\n\n        return (\n            group_uuid, {\n                **group_data_,\n                'tags': tags\n            }\n        )\n\n    return dict(\n        assembly_structure(group_uuid, data)\n        for group_uuid, data in group_data\n        if is_correct_uuid(group_uuid)\n    )\n\n\ndef main(*args, **kwargs):\n    cleared_group_data = prepare_data(tag_file.get_data())\n    create_or_update(cleared_group_data)\n    delete(cleared_group_data)\n", "path": "shopelectro/management/commands/_update_catalog/update_tags.py"}], "after_files": [{"content": "import glob\nimport logging\nimport os\nimport shutil\nimport subprocess\nimport time\nfrom contextlib import contextmanager\nfrom itertools import chain\nfrom typing import Iterator, Dict\nfrom uuid import UUID\nfrom xml.etree import ElementTree\n\nimport requests\nfrom django.conf import settings\n\n\nlogger = logging.getLogger(__name__)\nUUID_TYPE = str\nData = Dict[str, Dict[str, dict]]\nNOT_SAVE_TEMPLATE = '{entity} with name=\"{name}\" has no {field}. It\\'ll not be' \\\n                    ' saved'\n\n\ndef is_correct_uuid(uuid_):\n    try:\n        val = UUID(uuid_)\n    except (ValueError, TypeError):\n        return False\n    return str(val) == uuid_\n\n\nclass XmlFile:\n\n    namespace = '{urn:1C.ru:commerceml_2}'\n\n    def __init__(self, fetch_callback, xml_path_pattern, xpath_queries,\n                 extra_options=None):\n        self.fetch_callback = fetch_callback\n        self.xml_path_pattern = xml_path_pattern\n        self.xpath_queries = xpath_queries\n        self.extra_options = extra_options or {}\n\n    @property\n    def parsed_files(self):\n        \"\"\"Get parsed xml files, that matched the path pattern.\"\"\"\n        xml_files = glob.glob(os.path.join(\n            settings.ASSETS_DIR, self.xml_path_pattern\n        ))\n        assert xml_files, 'Files on path {} does not exist.'.format(\n            self.xml_path_pattern\n        )\n        return [ElementTree.parse(file) for file in xml_files]\n\n    @property\n    def xpaths(self):\n        \"\"\"Get xpath queries for xml.\"\"\"\n        return {\n            name: query.format(self.namespace)\n            for name, query in self.xpath_queries.items()\n        }\n\n    def get_data(self) -> Iterator:\n        \"\"\"\n        Get data from xml files.\n\n        Example files with products names or prices.\n        \"\"\"\n        return chain.from_iterable(\n            self.fetch_callback(file, self)\n            for file in self.parsed_files\n        )\n\n\n@contextmanager\ndef collect_errors(error_types: tuple):\n    errors = []\n\n    @contextmanager\n    def collect():\n        try:\n            yield\n        except error_types as error:\n            errors.append(error)\n    yield collect\n    if errors:\n        raise errors[0]\n\n\n@contextmanager\ndef download_catalog(destination):\n    \"\"\"Download catalog's xml files and delete after handle them.\"\"\"\n    wget_command = (\n        'wget -r -P {} ftp://{}:{}@{}/webdata'\n        ' 2>&1 | grep \"\u0432\u0440\u0435\u043c\u044f\\|time\\|Downloaded\"'.format(\n            destination,\n            settings.FTP_USER,\n            settings.FTP_PASS,\n            settings.FTP_IP,\n        )\n    )\n\n    subprocess.run(wget_command, shell=True)\n    assert os.path.exists(os.path.join(\n        destination, settings.FTP_IP)), 'Files do not downloaded...'\n    logger.info('Download catalog - completed...')\n\n    try:\n        yield\n    finally:\n        # remove downloaded data\n        shutil.rmtree(os.path.join(destination, settings.FTP_IP))\n\n\ndef report(error):\n    report_url = getattr(settings, 'SLACK_REPORT_URL', None)\n    if report_url is not None:\n        requests.post(\n            url=report_url,\n            json={\n                'text': '*\u041d\u0435 \u0443\u0434\u0430\u043b\u043e\u0441\u044c \u043e\u0431\u043d\u043e\u0432\u0438\u0442\u044c \u043a\u0430\u0442\u0430\u043b\u043e\u0433 Shopelectro.*\\n'\n                        '*\u0412\u0440\u0435\u043c\u044f*: {}\\n'\n                        '*\u041e\u0448\u0438\u0431\u043a\u0430*: {}'.format(time.ctime(), error),\n            }\n        )\n", "path": "shopelectro/management/commands/_update_catalog/utils.py"}, {"content": "import logging\nfrom copy import deepcopy\nfrom typing import Iterator, Dict\nfrom xml.etree.ElementTree import Element\n\nfrom django.db import transaction\n\nfrom shopelectro.management.commands._update_catalog.utils import (\n    Data, is_correct_uuid, UUID_TYPE, XmlFile\n)\nfrom shopelectro.models import Tag, TagGroup\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef fetch_tags(root: Element, config: XmlFile):\n    def get_uuid_name_pair(\n            element: Element,\n            uuid_xpath: str,\n            name_xpath: str,\n    ):\n        uuid = element.find(uuid_xpath).text\n        name = element.find(name_xpath).text\n\n        return uuid, name\n\n    tag_groups = root.findall(config.xpaths['tag_groups'])\n    for group in tag_groups:\n        group_uuid, group_name = get_uuid_name_pair(\n            group,\n            config.xpaths['tag_group_uuid'],\n            config.xpaths['tag_group_name'],\n        )\n\n        tags = group.findall(config.xpaths['tags'])\n        tags_data = (\n            get_uuid_name_pair(\n                tag,\n                config.xpaths['tag_uuid'],\n                config.xpaths['tag_name'],\n            ) for tag in tags\n        )\n\n        yield group_uuid, {\n            'name': group_name,\n            'tags_data': tags_data,\n        }\n\ntag_file = XmlFile(\n    fetch_callback=fetch_tags,\n    xml_path_pattern='**/webdata/**/properties/**/import*.xml',\n    xpath_queries={\n        'tag_groups': './/{}\u0421\u0432\u043e\u0439\u0441\u0442\u0432\u0430/',\n        'tag_group_uuid': '.{}\u0418\u0434',\n        'tag_group_name': '.{}\u041d\u0430\u0438\u043c\u0435\u043d\u043e\u0432\u0430\u043d\u0438\u0435',\n        'tags': '.{}\u0412\u0430\u0440\u0438\u0430\u043d\u0442\u044b\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0439/',\n        'tag_name': '.{}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435',\n        'tag_uuid': '.{}\u0418\u0434\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u044f',\n    },\n)\n\n\n@transaction.atomic\ndef create_or_update(data: Dict[UUID_TYPE, Data]):\n    group_data = deepcopy(data)\n\n    created_groups_count = 0\n    created_tags_count = 0\n\n    for group_uuid, data_ in group_data.items():\n        tags = data_.pop('tags')\n\n        group, group_created = TagGroup.objects.update_or_create(\n            uuid=group_uuid, defaults=data_\n        )\n\n        created_groups_count += int(group_created)\n\n        for tag_uuid, tag_data in tags.items():\n            _, tag_created = Tag.objects.update_or_create(\n                uuid=tag_uuid,\n                defaults={**tag_data, 'group': group}\n            )\n\n            created_tags_count += int(tag_created)\n\n    logger.info(f'{created_groups_count} tag groups were created.')\n    logger.info(f'{created_tags_count} tags were created.')\n\n\ndef prepare_data(group_data: Iterator) -> Dict[UUID_TYPE, Data]:\n    def assembly_structure(group_uuid: str, group_data_: dict):\n        tags_data = group_data_.pop('tags_data', [])\n        tags = {\n            tag_uuid: {'name': tag_name}\n            for tag_uuid, tag_name in tags_data\n            if is_correct_uuid(tag_uuid)\n        }\n\n        return (\n            group_uuid, {\n                **group_data_,\n                'tags': tags\n            }\n        )\n\n    return dict(\n        assembly_structure(group_uuid, data)\n        for group_uuid, data in group_data\n        if is_correct_uuid(group_uuid)\n    )\n\n\ndef main(*args, **kwargs):\n    cleared_group_data = prepare_data(tag_file.get_data())\n    create_or_update(cleared_group_data)\n", "path": "shopelectro/management/commands/_update_catalog/update_tags.py"}]}