We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
500 ошибка в админке
https://www.shopelectro.ru/admin/shopelectro/productpage/?has_category=no

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `shopelectro/management/commands/_update_catalog/update_products.py`
Content:
```
1 import logging
2 import typing
3 from collections import defaultdict
4 from copy import deepcopy
5 from functools import reduce
6 from itertools import chain
7 from typing import Dict, Iterator, List
8 from xml.etree.ElementTree import Element
9 
10 from django.conf import settings
11 from django.contrib.auth.models import User
12 from django.core.mail import send_mail
13 from django.db import transaction
14 from django.db.models import QuerySet
15 from django.template.loader import render_to_string
16 
17 from shopelectro.management.commands._update_catalog.utils import (
18     XmlFile, is_correct_uuid, NOT_SAVE_TEMPLATE, UUID, Data, floor
19 )
20 from shopelectro.models import Product, ProductPage, Tag
21 
22 
23 logger = logging.getLogger(__name__)
24 
25 
26 def fetch_products(root: Element, config: XmlFile) -> Iterator:
27     product_els = root.findall(config.xpaths['products'])
28     for product_el in product_els:
29         name = product_el.find(config.xpaths['name']).text
30         uuid = product_el.find(config.xpaths['uuid']).text
31         vendor_code = product_el.find(
32             config.xpaths['vendor_code']
33         ).text.lstrip('0')
34         content = product_el.find(config.xpaths['page_content']).text or ''
35 
36         tag_value_els = (
37             tag_el.find(config.xpaths['tag_value_uuid'])
38             for tag_el in product_el.findall(config.xpaths['tags'])
39             if tag_el is not None
40         )
41 
42         tag_uuids = list(filter(is_correct_uuid, (
43             tag_value.text
44             for tag_value in tag_value_els
45             # should use 'is not None', because __bool__ does not defined
46             if tag_value is not None
47         )))
48 
49         tags = Tag.objects.filter(uuid__in=tag_uuids)
50 
51         yield uuid, {
52             'name': name,
53             'vendor_code': vendor_code,
54             'page': {
55                 'content': content
56             },
57             'tags': tags
58         }
59 
60 
61 def fetch_prices(root: Element, config) -> typing.Iterator:
62     def get_price_values(prices_el):
63         return list(sorted(
64             float(price_el.find(config.xpaths['price']).text)
65             for price_el in prices_el.findall(config.xpaths['prices'])
66         ))
67 
68     def multiply(prices: typing.List[float]):
69         def floor_prices(prices, precision: floor):
70             return [
71                 floor(price * multiplier, precision)
72                 for price, multiplier in zip(prices, settings.PRICE_MULTIPLIERS)
73             ]
74         *wholesale_prices, retail_price = prices
75         return (
76             floor_prices(wholesale_prices, precision=2) +
77             floor_prices([retail_price], precision=0)
78         )
79 
80     product_price_els = root.findall(config.xpaths['product_prices'])
81     for prices_el in product_price_els:
82         product_uuid = prices_el.find(config.xpaths['product_uuid']).text
83         prices = dict(zip(
84             config.extra_options['price_types'],
85             multiply(get_price_values(prices_el))
86         ))
87         yield product_uuid, prices
88 
89 
90 def fetch_in_stock(root: Element, config: XmlFile) -> Iterator:
91     product_els = root.findall(config.xpaths['products'])
92     for product_el in product_els:
93         uuid = product_el.find(config.xpaths['product_uuid']).text
94         in_stock = product_el.find(config.xpaths['in_stock']).text
95 
96         if not (in_stock.isdigit() and int(in_stock) >= 0):
97             in_stock = 0
98 
99         yield uuid, {
100             'in_stock': in_stock,
101         }
102 
103 
104 product_file = XmlFile(
105     fetch_callback=fetch_products,
106     xml_path_pattern='**/webdata/**/goods/**/import*.xml',
107     xpath_queries={
108         'products': './/{}Товары/',
109         'name': '.{}Наименование',
110         'uuid': '.{}Ид',
111         'page_content': '.{}Описание',
112         'tags': '.{}ЗначенияСвойств/',
113         'tag_value_uuid': '.{}Значение',
114         'vendor_code': '.{0}ЗначенияРеквизитов/{0}ЗначениеРеквизита'
115                        '[{0}Наименование="Код"]/{0}Значение',
116     },
117 )
118 
119 price_file = XmlFile(
120     fetch_callback=fetch_prices,
121     xml_path_pattern='**/webdata/**/goods/**/prices*.xml',
122     xpath_queries={
123         'product_prices': './/{}Предложения/',
124         'product_uuid': '.{}Ид',
125         'prices': '.{}Цены/',
126         'price': '.{}ЦенаЗаЕдиницу',
127     },
128     extra_options={
129         'price_types': [
130             'purchase_price', 'wholesale_large', 'wholesale_medium',
131             'wholesale_small', 'price',
132         ],
133     },
134 )
135 
136 
137 in_stock_file = XmlFile(
138     fetch_callback=fetch_in_stock,
139     xml_path_pattern='**/webdata/**/goods/**/rests*.xml',
140     xpath_queries={
141         'products': './/{}Предложения/',
142         'product_uuid': '.{}Ид',
143         'in_stock': './/{}Количество',
144     },
145 )
146 
147 
148 def merge_data(*data) -> Dict[UUID, Data]:
149     """
150     Merge data from xml files with different structure.
151 
152     Example: files with product names and prices.
153     """
154     product_data = defaultdict(dict)
155     for key, data in chain.from_iterable(filter(None, data)):
156         product_data[key].update(data)
157 
158     return product_data
159 
160 
161 def clean_data(data: Dict[UUID, Data]):
162     def has_all_prices(_, product_data):
163         price_types = price_file.extra_options['price_types']
164         has = all(
165             product_data.get(price_type)
166             for price_type in price_types
167         )
168         if not has:
169             logger.info(NOT_SAVE_TEMPLATE.format(
170                 entity='Product',
171                 name=product_data['name'],
172                 field='price'
173             ))
174         return has
175 
176     def has_vendor_code(_, product_data):
177         has = bool(product_data['vendor_code'])
178 
179         if not has:
180             logger.info(NOT_SAVE_TEMPLATE.format(
181                 entity='Product',
182                 name=product_data['name'],
183                 field='vendor_code'
184             ))
185 
186         return has
187 
188     def has_uuid(uuid, product_data):
189         has = is_correct_uuid(uuid)
190         if not has:
191             logger.info(NOT_SAVE_TEMPLATE.format(
192                 entity='Product',
193                 name=product_data['name'],
194                 field='uuid'
195             ))
196         return has
197 
198     def filter_(product_data):
199         return all(
200             f(*product_data)
201             for f in [has_all_prices, has_uuid, has_vendor_code]
202         )
203 
204     cleaned_data = dict(
205         product_data
206         for product_data in data.items()
207         if filter_(product_data)
208     )
209 
210     return cleaned_data
211 
212 
213 def report(recipients=None, message=None):
214     message = message or render_to_string('report.html')
215 
216     user_query = (
217         User.objects
218             .filter(is_staff=True, is_superuser=False, is_active=True, email__isnull=False)
219     )
220 
221     recipient_list = recipients or [user.email for user in user_query]
222 
223     if recipient_list:
224         send_mail(
225             subject='Обновления каталога товаров',
226             message=message,
227             from_email=settings.EMAIL_SENDER,
228             recipient_list=recipient_list,
229             html_message=message,
230         )
231 
232         logger.info('Sent message to {}'.format(
233             reduce(lambda x, y: '{}, {}'.format(x, y), recipient_list)
234         ))
235 
236 
237 @transaction.atomic
238 def delete(data: Dict[UUID, Data]):
239     uuids = list(data)
240     pages_to_deactivate = ProductPage.objects.exclude(
241         shopelectro_product__uuid__in=uuids)
242     pages_to_deactivate.update(is_active=False)
243     deactivated_count = pages_to_deactivate.count()
244     logger.info(f'{deactivated_count} products and {deactivated_count} pages were deleted.')
245 
246 
247 @transaction.atomic
248 def update(data: Dict[UUID, Data]) -> QuerySet:
249     def save(product, field, value):
250         if field == 'name' and getattr(product, field, None):
251             return
252         elif field == 'page':
253             for page_field, page_value in value.items():
254                 if not getattr(product.page, page_field, ''):
255                     setattr(product.page, page_field, page_value)
256         elif field == 'tags':
257             product.tags = merge(list(product.tags.all()), value)
258         else:
259             setattr(product, field, value)
260 
261     def merge(left: List, right: List) -> List:
262         """Merge two arrays with order preserving."""
263         # Dirty patch for preserving tags, appended from admin.
264         # Still waiting 1C throwing out.
265         return left + [e for e in right if e not in left]
266 
267     products = Product.objects.filter(uuid__in=data)
268 
269     for product in products:
270         product_data = data[str(product.uuid)]
271         for field, value in product_data.items():
272             save(product, field, value)
273         product.save()
274 
275     logger.info('{} products were updated.'.format(products.count()))
276     return products
277 
278 
279 @transaction.atomic
280 def create(data: Dict[UUID, Data], updated_products: QuerySet) -> QuerySet:
281     data = deepcopy(data)
282     uuids_for_create = (
283         set(data) - set(str(product.uuid) for product in updated_products)
284     )
285 
286     for uuid in uuids_for_create:
287         product_data = data.get(uuid)
288         tags = product_data.pop('tags', {})
289         page_data = product_data.pop('page', {})
290 
291         new_product = Product.objects.create(**product_data, uuid=uuid)
292         new_product.tags.set(tags)
293         for field, value in page_data.items():
294             setattr(new_product.page, field, value)
295         new_product.page.save()
296 
297     created_products = Product.objects.filter(uuid__in=uuids_for_create)
298 
299     logger.info('{} products were created.'.format(created_products.count()))
300     return created_products
301 
302 
303 class UpdateProductError(Exception):
304     pass
305 
306 
307 def main(*args, **kwargs):
308     cleaned_product_data = clean_data(merge_data(
309         product_file.get_data(),
310         price_file.get_data(),
311         in_stock_file.get_data(),
312     ))
313 
314     if not cleaned_product_data:
315 
316         parsed_files = {
317             'product_files': list(product_file.parsed_files),
318             'price_files': list(price_file.parsed_files),
319             'in_stock_files': list(in_stock_file.parsed_files),
320         }
321 
322         if not any(parsed_files.values()):
323             message = 'Files does not exist: {}'.format(parsed_files)
324         else:
325             # file structure is unstable.
326             # You should adapt code for it if you got this error
327             message = (
328                 'The file structure has changed'
329                 ' or it does not contain the required data.'
330             )
331 
332         raise UpdateProductError(message)
333 
334     delete(cleaned_product_data)
335     updated_products = update(cleaned_product_data)
336     created_products = create(cleaned_product_data, updated_products)
337 
338     if created_products.exists():
339         report(kwargs['recipients'])
340
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/shopelectro/management/commands/_update_catalog/update_products.py b/shopelectro/management/commands/_update_catalog/update_products.py
--- a/shopelectro/management/commands/_update_catalog/update_products.py
+++ b/shopelectro/management/commands/_update_catalog/update_products.py
@@ -236,6 +236,11 @@
 
 @transaction.atomic
 def delete(data: Dict[UUID, Data]):
+    """
+    Deactivate stale pages.
+
+    Deactivate all pages that are still in db, but already not in `data`.
+    """
     uuids = list(data)
     pages_to_deactivate = ProductPage.objects.exclude(
         shopelectro_product__uuid__in=uuids)

{"golden_diff": "diff --git a/shopelectro/management/commands/_update_catalog/update_products.py b/shopelectro/management/commands/_update_catalog/update_products.py\n--- a/shopelectro/management/commands/_update_catalog/update_products.py\n+++ b/shopelectro/management/commands/_update_catalog/update_products.py\n@@ -236,6 +236,11 @@\n \n @transaction.atomic\n def delete(data: Dict[UUID, Data]):\n+    \"\"\"\n+    Deactivate stale pages.\n+\n+    Deactivate all pages that are still in db, but already not in `data`.\n+    \"\"\"\n     uuids = list(data)\n     pages_to_deactivate = ProductPage.objects.exclude(\n         shopelectro_product__uuid__in=uuids)\n", "issue": "500 \u043e\u0448\u0438\u0431\u043a\u0430 \u0432 \u0430\u0434\u043c\u0438\u043d\u043a\u0435\nhttps://www.shopelectro.ru/admin/shopelectro/productpage/?has_category=no\n", "before_files": [{"content": "import logging\nimport typing\nfrom collections import defaultdict\nfrom copy import deepcopy\nfrom functools import reduce\nfrom itertools import chain\nfrom typing import Dict, Iterator, List\nfrom xml.etree.ElementTree import Element\n\nfrom django.conf import settings\nfrom django.contrib.auth.models import User\nfrom django.core.mail import send_mail\nfrom django.db import transaction\nfrom django.db.models import QuerySet\nfrom django.template.loader import render_to_string\n\nfrom shopelectro.management.commands._update_catalog.utils import (\n    XmlFile, is_correct_uuid, NOT_SAVE_TEMPLATE, UUID, Data, floor\n)\nfrom shopelectro.models import Product, ProductPage, Tag\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef fetch_products(root: Element, config: XmlFile) -> Iterator:\n    product_els = root.findall(config.xpaths['products'])\n    for product_el in product_els:\n        name = product_el.find(config.xpaths['name']).text\n        uuid = product_el.find(config.xpaths['uuid']).text\n        vendor_code = product_el.find(\n            config.xpaths['vendor_code']\n        ).text.lstrip('0')\n        content = product_el.find(config.xpaths['page_content']).text or ''\n\n        tag_value_els = (\n            tag_el.find(config.xpaths['tag_value_uuid'])\n            for tag_el in product_el.findall(config.xpaths['tags'])\n            if tag_el is not None\n        )\n\n        tag_uuids = list(filter(is_correct_uuid, (\n            tag_value.text\n            for tag_value in tag_value_els\n            # should use 'is not None', because __bool__ does not defined\n            if tag_value is not None\n        )))\n\n        tags = Tag.objects.filter(uuid__in=tag_uuids)\n\n        yield uuid, {\n            'name': name,\n            'vendor_code': vendor_code,\n            'page': {\n                'content': content\n            },\n            'tags': tags\n        }\n\n\ndef fetch_prices(root: Element, config) -> typing.Iterator:\n    def get_price_values(prices_el):\n        return list(sorted(\n            float(price_el.find(config.xpaths['price']).text)\n            for price_el in prices_el.findall(config.xpaths['prices'])\n        ))\n\n    def multiply(prices: typing.List[float]):\n        def floor_prices(prices, precision: floor):\n            return [\n                floor(price * multiplier, precision)\n                for price, multiplier in zip(prices, settings.PRICE_MULTIPLIERS)\n            ]\n        *wholesale_prices, retail_price = prices\n        return (\n            floor_prices(wholesale_prices, precision=2) +\n            floor_prices([retail_price], precision=0)\n        )\n\n    product_price_els = root.findall(config.xpaths['product_prices'])\n    for prices_el in product_price_els:\n        product_uuid = prices_el.find(config.xpaths['product_uuid']).text\n        prices = dict(zip(\n            config.extra_options['price_types'],\n            multiply(get_price_values(prices_el))\n        ))\n        yield product_uuid, prices\n\n\ndef fetch_in_stock(root: Element, config: XmlFile) -> Iterator:\n    product_els = root.findall(config.xpaths['products'])\n    for product_el in product_els:\n        uuid = product_el.find(config.xpaths['product_uuid']).text\n        in_stock = product_el.find(config.xpaths['in_stock']).text\n\n        if not (in_stock.isdigit() and int(in_stock) >= 0):\n            in_stock = 0\n\n        yield uuid, {\n            'in_stock': in_stock,\n        }\n\n\nproduct_file = XmlFile(\n    fetch_callback=fetch_products,\n    xml_path_pattern='**/webdata/**/goods/**/import*.xml',\n    xpath_queries={\n        'products': './/{}\u0422\u043e\u0432\u0430\u0440\u044b/',\n        'name': '.{}\u041d\u0430\u0438\u043c\u0435\u043d\u043e\u0432\u0430\u043d\u0438\u0435',\n        'uuid': '.{}\u0418\u0434',\n        'page_content': '.{}\u041e\u043f\u0438\u0441\u0430\u043d\u0438\u0435',\n        'tags': '.{}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u044f\u0421\u0432\u043e\u0439\u0441\u0442\u0432/',\n        'tag_value_uuid': '.{}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435',\n        'vendor_code': '.{0}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u044f\u0420\u0435\u043a\u0432\u0438\u0437\u0438\u0442\u043e\u0432/{0}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435\u0420\u0435\u043a\u0432\u0438\u0437\u0438\u0442\u0430'\n                       '[{0}\u041d\u0430\u0438\u043c\u0435\u043d\u043e\u0432\u0430\u043d\u0438\u0435=\"\u041a\u043e\u0434\"]/{0}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435',\n    },\n)\n\nprice_file = XmlFile(\n    fetch_callback=fetch_prices,\n    xml_path_pattern='**/webdata/**/goods/**/prices*.xml',\n    xpath_queries={\n        'product_prices': './/{}\u041f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u044f/',\n        'product_uuid': '.{}\u0418\u0434',\n        'prices': '.{}\u0426\u0435\u043d\u044b/',\n        'price': '.{}\u0426\u0435\u043d\u0430\u0417\u0430\u0415\u0434\u0438\u043d\u0438\u0446\u0443',\n    },\n    extra_options={\n        'price_types': [\n            'purchase_price', 'wholesale_large', 'wholesale_medium',\n            'wholesale_small', 'price',\n        ],\n    },\n)\n\n\nin_stock_file = XmlFile(\n    fetch_callback=fetch_in_stock,\n    xml_path_pattern='**/webdata/**/goods/**/rests*.xml',\n    xpath_queries={\n        'products': './/{}\u041f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u044f/',\n        'product_uuid': '.{}\u0418\u0434',\n        'in_stock': './/{}\u041a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e',\n    },\n)\n\n\ndef merge_data(*data) -> Dict[UUID, Data]:\n    \"\"\"\n    Merge data from xml files with different structure.\n\n    Example: files with product names and prices.\n    \"\"\"\n    product_data = defaultdict(dict)\n    for key, data in chain.from_iterable(filter(None, data)):\n        product_data[key].update(data)\n\n    return product_data\n\n\ndef clean_data(data: Dict[UUID, Data]):\n    def has_all_prices(_, product_data):\n        price_types = price_file.extra_options['price_types']\n        has = all(\n            product_data.get(price_type)\n            for price_type in price_types\n        )\n        if not has:\n            logger.info(NOT_SAVE_TEMPLATE.format(\n                entity='Product',\n                name=product_data['name'],\n                field='price'\n            ))\n        return has\n\n    def has_vendor_code(_, product_data):\n        has = bool(product_data['vendor_code'])\n\n        if not has:\n            logger.info(NOT_SAVE_TEMPLATE.format(\n                entity='Product',\n                name=product_data['name'],\n                field='vendor_code'\n            ))\n\n        return has\n\n    def has_uuid(uuid, product_data):\n        has = is_correct_uuid(uuid)\n        if not has:\n            logger.info(NOT_SAVE_TEMPLATE.format(\n                entity='Product',\n                name=product_data['name'],\n                field='uuid'\n            ))\n        return has\n\n    def filter_(product_data):\n        return all(\n            f(*product_data)\n            for f in [has_all_prices, has_uuid, has_vendor_code]\n        )\n\n    cleaned_data = dict(\n        product_data\n        for product_data in data.items()\n        if filter_(product_data)\n    )\n\n    return cleaned_data\n\n\ndef report(recipients=None, message=None):\n    message = message or render_to_string('report.html')\n\n    user_query = (\n        User.objects\n            .filter(is_staff=True, is_superuser=False, is_active=True, email__isnull=False)\n    )\n\n    recipient_list = recipients or [user.email for user in user_query]\n\n    if recipient_list:\n        send_mail(\n            subject='\u041e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u044f \u043a\u0430\u0442\u0430\u043b\u043e\u0433\u0430 \u0442\u043e\u0432\u0430\u0440\u043e\u0432',\n            message=message,\n            from_email=settings.EMAIL_SENDER,\n            recipient_list=recipient_list,\n            html_message=message,\n        )\n\n        logger.info('Sent message to {}'.format(\n            reduce(lambda x, y: '{}, {}'.format(x, y), recipient_list)\n        ))\n\n\n@transaction.atomic\ndef delete(data: Dict[UUID, Data]):\n    uuids = list(data)\n    pages_to_deactivate = ProductPage.objects.exclude(\n        shopelectro_product__uuid__in=uuids)\n    pages_to_deactivate.update(is_active=False)\n    deactivated_count = pages_to_deactivate.count()\n    logger.info(f'{deactivated_count} products and {deactivated_count} pages were deleted.')\n\n\n@transaction.atomic\ndef update(data: Dict[UUID, Data]) -> QuerySet:\n    def save(product, field, value):\n        if field == 'name' and getattr(product, field, None):\n            return\n        elif field == 'page':\n            for page_field, page_value in value.items():\n                if not getattr(product.page, page_field, ''):\n                    setattr(product.page, page_field, page_value)\n        elif field == 'tags':\n            product.tags = merge(list(product.tags.all()), value)\n        else:\n            setattr(product, field, value)\n\n    def merge(left: List, right: List) -> List:\n        \"\"\"Merge two arrays with order preserving.\"\"\"\n        # Dirty patch for preserving tags, appended from admin.\n        # Still waiting 1C throwing out.\n        return left + [e for e in right if e not in left]\n\n    products = Product.objects.filter(uuid__in=data)\n\n    for product in products:\n        product_data = data[str(product.uuid)]\n        for field, value in product_data.items():\n            save(product, field, value)\n        product.save()\n\n    logger.info('{} products were updated.'.format(products.count()))\n    return products\n\n\n@transaction.atomic\ndef create(data: Dict[UUID, Data], updated_products: QuerySet) -> QuerySet:\n    data = deepcopy(data)\n    uuids_for_create = (\n        set(data) - set(str(product.uuid) for product in updated_products)\n    )\n\n    for uuid in uuids_for_create:\n        product_data = data.get(uuid)\n        tags = product_data.pop('tags', {})\n        page_data = product_data.pop('page', {})\n\n        new_product = Product.objects.create(**product_data, uuid=uuid)\n        new_product.tags.set(tags)\n        for field, value in page_data.items():\n            setattr(new_product.page, field, value)\n        new_product.page.save()\n\n    created_products = Product.objects.filter(uuid__in=uuids_for_create)\n\n    logger.info('{} products were created.'.format(created_products.count()))\n    return created_products\n\n\nclass UpdateProductError(Exception):\n    pass\n\n\ndef main(*args, **kwargs):\n    cleaned_product_data = clean_data(merge_data(\n        product_file.get_data(),\n        price_file.get_data(),\n        in_stock_file.get_data(),\n    ))\n\n    if not cleaned_product_data:\n\n        parsed_files = {\n            'product_files': list(product_file.parsed_files),\n            'price_files': list(price_file.parsed_files),\n            'in_stock_files': list(in_stock_file.parsed_files),\n        }\n\n        if not any(parsed_files.values()):\n            message = 'Files does not exist: {}'.format(parsed_files)\n        else:\n            # file structure is unstable.\n            # You should adapt code for it if you got this error\n            message = (\n                'The file structure has changed'\n                ' or it does not contain the required data.'\n            )\n\n        raise UpdateProductError(message)\n\n    delete(cleaned_product_data)\n    updated_products = update(cleaned_product_data)\n    created_products = create(cleaned_product_data, updated_products)\n\n    if created_products.exists():\n        report(kwargs['recipients'])\n", "path": "shopelectro/management/commands/_update_catalog/update_products.py"}], "after_files": [{"content": "import logging\nimport typing\nfrom collections import defaultdict\nfrom copy import deepcopy\nfrom functools import reduce\nfrom itertools import chain\nfrom typing import Dict, Iterator, List\nfrom xml.etree.ElementTree import Element\n\nfrom django.conf import settings\nfrom django.contrib.auth.models import User\nfrom django.core.mail import send_mail\nfrom django.db import transaction\nfrom django.db.models import QuerySet\nfrom django.template.loader import render_to_string\n\nfrom shopelectro.management.commands._update_catalog.utils import (\n    XmlFile, is_correct_uuid, NOT_SAVE_TEMPLATE, UUID, Data, floor\n)\nfrom shopelectro.models import Product, ProductPage, Tag\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef fetch_products(root: Element, config: XmlFile) -> Iterator:\n    product_els = root.findall(config.xpaths['products'])\n    for product_el in product_els:\n        name = product_el.find(config.xpaths['name']).text\n        uuid = product_el.find(config.xpaths['uuid']).text\n        vendor_code = product_el.find(\n            config.xpaths['vendor_code']\n        ).text.lstrip('0')\n        content = product_el.find(config.xpaths['page_content']).text or ''\n\n        tag_value_els = (\n            tag_el.find(config.xpaths['tag_value_uuid'])\n            for tag_el in product_el.findall(config.xpaths['tags'])\n            if tag_el is not None\n        )\n\n        tag_uuids = list(filter(is_correct_uuid, (\n            tag_value.text\n            for tag_value in tag_value_els\n            # should use 'is not None', because __bool__ does not defined\n            if tag_value is not None\n        )))\n\n        tags = Tag.objects.filter(uuid__in=tag_uuids)\n\n        yield uuid, {\n            'name': name,\n            'vendor_code': vendor_code,\n            'page': {\n                'content': content\n            },\n            'tags': tags\n        }\n\n\ndef fetch_prices(root: Element, config) -> typing.Iterator:\n    def get_price_values(prices_el):\n        return list(sorted(\n            float(price_el.find(config.xpaths['price']).text)\n            for price_el in prices_el.findall(config.xpaths['prices'])\n        ))\n\n    def multiply(prices: typing.List[float]):\n        def floor_prices(prices, precision: floor):\n            return [\n                floor(price * multiplier, precision)\n                for price, multiplier in zip(prices, settings.PRICE_MULTIPLIERS)\n            ]\n        *wholesale_prices, retail_price = prices\n        return (\n            floor_prices(wholesale_prices, precision=2) +\n            floor_prices([retail_price], precision=0)\n        )\n\n    product_price_els = root.findall(config.xpaths['product_prices'])\n    for prices_el in product_price_els:\n        product_uuid = prices_el.find(config.xpaths['product_uuid']).text\n        prices = dict(zip(\n            config.extra_options['price_types'],\n            multiply(get_price_values(prices_el))\n        ))\n        yield product_uuid, prices\n\n\ndef fetch_in_stock(root: Element, config: XmlFile) -> Iterator:\n    product_els = root.findall(config.xpaths['products'])\n    for product_el in product_els:\n        uuid = product_el.find(config.xpaths['product_uuid']).text\n        in_stock = product_el.find(config.xpaths['in_stock']).text\n\n        if not (in_stock.isdigit() and int(in_stock) >= 0):\n            in_stock = 0\n\n        yield uuid, {\n            'in_stock': in_stock,\n        }\n\n\nproduct_file = XmlFile(\n    fetch_callback=fetch_products,\n    xml_path_pattern='**/webdata/**/goods/**/import*.xml',\n    xpath_queries={\n        'products': './/{}\u0422\u043e\u0432\u0430\u0440\u044b/',\n        'name': '.{}\u041d\u0430\u0438\u043c\u0435\u043d\u043e\u0432\u0430\u043d\u0438\u0435',\n        'uuid': '.{}\u0418\u0434',\n        'page_content': '.{}\u041e\u043f\u0438\u0441\u0430\u043d\u0438\u0435',\n        'tags': '.{}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u044f\u0421\u0432\u043e\u0439\u0441\u0442\u0432/',\n        'tag_value_uuid': '.{}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435',\n        'vendor_code': '.{0}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u044f\u0420\u0435\u043a\u0432\u0438\u0437\u0438\u0442\u043e\u0432/{0}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435\u0420\u0435\u043a\u0432\u0438\u0437\u0438\u0442\u0430'\n                       '[{0}\u041d\u0430\u0438\u043c\u0435\u043d\u043e\u0432\u0430\u043d\u0438\u0435=\"\u041a\u043e\u0434\"]/{0}\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435',\n    },\n)\n\nprice_file = XmlFile(\n    fetch_callback=fetch_prices,\n    xml_path_pattern='**/webdata/**/goods/**/prices*.xml',\n    xpath_queries={\n        'product_prices': './/{}\u041f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u044f/',\n        'product_uuid': '.{}\u0418\u0434',\n        'prices': '.{}\u0426\u0435\u043d\u044b/',\n        'price': '.{}\u0426\u0435\u043d\u0430\u0417\u0430\u0415\u0434\u0438\u043d\u0438\u0446\u0443',\n    },\n    extra_options={\n        'price_types': [\n            'purchase_price', 'wholesale_large', 'wholesale_medium',\n            'wholesale_small', 'price',\n        ],\n    },\n)\n\n\nin_stock_file = XmlFile(\n    fetch_callback=fetch_in_stock,\n    xml_path_pattern='**/webdata/**/goods/**/rests*.xml',\n    xpath_queries={\n        'products': './/{}\u041f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u044f/',\n        'product_uuid': '.{}\u0418\u0434',\n        'in_stock': './/{}\u041a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e',\n    },\n)\n\n\ndef merge_data(*data) -> Dict[UUID, Data]:\n    \"\"\"\n    Merge data from xml files with different structure.\n\n    Example: files with product names and prices.\n    \"\"\"\n    product_data = defaultdict(dict)\n    for key, data in chain.from_iterable(filter(None, data)):\n        product_data[key].update(data)\n\n    return product_data\n\n\ndef clean_data(data: Dict[UUID, Data]):\n    def has_all_prices(_, product_data):\n        price_types = price_file.extra_options['price_types']\n        has = all(\n            product_data.get(price_type)\n            for price_type in price_types\n        )\n        if not has:\n            logger.info(NOT_SAVE_TEMPLATE.format(\n                entity='Product',\n                name=product_data['name'],\n                field='price'\n            ))\n        return has\n\n    def has_vendor_code(_, product_data):\n        has = bool(product_data['vendor_code'])\n\n        if not has:\n            logger.info(NOT_SAVE_TEMPLATE.format(\n                entity='Product',\n                name=product_data['name'],\n                field='vendor_code'\n            ))\n\n        return has\n\n    def has_uuid(uuid, product_data):\n        has = is_correct_uuid(uuid)\n        if not has:\n            logger.info(NOT_SAVE_TEMPLATE.format(\n                entity='Product',\n                name=product_data['name'],\n                field='uuid'\n            ))\n        return has\n\n    def filter_(product_data):\n        return all(\n            f(*product_data)\n            for f in [has_all_prices, has_uuid, has_vendor_code]\n        )\n\n    cleaned_data = dict(\n        product_data\n        for product_data in data.items()\n        if filter_(product_data)\n    )\n\n    return cleaned_data\n\n\ndef report(recipients=None, message=None):\n    message = message or render_to_string('report.html')\n\n    user_query = (\n        User.objects\n            .filter(is_staff=True, is_superuser=False, is_active=True, email__isnull=False)\n    )\n\n    recipient_list = recipients or [user.email for user in user_query]\n\n    if recipient_list:\n        send_mail(\n            subject='\u041e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u044f \u043a\u0430\u0442\u0430\u043b\u043e\u0433\u0430 \u0442\u043e\u0432\u0430\u0440\u043e\u0432',\n            message=message,\n            from_email=settings.EMAIL_SENDER,\n            recipient_list=recipient_list,\n            html_message=message,\n        )\n\n        logger.info('Sent message to {}'.format(\n            reduce(lambda x, y: '{}, {}'.format(x, y), recipient_list)\n        ))\n\n\n@transaction.atomic\ndef delete(data: Dict[UUID, Data]):\n    \"\"\"\n    Deactivate stale pages.\n\n    Deactivate all pages that are still in db, but already not in `data`.\n    \"\"\"\n    uuids = list(data)\n    pages_to_deactivate = ProductPage.objects.exclude(\n        shopelectro_product__uuid__in=uuids)\n    pages_to_deactivate.update(is_active=False)\n    deactivated_count = pages_to_deactivate.count()\n    logger.info(f'{deactivated_count} products and {deactivated_count} pages were deleted.')\n\n\n@transaction.atomic\ndef update(data: Dict[UUID, Data]) -> QuerySet:\n    def save(product, field, value):\n        if field == 'name' and getattr(product, field, None):\n            return\n        elif field == 'page':\n            for page_field, page_value in value.items():\n                if not getattr(product.page, page_field, ''):\n                    setattr(product.page, page_field, page_value)\n        elif field == 'tags':\n            product.tags = merge(list(product.tags.all()), value)\n        else:\n            setattr(product, field, value)\n\n    def merge(left: List, right: List) -> List:\n        \"\"\"Merge two arrays with order preserving.\"\"\"\n        # Dirty patch for preserving tags, appended from admin.\n        # Still waiting 1C throwing out.\n        return left + [e for e in right if e not in left]\n\n    products = Product.objects.filter(uuid__in=data)\n\n    for product in products:\n        product_data = data[str(product.uuid)]\n        for field, value in product_data.items():\n            save(product, field, value)\n        product.save()\n\n    logger.info('{} products were updated.'.format(products.count()))\n    return products\n\n\n@transaction.atomic\ndef create(data: Dict[UUID, Data], updated_products: QuerySet) -> QuerySet:\n    data = deepcopy(data)\n    uuids_for_create = (\n        set(data) - set(str(product.uuid) for product in updated_products)\n    )\n\n    for uuid in uuids_for_create:\n        product_data = data.get(uuid)\n        tags = product_data.pop('tags', {})\n        page_data = product_data.pop('page', {})\n\n        new_product = Product.objects.create(**product_data, uuid=uuid)\n        new_product.tags.set(tags)\n        for field, value in page_data.items():\n            setattr(new_product.page, field, value)\n        new_product.page.save()\n\n    created_products = Product.objects.filter(uuid__in=uuids_for_create)\n\n    logger.info('{} products were created.'.format(created_products.count()))\n    return created_products\n\n\nclass UpdateProductError(Exception):\n    pass\n\n\ndef main(*args, **kwargs):\n    cleaned_product_data = clean_data(merge_data(\n        product_file.get_data(),\n        price_file.get_data(),\n        in_stock_file.get_data(),\n    ))\n\n    if not cleaned_product_data:\n\n        parsed_files = {\n            'product_files': list(product_file.parsed_files),\n            'price_files': list(price_file.parsed_files),\n            'in_stock_files': list(in_stock_file.parsed_files),\n        }\n\n        if not any(parsed_files.values()):\n            message = 'Files does not exist: {}'.format(parsed_files)\n        else:\n            # file structure is unstable.\n            # You should adapt code for it if you got this error\n            message = (\n                'The file structure has changed'\n                ' or it does not contain the required data.'\n            )\n\n        raise UpdateProductError(message)\n\n    delete(cleaned_product_data)\n    updated_products = update(cleaned_product_data)\n    created_products = create(cleaned_product_data, updated_products)\n\n    if created_products.exists():\n        report(kwargs['recipients'])\n", "path": "shopelectro/management/commands/_update_catalog/update_products.py"}]}