We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
cvat_format/extractor type casting value to float, causing issues
We have a text type attribute for a bounding box label in CVAT. The values for this attribute are all digits some of which start with zeros. Ex: 0012345, 0123456, 1234567, etc.

We are using datumaro to automatically do some checks on the annotations imported from CVAT. It looks like the extractor for CVAT is trying to typecast all imported numeric [attribute values to float](https://github.com/openvinotoolkit/datumaro/blob/6a2022ed467c7557fb2055c143452bc4ca7f6e86/datumaro/plugins/cvat_format/extractor.py#L96). This is causing an issue for us since this removes the leading zeros automatically from the value, changing user input.

Is there any reason behind typecasting the the attribute value to float? Can this typecasting step be deleted. If required a user can always do the typecasting later. 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `datumaro/plugins/cvat_format/extractor.py`
Content:
```
1 # Copyright (C) 2019-2020 Intel Corporation
2 #
3 # SPDX-License-Identifier: MIT
4 
5 from collections import OrderedDict
6 import os.path as osp
7 
8 from defusedxml import ElementTree
9 
10 from datumaro.components.extractor import (
11     AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, Points,
12     Polygon, PolyLine, SourceExtractor,
13 )
14 from datumaro.util.image import Image
15 
16 from .format import CvatPath
17 
18 
19 class CvatExtractor(SourceExtractor):
20     _SUPPORTED_SHAPES = ('box', 'polygon', 'polyline', 'points')
21 
22     def __init__(self, path, subset=None):
23         assert osp.isfile(path), path
24         rootpath = osp.dirname(path)
25         images_dir = ''
26         if osp.isdir(osp.join(rootpath, CvatPath.IMAGES_DIR)):
27             images_dir = osp.join(rootpath, CvatPath.IMAGES_DIR)
28         self._images_dir = images_dir
29         self._path = path
30 
31         if not subset:
32             subset = osp.splitext(osp.basename(path))[0]
33         super().__init__(subset=subset)
34 
35         items, categories = self._parse(path)
36         self._items = list(self._load_items(items).values())
37         self._categories = categories
38 
39     @classmethod
40     def _parse(cls, path):
41         context = ElementTree.iterparse(path, events=("start", "end"))
42         context = iter(context)
43 
44         categories, frame_size = cls._parse_meta(context)
45 
46         items = OrderedDict()
47 
48         track = None
49         shape = None
50         tag = None
51         attributes = None
52         image = None
53         for ev, el in context:
54             if ev == 'start':
55                 if el.tag == 'track':
56                     track = {
57                         'id': el.attrib['id'],
58                         'label': el.attrib.get('label'),
59                         'group': int(el.attrib.get('group_id', 0)),
60                         'height': frame_size[0],
61                         'width': frame_size[1],
62                     }
63                 elif el.tag == 'image':
64                     image = {
65                         'name': el.attrib.get('name'),
66                         'frame': el.attrib['id'],
67                         'width': el.attrib.get('width'),
68                         'height': el.attrib.get('height'),
69                     }
70                 elif el.tag in cls._SUPPORTED_SHAPES and (track or image):
71                     attributes = {}
72                     shape = {
73                         'type': None,
74                         'attributes': attributes,
75                     }
76                     if track:
77                         shape.update(track)
78                         shape['track_id'] = int(track['id'])
79                     if image:
80                         shape.update(image)
81                 elif el.tag == 'tag' and image:
82                     attributes = {}
83                     tag = {
84                         'frame': image['frame'],
85                         'attributes': attributes,
86                         'group': int(el.attrib.get('group_id', 0)),
87                         'label': el.attrib['label'],
88                     }
89             elif ev == 'end':
90                 if el.tag == 'attribute' and attributes is not None:
91                     attr_value = el.text or ''
92                     if el.text in ['true', 'false']:
93                         attr_value = attr_value == 'true'
94                     else:
95                         try:
96                             attr_value = float(attr_value)
97                         except ValueError:
98                             pass
99                     attributes[el.attrib['name']] = attr_value
100                 elif el.tag in cls._SUPPORTED_SHAPES:
101                     if track is not None:
102                         shape['frame'] = el.attrib['frame']
103                         shape['outside'] = (el.attrib.get('outside') == '1')
104                         shape['keyframe'] = (el.attrib.get('keyframe') == '1')
105                     if image is not None:
106                         shape['label'] = el.attrib.get('label')
107                         shape['group'] = int(el.attrib.get('group_id', 0))
108 
109                     shape['type'] = el.tag
110                     shape['occluded'] = (el.attrib.get('occluded') == '1')
111                     shape['z_order'] = int(el.attrib.get('z_order', 0))
112 
113                     if el.tag == 'box':
114                         shape['points'] = list(map(float, [
115                             el.attrib['xtl'], el.attrib['ytl'],
116                             el.attrib['xbr'], el.attrib['ybr'],
117                         ]))
118                     else:
119                         shape['points'] = []
120                         for pair in el.attrib['points'].split(';'):
121                             shape['points'].extend(map(float, pair.split(',')))
122 
123                     frame_desc = items.get(shape['frame'], {'annotations': []})
124                     frame_desc['annotations'].append(
125                         cls._parse_shape_ann(shape, categories))
126                     items[shape['frame']] = frame_desc
127                     shape = None
128 
129                 elif el.tag == 'tag':
130                     frame_desc = items.get(tag['frame'], {'annotations': []})
131                     frame_desc['annotations'].append(
132                         cls._parse_tag_ann(tag, categories))
133                     items[tag['frame']] = frame_desc
134                     tag = None
135                 elif el.tag == 'track':
136                     track = None
137                 elif el.tag == 'image':
138                     frame_desc = items.get(image['frame'], {'annotations': []})
139                     frame_desc.update({
140                         'name': image.get('name'),
141                         'height': image.get('height'),
142                         'width': image.get('width'),
143                     })
144                     items[image['frame']] = frame_desc
145                     image = None
146                 el.clear()
147 
148         return items, categories
149 
150     @staticmethod
151     def _parse_meta(context):
152         ev, el = next(context)
153         if not (ev == 'start' and el.tag == 'annotations'):
154             raise Exception("Unexpected token ")
155 
156         categories = {}
157 
158         frame_size = None
159         mode = None
160         labels = OrderedDict()
161         label = None
162 
163         # Recursive descent parser
164         el = None
165         states = ['annotations']
166         def accepted(expected_state, tag, next_state=None):
167             state = states[-1]
168             if state == expected_state and el is not None and el.tag == tag:
169                 if not next_state:
170                     next_state = tag
171                 states.append(next_state)
172                 return True
173             return False
174         def consumed(expected_state, tag):
175             state = states[-1]
176             if state == expected_state and el is not None and el.tag == tag:
177                 states.pop()
178                 return True
179             return False
180 
181         for ev, el in context:
182             if ev == 'start':
183                 if accepted('annotations', 'meta'): pass
184                 elif accepted('meta', 'task'): pass
185                 elif accepted('task', 'mode'): pass
186                 elif accepted('task', 'original_size'):
187                     frame_size = [None, None]
188                 elif accepted('original_size', 'height', next_state='frame_height'): pass
189                 elif accepted('original_size', 'width', next_state='frame_width'): pass
190                 elif accepted('task', 'labels'): pass
191                 elif accepted('labels', 'label'):
192                     label = { 'name': None, 'attributes': set() }
193                 elif accepted('label', 'name', next_state='label_name'): pass
194                 elif accepted('label', 'attributes'): pass
195                 elif accepted('attributes', 'attribute'): pass
196                 elif accepted('attribute', 'name', next_state='attr_name'): pass
197                 elif accepted('annotations', 'image') or \
198                      accepted('annotations', 'track') or \
199                      accepted('annotations', 'tag'):
200                     break
201                 else:
202                     pass
203             elif ev == 'end':
204                 if consumed('meta', 'meta'):
205                     break
206                 elif consumed('task', 'task'): pass
207                 elif consumed('mode', 'mode'):
208                     mode = el.text
209                 elif consumed('original_size', 'original_size'): pass
210                 elif consumed('frame_height', 'height'):
211                     frame_size[0] = int(el.text)
212                 elif consumed('frame_width', 'width'):
213                     frame_size[1] = int(el.text)
214                 elif consumed('label_name', 'name'):
215                     label['name'] = el.text
216                 elif consumed('attr_name', 'name'):
217                     label['attributes'].add(el.text)
218                 elif consumed('attribute', 'attribute'): pass
219                 elif consumed('attributes', 'attributes'): pass
220                 elif consumed('label', 'label'):
221                     labels[label['name']] = label['attributes']
222                     label = None
223                 elif consumed('labels', 'labels'): pass
224                 else:
225                     pass
226 
227         assert len(states) == 1 and states[0] == 'annotations', \
228             "Expected 'meta' section in the annotation file, path: %s" % states
229 
230         common_attrs = ['occluded']
231         if mode == 'interpolation':
232             common_attrs.append('keyframe')
233             common_attrs.append('outside')
234             common_attrs.append('track_id')
235 
236         label_cat = LabelCategories(attributes=common_attrs)
237         for label, attrs in labels.items():
238             label_cat.add(label, attributes=attrs)
239 
240         categories[AnnotationType.label] = label_cat
241 
242         return categories, frame_size
243 
244     @classmethod
245     def _parse_shape_ann(cls, ann, categories):
246         ann_id = ann.get('id', 0)
247         ann_type = ann['type']
248 
249         attributes = ann.get('attributes') or {}
250         if 'occluded' in categories[AnnotationType.label].attributes:
251             attributes['occluded'] = ann.get('occluded', False)
252         if 'outside' in ann:
253             attributes['outside'] = ann['outside']
254         if 'keyframe' in ann:
255             attributes['keyframe'] = ann['keyframe']
256         if 'track_id' in ann:
257             attributes['track_id'] = ann['track_id']
258 
259         group = ann.get('group')
260 
261         label = ann.get('label')
262         label_id = categories[AnnotationType.label].find(label)[0]
263 
264         z_order = ann.get('z_order', 0)
265         points = ann.get('points', [])
266 
267         if ann_type == 'polyline':
268             return PolyLine(points, label=label_id, z_order=z_order,
269                 id=ann_id, attributes=attributes, group=group)
270 
271         elif ann_type == 'polygon':
272             return Polygon(points, label=label_id, z_order=z_order,
273                 id=ann_id, attributes=attributes, group=group)
274 
275         elif ann_type == 'points':
276             return Points(points, label=label_id, z_order=z_order,
277                 id=ann_id, attributes=attributes, group=group)
278 
279         elif ann_type == 'box':
280             x, y = points[0], points[1]
281             w, h = points[2] - x, points[3] - y
282             return Bbox(x, y, w, h, label=label_id, z_order=z_order,
283                 id=ann_id, attributes=attributes, group=group)
284 
285         else:
286             raise NotImplementedError("Unknown annotation type '%s'" % ann_type)
287 
288     @classmethod
289     def _parse_tag_ann(cls, ann, categories):
290         label = ann.get('label')
291         label_id = categories[AnnotationType.label].find(label)[0]
292         group = ann.get('group')
293         attributes = ann.get('attributes')
294         return Label(label_id, attributes=attributes, group=group)
295 
296     def _load_items(self, parsed):
297         for frame_id, item_desc in parsed.items():
298             name = item_desc.get('name', 'frame_%06d.png' % int(frame_id))
299             image = osp.join(self._images_dir, name)
300             image_size = (item_desc.get('height'), item_desc.get('width'))
301             if all(image_size):
302                 image = Image(path=image, size=tuple(map(int, image_size)))
303 
304             parsed[frame_id] = DatasetItem(id=osp.splitext(name)[0],
305                 subset=self._subset, image=image,
306                 annotations=item_desc.get('annotations'),
307                 attributes={'frame': int(frame_id)})
308         return parsed
309 
310 class CvatImporter(Importer):
311     @classmethod
312     def find_sources(cls, path):
313         return cls._find_sources_recursive(path, '.xml', 'cvat')
314
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/datumaro/plugins/cvat_format/extractor.py b/datumaro/plugins/cvat_format/extractor.py
--- a/datumaro/plugins/cvat_format/extractor.py
+++ b/datumaro/plugins/cvat_format/extractor.py
@@ -41,7 +41,7 @@
         context = ElementTree.iterparse(path, events=("start", "end"))
         context = iter(context)
 
-        categories, frame_size = cls._parse_meta(context)
+        categories, frame_size, attribute_types = cls._parse_meta(context)
 
         items = OrderedDict()
 
@@ -89,9 +89,10 @@
             elif ev == 'end':
                 if el.tag == 'attribute' and attributes is not None:
                     attr_value = el.text or ''
+                    attr_type = attribute_types.get(el.attrib['name'])
                     if el.text in ['true', 'false']:
                         attr_value = attr_value == 'true'
-                    else:
+                    elif  attr_type is not None and attr_type != 'text':
                         try:
                             attr_value = float(attr_value)
                         except ValueError:
@@ -189,11 +190,12 @@
                 elif accepted('original_size', 'width', next_state='frame_width'): pass
                 elif accepted('task', 'labels'): pass
                 elif accepted('labels', 'label'):
-                    label = { 'name': None, 'attributes': set() }
+                    label = { 'name': None, 'attributes': [] }
                 elif accepted('label', 'name', next_state='label_name'): pass
                 elif accepted('label', 'attributes'): pass
                 elif accepted('attributes', 'attribute'): pass
                 elif accepted('attribute', 'name', next_state='attr_name'): pass
+                elif accepted('attribute', 'input_type', next_state='attr_type'): pass
                 elif accepted('annotations', 'image') or \
                      accepted('annotations', 'track') or \
                      accepted('annotations', 'tag'):
@@ -214,7 +216,9 @@
                 elif consumed('label_name', 'name'):
                     label['name'] = el.text
                 elif consumed('attr_name', 'name'):
-                    label['attributes'].add(el.text)
+                    label['attributes'].append({'name': el.text})
+                elif consumed('attr_type', 'input_type'):
+                    label['attributes'][-1]['input_type'] = el.text
                 elif consumed('attribute', 'attribute'): pass
                 elif consumed('attributes', 'attributes'): pass
                 elif consumed('label', 'label'):
@@ -234,12 +238,15 @@
             common_attrs.append('track_id')
 
         label_cat = LabelCategories(attributes=common_attrs)
+        attribute_types = {}
         for label, attrs in labels.items():
-            label_cat.add(label, attributes=attrs)
+            attr_names = {v['name'] for v in attrs}
+            label_cat.add(label, attributes=attr_names)
+            for attr in attrs:
+                attribute_types[attr['name']] = attr['input_type']
 
         categories[AnnotationType.label] = label_cat
-
-        return categories, frame_size
+        return categories, frame_size, attribute_types
 
     @classmethod
     def _parse_shape_ann(cls, ann, categories):

{"golden_diff": "diff --git a/datumaro/plugins/cvat_format/extractor.py b/datumaro/plugins/cvat_format/extractor.py\n--- a/datumaro/plugins/cvat_format/extractor.py\n+++ b/datumaro/plugins/cvat_format/extractor.py\n@@ -41,7 +41,7 @@\n         context = ElementTree.iterparse(path, events=(\"start\", \"end\"))\n         context = iter(context)\n \n-        categories, frame_size = cls._parse_meta(context)\n+        categories, frame_size, attribute_types = cls._parse_meta(context)\n \n         items = OrderedDict()\n \n@@ -89,9 +89,10 @@\n             elif ev == 'end':\n                 if el.tag == 'attribute' and attributes is not None:\n                     attr_value = el.text or ''\n+                    attr_type = attribute_types.get(el.attrib['name'])\n                     if el.text in ['true', 'false']:\n                         attr_value = attr_value == 'true'\n-                    else:\n+                    elif  attr_type is not None and attr_type != 'text':\n                         try:\n                             attr_value = float(attr_value)\n                         except ValueError:\n@@ -189,11 +190,12 @@\n                 elif accepted('original_size', 'width', next_state='frame_width'): pass\n                 elif accepted('task', 'labels'): pass\n                 elif accepted('labels', 'label'):\n-                    label = { 'name': None, 'attributes': set() }\n+                    label = { 'name': None, 'attributes': [] }\n                 elif accepted('label', 'name', next_state='label_name'): pass\n                 elif accepted('label', 'attributes'): pass\n                 elif accepted('attributes', 'attribute'): pass\n                 elif accepted('attribute', 'name', next_state='attr_name'): pass\n+                elif accepted('attribute', 'input_type', next_state='attr_type'): pass\n                 elif accepted('annotations', 'image') or \\\n                      accepted('annotations', 'track') or \\\n                      accepted('annotations', 'tag'):\n@@ -214,7 +216,9 @@\n                 elif consumed('label_name', 'name'):\n                     label['name'] = el.text\n                 elif consumed('attr_name', 'name'):\n-                    label['attributes'].add(el.text)\n+                    label['attributes'].append({'name': el.text})\n+                elif consumed('attr_type', 'input_type'):\n+                    label['attributes'][-1]['input_type'] = el.text\n                 elif consumed('attribute', 'attribute'): pass\n                 elif consumed('attributes', 'attributes'): pass\n                 elif consumed('label', 'label'):\n@@ -234,12 +238,15 @@\n             common_attrs.append('track_id')\n \n         label_cat = LabelCategories(attributes=common_attrs)\n+        attribute_types = {}\n         for label, attrs in labels.items():\n-            label_cat.add(label, attributes=attrs)\n+            attr_names = {v['name'] for v in attrs}\n+            label_cat.add(label, attributes=attr_names)\n+            for attr in attrs:\n+                attribute_types[attr['name']] = attr['input_type']\n \n         categories[AnnotationType.label] = label_cat\n-\n-        return categories, frame_size\n+        return categories, frame_size, attribute_types\n \n     @classmethod\n     def _parse_shape_ann(cls, ann, categories):\n", "issue": "cvat_format/extractor type casting value to float, causing issues\nWe have a text type attribute for a bounding box label in CVAT. The values for this attribute are all digits some of which start with zeros. Ex: 0012345, 0123456, 1234567, etc.\r\n\r\nWe are using datumaro to automatically do some checks on the annotations imported from CVAT. It looks like the extractor for CVAT is trying to typecast all imported numeric [attribute values to float](https://github.com/openvinotoolkit/datumaro/blob/6a2022ed467c7557fb2055c143452bc4ca7f6e86/datumaro/plugins/cvat_format/extractor.py#L96). This is causing an issue for us since this removes the leading zeros automatically from the value, changing user input.\r\n\r\nIs there any reason behind typecasting the the attribute value to float? Can this typecasting step be deleted. If required a user can always do the typecasting later. \n", "before_files": [{"content": "# Copyright (C) 2019-2020 Intel Corporation\n#\n# SPDX-License-Identifier: MIT\n\nfrom collections import OrderedDict\nimport os.path as osp\n\nfrom defusedxml import ElementTree\n\nfrom datumaro.components.extractor import (\n    AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, Points,\n    Polygon, PolyLine, SourceExtractor,\n)\nfrom datumaro.util.image import Image\n\nfrom .format import CvatPath\n\n\nclass CvatExtractor(SourceExtractor):\n    _SUPPORTED_SHAPES = ('box', 'polygon', 'polyline', 'points')\n\n    def __init__(self, path, subset=None):\n        assert osp.isfile(path), path\n        rootpath = osp.dirname(path)\n        images_dir = ''\n        if osp.isdir(osp.join(rootpath, CvatPath.IMAGES_DIR)):\n            images_dir = osp.join(rootpath, CvatPath.IMAGES_DIR)\n        self._images_dir = images_dir\n        self._path = path\n\n        if not subset:\n            subset = osp.splitext(osp.basename(path))[0]\n        super().__init__(subset=subset)\n\n        items, categories = self._parse(path)\n        self._items = list(self._load_items(items).values())\n        self._categories = categories\n\n    @classmethod\n    def _parse(cls, path):\n        context = ElementTree.iterparse(path, events=(\"start\", \"end\"))\n        context = iter(context)\n\n        categories, frame_size = cls._parse_meta(context)\n\n        items = OrderedDict()\n\n        track = None\n        shape = None\n        tag = None\n        attributes = None\n        image = None\n        for ev, el in context:\n            if ev == 'start':\n                if el.tag == 'track':\n                    track = {\n                        'id': el.attrib['id'],\n                        'label': el.attrib.get('label'),\n                        'group': int(el.attrib.get('group_id', 0)),\n                        'height': frame_size[0],\n                        'width': frame_size[1],\n                    }\n                elif el.tag == 'image':\n                    image = {\n                        'name': el.attrib.get('name'),\n                        'frame': el.attrib['id'],\n                        'width': el.attrib.get('width'),\n                        'height': el.attrib.get('height'),\n                    }\n                elif el.tag in cls._SUPPORTED_SHAPES and (track or image):\n                    attributes = {}\n                    shape = {\n                        'type': None,\n                        'attributes': attributes,\n                    }\n                    if track:\n                        shape.update(track)\n                        shape['track_id'] = int(track['id'])\n                    if image:\n                        shape.update(image)\n                elif el.tag == 'tag' and image:\n                    attributes = {}\n                    tag = {\n                        'frame': image['frame'],\n                        'attributes': attributes,\n                        'group': int(el.attrib.get('group_id', 0)),\n                        'label': el.attrib['label'],\n                    }\n            elif ev == 'end':\n                if el.tag == 'attribute' and attributes is not None:\n                    attr_value = el.text or ''\n                    if el.text in ['true', 'false']:\n                        attr_value = attr_value == 'true'\n                    else:\n                        try:\n                            attr_value = float(attr_value)\n                        except ValueError:\n                            pass\n                    attributes[el.attrib['name']] = attr_value\n                elif el.tag in cls._SUPPORTED_SHAPES:\n                    if track is not None:\n                        shape['frame'] = el.attrib['frame']\n                        shape['outside'] = (el.attrib.get('outside') == '1')\n                        shape['keyframe'] = (el.attrib.get('keyframe') == '1')\n                    if image is not None:\n                        shape['label'] = el.attrib.get('label')\n                        shape['group'] = int(el.attrib.get('group_id', 0))\n\n                    shape['type'] = el.tag\n                    shape['occluded'] = (el.attrib.get('occluded') == '1')\n                    shape['z_order'] = int(el.attrib.get('z_order', 0))\n\n                    if el.tag == 'box':\n                        shape['points'] = list(map(float, [\n                            el.attrib['xtl'], el.attrib['ytl'],\n                            el.attrib['xbr'], el.attrib['ybr'],\n                        ]))\n                    else:\n                        shape['points'] = []\n                        for pair in el.attrib['points'].split(';'):\n                            shape['points'].extend(map(float, pair.split(',')))\n\n                    frame_desc = items.get(shape['frame'], {'annotations': []})\n                    frame_desc['annotations'].append(\n                        cls._parse_shape_ann(shape, categories))\n                    items[shape['frame']] = frame_desc\n                    shape = None\n\n                elif el.tag == 'tag':\n                    frame_desc = items.get(tag['frame'], {'annotations': []})\n                    frame_desc['annotations'].append(\n                        cls._parse_tag_ann(tag, categories))\n                    items[tag['frame']] = frame_desc\n                    tag = None\n                elif el.tag == 'track':\n                    track = None\n                elif el.tag == 'image':\n                    frame_desc = items.get(image['frame'], {'annotations': []})\n                    frame_desc.update({\n                        'name': image.get('name'),\n                        'height': image.get('height'),\n                        'width': image.get('width'),\n                    })\n                    items[image['frame']] = frame_desc\n                    image = None\n                el.clear()\n\n        return items, categories\n\n    @staticmethod\n    def _parse_meta(context):\n        ev, el = next(context)\n        if not (ev == 'start' and el.tag == 'annotations'):\n            raise Exception(\"Unexpected token \")\n\n        categories = {}\n\n        frame_size = None\n        mode = None\n        labels = OrderedDict()\n        label = None\n\n        # Recursive descent parser\n        el = None\n        states = ['annotations']\n        def accepted(expected_state, tag, next_state=None):\n            state = states[-1]\n            if state == expected_state and el is not None and el.tag == tag:\n                if not next_state:\n                    next_state = tag\n                states.append(next_state)\n                return True\n            return False\n        def consumed(expected_state, tag):\n            state = states[-1]\n            if state == expected_state and el is not None and el.tag == tag:\n                states.pop()\n                return True\n            return False\n\n        for ev, el in context:\n            if ev == 'start':\n                if accepted('annotations', 'meta'): pass\n                elif accepted('meta', 'task'): pass\n                elif accepted('task', 'mode'): pass\n                elif accepted('task', 'original_size'):\n                    frame_size = [None, None]\n                elif accepted('original_size', 'height', next_state='frame_height'): pass\n                elif accepted('original_size', 'width', next_state='frame_width'): pass\n                elif accepted('task', 'labels'): pass\n                elif accepted('labels', 'label'):\n                    label = { 'name': None, 'attributes': set() }\n                elif accepted('label', 'name', next_state='label_name'): pass\n                elif accepted('label', 'attributes'): pass\n                elif accepted('attributes', 'attribute'): pass\n                elif accepted('attribute', 'name', next_state='attr_name'): pass\n                elif accepted('annotations', 'image') or \\\n                     accepted('annotations', 'track') or \\\n                     accepted('annotations', 'tag'):\n                    break\n                else:\n                    pass\n            elif ev == 'end':\n                if consumed('meta', 'meta'):\n                    break\n                elif consumed('task', 'task'): pass\n                elif consumed('mode', 'mode'):\n                    mode = el.text\n                elif consumed('original_size', 'original_size'): pass\n                elif consumed('frame_height', 'height'):\n                    frame_size[0] = int(el.text)\n                elif consumed('frame_width', 'width'):\n                    frame_size[1] = int(el.text)\n                elif consumed('label_name', 'name'):\n                    label['name'] = el.text\n                elif consumed('attr_name', 'name'):\n                    label['attributes'].add(el.text)\n                elif consumed('attribute', 'attribute'): pass\n                elif consumed('attributes', 'attributes'): pass\n                elif consumed('label', 'label'):\n                    labels[label['name']] = label['attributes']\n                    label = None\n                elif consumed('labels', 'labels'): pass\n                else:\n                    pass\n\n        assert len(states) == 1 and states[0] == 'annotations', \\\n            \"Expected 'meta' section in the annotation file, path: %s\" % states\n\n        common_attrs = ['occluded']\n        if mode == 'interpolation':\n            common_attrs.append('keyframe')\n            common_attrs.append('outside')\n            common_attrs.append('track_id')\n\n        label_cat = LabelCategories(attributes=common_attrs)\n        for label, attrs in labels.items():\n            label_cat.add(label, attributes=attrs)\n\n        categories[AnnotationType.label] = label_cat\n\n        return categories, frame_size\n\n    @classmethod\n    def _parse_shape_ann(cls, ann, categories):\n        ann_id = ann.get('id', 0)\n        ann_type = ann['type']\n\n        attributes = ann.get('attributes') or {}\n        if 'occluded' in categories[AnnotationType.label].attributes:\n            attributes['occluded'] = ann.get('occluded', False)\n        if 'outside' in ann:\n            attributes['outside'] = ann['outside']\n        if 'keyframe' in ann:\n            attributes['keyframe'] = ann['keyframe']\n        if 'track_id' in ann:\n            attributes['track_id'] = ann['track_id']\n\n        group = ann.get('group')\n\n        label = ann.get('label')\n        label_id = categories[AnnotationType.label].find(label)[0]\n\n        z_order = ann.get('z_order', 0)\n        points = ann.get('points', [])\n\n        if ann_type == 'polyline':\n            return PolyLine(points, label=label_id, z_order=z_order,\n                id=ann_id, attributes=attributes, group=group)\n\n        elif ann_type == 'polygon':\n            return Polygon(points, label=label_id, z_order=z_order,\n                id=ann_id, attributes=attributes, group=group)\n\n        elif ann_type == 'points':\n            return Points(points, label=label_id, z_order=z_order,\n                id=ann_id, attributes=attributes, group=group)\n\n        elif ann_type == 'box':\n            x, y = points[0], points[1]\n            w, h = points[2] - x, points[3] - y\n            return Bbox(x, y, w, h, label=label_id, z_order=z_order,\n                id=ann_id, attributes=attributes, group=group)\n\n        else:\n            raise NotImplementedError(\"Unknown annotation type '%s'\" % ann_type)\n\n    @classmethod\n    def _parse_tag_ann(cls, ann, categories):\n        label = ann.get('label')\n        label_id = categories[AnnotationType.label].find(label)[0]\n        group = ann.get('group')\n        attributes = ann.get('attributes')\n        return Label(label_id, attributes=attributes, group=group)\n\n    def _load_items(self, parsed):\n        for frame_id, item_desc in parsed.items():\n            name = item_desc.get('name', 'frame_%06d.png' % int(frame_id))\n            image = osp.join(self._images_dir, name)\n            image_size = (item_desc.get('height'), item_desc.get('width'))\n            if all(image_size):\n                image = Image(path=image, size=tuple(map(int, image_size)))\n\n            parsed[frame_id] = DatasetItem(id=osp.splitext(name)[0],\n                subset=self._subset, image=image,\n                annotations=item_desc.get('annotations'),\n                attributes={'frame': int(frame_id)})\n        return parsed\n\nclass CvatImporter(Importer):\n    @classmethod\n    def find_sources(cls, path):\n        return cls._find_sources_recursive(path, '.xml', 'cvat')\n", "path": "datumaro/plugins/cvat_format/extractor.py"}], "after_files": [{"content": "# Copyright (C) 2019-2020 Intel Corporation\n#\n# SPDX-License-Identifier: MIT\n\nfrom collections import OrderedDict\nimport os.path as osp\n\nfrom defusedxml import ElementTree\n\nfrom datumaro.components.extractor import (\n    AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, Points,\n    Polygon, PolyLine, SourceExtractor,\n)\nfrom datumaro.util.image import Image\n\nfrom .format import CvatPath\n\n\nclass CvatExtractor(SourceExtractor):\n    _SUPPORTED_SHAPES = ('box', 'polygon', 'polyline', 'points')\n\n    def __init__(self, path, subset=None):\n        assert osp.isfile(path), path\n        rootpath = osp.dirname(path)\n        images_dir = ''\n        if osp.isdir(osp.join(rootpath, CvatPath.IMAGES_DIR)):\n            images_dir = osp.join(rootpath, CvatPath.IMAGES_DIR)\n        self._images_dir = images_dir\n        self._path = path\n\n        if not subset:\n            subset = osp.splitext(osp.basename(path))[0]\n        super().__init__(subset=subset)\n\n        items, categories = self._parse(path)\n        self._items = list(self._load_items(items).values())\n        self._categories = categories\n\n    @classmethod\n    def _parse(cls, path):\n        context = ElementTree.iterparse(path, events=(\"start\", \"end\"))\n        context = iter(context)\n\n        categories, frame_size, attribute_types = cls._parse_meta(context)\n\n        items = OrderedDict()\n\n        track = None\n        shape = None\n        tag = None\n        attributes = None\n        image = None\n        for ev, el in context:\n            if ev == 'start':\n                if el.tag == 'track':\n                    track = {\n                        'id': el.attrib['id'],\n                        'label': el.attrib.get('label'),\n                        'group': int(el.attrib.get('group_id', 0)),\n                        'height': frame_size[0],\n                        'width': frame_size[1],\n                    }\n                elif el.tag == 'image':\n                    image = {\n                        'name': el.attrib.get('name'),\n                        'frame': el.attrib['id'],\n                        'width': el.attrib.get('width'),\n                        'height': el.attrib.get('height'),\n                    }\n                elif el.tag in cls._SUPPORTED_SHAPES and (track or image):\n                    attributes = {}\n                    shape = {\n                        'type': None,\n                        'attributes': attributes,\n                    }\n                    if track:\n                        shape.update(track)\n                        shape['track_id'] = int(track['id'])\n                    if image:\n                        shape.update(image)\n                elif el.tag == 'tag' and image:\n                    attributes = {}\n                    tag = {\n                        'frame': image['frame'],\n                        'attributes': attributes,\n                        'group': int(el.attrib.get('group_id', 0)),\n                        'label': el.attrib['label'],\n                    }\n            elif ev == 'end':\n                if el.tag == 'attribute' and attributes is not None:\n                    attr_value = el.text or ''\n                    attr_type = attribute_types.get(el.attrib['name'])\n                    if el.text in ['true', 'false']:\n                        attr_value = attr_value == 'true'\n                    elif  attr_type is not None and attr_type != 'text':\n                        try:\n                            attr_value = float(attr_value)\n                        except ValueError:\n                            pass\n                    attributes[el.attrib['name']] = attr_value\n                elif el.tag in cls._SUPPORTED_SHAPES:\n                    if track is not None:\n                        shape['frame'] = el.attrib['frame']\n                        shape['outside'] = (el.attrib.get('outside') == '1')\n                        shape['keyframe'] = (el.attrib.get('keyframe') == '1')\n                    if image is not None:\n                        shape['label'] = el.attrib.get('label')\n                        shape['group'] = int(el.attrib.get('group_id', 0))\n\n                    shape['type'] = el.tag\n                    shape['occluded'] = (el.attrib.get('occluded') == '1')\n                    shape['z_order'] = int(el.attrib.get('z_order', 0))\n\n                    if el.tag == 'box':\n                        shape['points'] = list(map(float, [\n                            el.attrib['xtl'], el.attrib['ytl'],\n                            el.attrib['xbr'], el.attrib['ybr'],\n                        ]))\n                    else:\n                        shape['points'] = []\n                        for pair in el.attrib['points'].split(';'):\n                            shape['points'].extend(map(float, pair.split(',')))\n\n                    frame_desc = items.get(shape['frame'], {'annotations': []})\n                    frame_desc['annotations'].append(\n                        cls._parse_shape_ann(shape, categories))\n                    items[shape['frame']] = frame_desc\n                    shape = None\n\n                elif el.tag == 'tag':\n                    frame_desc = items.get(tag['frame'], {'annotations': []})\n                    frame_desc['annotations'].append(\n                        cls._parse_tag_ann(tag, categories))\n                    items[tag['frame']] = frame_desc\n                    tag = None\n                elif el.tag == 'track':\n                    track = None\n                elif el.tag == 'image':\n                    frame_desc = items.get(image['frame'], {'annotations': []})\n                    frame_desc.update({\n                        'name': image.get('name'),\n                        'height': image.get('height'),\n                        'width': image.get('width'),\n                    })\n                    items[image['frame']] = frame_desc\n                    image = None\n                el.clear()\n\n        return items, categories\n\n    @staticmethod\n    def _parse_meta(context):\n        ev, el = next(context)\n        if not (ev == 'start' and el.tag == 'annotations'):\n            raise Exception(\"Unexpected token \")\n\n        categories = {}\n\n        frame_size = None\n        mode = None\n        labels = OrderedDict()\n        label = None\n\n        # Recursive descent parser\n        el = None\n        states = ['annotations']\n        def accepted(expected_state, tag, next_state=None):\n            state = states[-1]\n            if state == expected_state and el is not None and el.tag == tag:\n                if not next_state:\n                    next_state = tag\n                states.append(next_state)\n                return True\n            return False\n        def consumed(expected_state, tag):\n            state = states[-1]\n            if state == expected_state and el is not None and el.tag == tag:\n                states.pop()\n                return True\n            return False\n\n        for ev, el in context:\n            if ev == 'start':\n                if accepted('annotations', 'meta'): pass\n                elif accepted('meta', 'task'): pass\n                elif accepted('task', 'mode'): pass\n                elif accepted('task', 'original_size'):\n                    frame_size = [None, None]\n                elif accepted('original_size', 'height', next_state='frame_height'): pass\n                elif accepted('original_size', 'width', next_state='frame_width'): pass\n                elif accepted('task', 'labels'): pass\n                elif accepted('labels', 'label'):\n                    label = { 'name': None, 'attributes': [] }\n                elif accepted('label', 'name', next_state='label_name'): pass\n                elif accepted('label', 'attributes'): pass\n                elif accepted('attributes', 'attribute'): pass\n                elif accepted('attribute', 'name', next_state='attr_name'): pass\n                elif accepted('attribute', 'input_type', next_state='attr_type'): pass\n                elif accepted('annotations', 'image') or \\\n                     accepted('annotations', 'track') or \\\n                     accepted('annotations', 'tag'):\n                    break\n                else:\n                    pass\n            elif ev == 'end':\n                if consumed('meta', 'meta'):\n                    break\n                elif consumed('task', 'task'): pass\n                elif consumed('mode', 'mode'):\n                    mode = el.text\n                elif consumed('original_size', 'original_size'): pass\n                elif consumed('frame_height', 'height'):\n                    frame_size[0] = int(el.text)\n                elif consumed('frame_width', 'width'):\n                    frame_size[1] = int(el.text)\n                elif consumed('label_name', 'name'):\n                    label['name'] = el.text\n                elif consumed('attr_name', 'name'):\n                    label['attributes'].append({'name': el.text})\n                elif consumed('attr_type', 'input_type'):\n                    label['attributes'][-1]['input_type'] = el.text\n                elif consumed('attribute', 'attribute'): pass\n                elif consumed('attributes', 'attributes'): pass\n                elif consumed('label', 'label'):\n                    labels[label['name']] = label['attributes']\n                    label = None\n                elif consumed('labels', 'labels'): pass\n                else:\n                    pass\n\n        assert len(states) == 1 and states[0] == 'annotations', \\\n            \"Expected 'meta' section in the annotation file, path: %s\" % states\n\n        common_attrs = ['occluded']\n        if mode == 'interpolation':\n            common_attrs.append('keyframe')\n            common_attrs.append('outside')\n            common_attrs.append('track_id')\n\n        label_cat = LabelCategories(attributes=common_attrs)\n        attribute_types = {}\n        for label, attrs in labels.items():\n            attr_names = {v['name'] for v in attrs}\n            label_cat.add(label, attributes=attr_names)\n            for attr in attrs:\n                attribute_types[attr['name']] = attr['input_type']\n\n        categories[AnnotationType.label] = label_cat\n        return categories, frame_size, attribute_types\n\n    @classmethod\n    def _parse_shape_ann(cls, ann, categories):\n        ann_id = ann.get('id', 0)\n        ann_type = ann['type']\n\n        attributes = ann.get('attributes') or {}\n        if 'occluded' in categories[AnnotationType.label].attributes:\n            attributes['occluded'] = ann.get('occluded', False)\n        if 'outside' in ann:\n            attributes['outside'] = ann['outside']\n        if 'keyframe' in ann:\n            attributes['keyframe'] = ann['keyframe']\n        if 'track_id' in ann:\n            attributes['track_id'] = ann['track_id']\n\n        group = ann.get('group')\n\n        label = ann.get('label')\n        label_id = categories[AnnotationType.label].find(label)[0]\n\n        z_order = ann.get('z_order', 0)\n        points = ann.get('points', [])\n\n        if ann_type == 'polyline':\n            return PolyLine(points, label=label_id, z_order=z_order,\n                id=ann_id, attributes=attributes, group=group)\n\n        elif ann_type == 'polygon':\n            return Polygon(points, label=label_id, z_order=z_order,\n                id=ann_id, attributes=attributes, group=group)\n\n        elif ann_type == 'points':\n            return Points(points, label=label_id, z_order=z_order,\n                id=ann_id, attributes=attributes, group=group)\n\n        elif ann_type == 'box':\n            x, y = points[0], points[1]\n            w, h = points[2] - x, points[3] - y\n            return Bbox(x, y, w, h, label=label_id, z_order=z_order,\n                id=ann_id, attributes=attributes, group=group)\n\n        else:\n            raise NotImplementedError(\"Unknown annotation type '%s'\" % ann_type)\n\n    @classmethod\n    def _parse_tag_ann(cls, ann, categories):\n        label = ann.get('label')\n        label_id = categories[AnnotationType.label].find(label)[0]\n        group = ann.get('group')\n        attributes = ann.get('attributes')\n        return Label(label_id, attributes=attributes, group=group)\n\n    def _load_items(self, parsed):\n        for frame_id, item_desc in parsed.items():\n            name = item_desc.get('name', 'frame_%06d.png' % int(frame_id))\n            image = osp.join(self._images_dir, name)\n            image_size = (item_desc.get('height'), item_desc.get('width'))\n            if all(image_size):\n                image = Image(path=image, size=tuple(map(int, image_size)))\n\n            parsed[frame_id] = DatasetItem(id=osp.splitext(name)[0],\n                subset=self._subset, image=image,\n                annotations=item_desc.get('annotations'),\n                attributes={'frame': int(frame_id)})\n        return parsed\n\nclass CvatImporter(Importer):\n    @classmethod\n    def find_sources(cls, path):\n        return cls._find_sources_recursive(path, '.xml', 'cvat')\n", "path": "datumaro/plugins/cvat_format/extractor.py"}]}