prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Dataset patches do not remove empty (e.g. renamed) subsets
  Steps to reproduce:
  1. Create a project
  2. Import a dataset
  3. Rename a subset (e.g. `datum transform -t random_split`)
  
  Depending on the format, the exported dataset will contain annotations from renamed-from and renamed-to subsets. This leads to duplication of annotations in different subsets, which are then found and merged together on importing.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `datumaro/components/converter.py`
  Content:
  ```
  1 # Copyright (C) 2019-2021 Intel Corporation
  2 #
  3 # SPDX-License-Identifier: MIT
  4 
  5 from typing import Union
  6 import logging as log
  7 import os
  8 import os.path as osp
  9 import shutil
  10 
  11 from datumaro.components.cli_plugin import CliPlugin
  12 from datumaro.components.dataset import DatasetPatch
  13 from datumaro.components.extractor import DatasetItem
  14 from datumaro.util.image import Image
  15 
  16 
  17 class Converter(CliPlugin):
  18     DEFAULT_IMAGE_EXT = None
  19 
  20     @classmethod
  21     def build_cmdline_parser(cls, **kwargs):
  22         parser = super().build_cmdline_parser(**kwargs)
  23         parser.add_argument('--save-images', action='store_true',
  24             help="Save images (default: %(default)s)")
  25         parser.add_argument('--image-ext', default=None,
  26             help="Image extension (default: keep or use format default%s)" % \
  27                 (' ' + cls.DEFAULT_IMAGE_EXT if cls.DEFAULT_IMAGE_EXT else ''))
  28 
  29         return parser
  30 
  31     @classmethod
  32     def convert(cls, extractor, save_dir, **options):
  33         converter = cls(extractor, save_dir, **options)
  34         return converter.apply()
  35 
  36     @classmethod
  37     def patch(cls, dataset, patch, save_dir, **options):
  38         return cls.convert(dataset, save_dir, **options)
  39 
  40     def apply(self):
  41         raise NotImplementedError("Should be implemented in a subclass")
  42 
  43     def __init__(self, extractor, save_dir, save_images=False,
  44             image_ext=None, default_image_ext=None):
  45         default_image_ext = default_image_ext or self.DEFAULT_IMAGE_EXT
  46         assert default_image_ext
  47         self._default_image_ext = default_image_ext
  48 
  49         self._save_images = save_images
  50         self._image_ext = image_ext
  51 
  52         self._extractor = extractor
  53         self._save_dir = save_dir
  54 
  55         # TODO: refactor this variable.
  56         # Can be used by a subclass to store the current patch info
  57         if isinstance(extractor, DatasetPatch.DatasetPatchWrapper):
  58             self._patch = extractor.patch
  59         else:
  60             self._patch = None
  61 
  62     def _find_image_ext(self, item: Union[DatasetItem, Image]):
  63         src_ext = None
  64 
  65         if isinstance(item, DatasetItem) and item.has_image:
  66             src_ext = item.image.ext
  67         elif isinstance(item, Image):
  68             src_ext = item.ext
  69 
  70         return self._image_ext or src_ext or self._default_image_ext
  71 
  72     def _make_item_filename(self, item, *, name=None, subdir=None):
  73         name = name or item.id
  74         subdir = subdir or ''
  75         return osp.join(subdir, name)
  76 
  77     def _make_image_filename(self, item, *, name=None, subdir=None):
  78         return self._make_item_filename(item, name=name, subdir=subdir) + \
  79             self._find_image_ext(item)
  80 
  81     def _make_pcd_filename(self, item, *, name=None, subdir=None):
  82         return self._make_item_filename(item, name=name, subdir=subdir) + '.pcd'
  83 
  84     def _save_image(self, item, path=None, *,
  85             name=None, subdir=None, basedir=None):
  86         assert not ((subdir or name or basedir) and path), \
  87             "Can't use both subdir or name or basedir and path arguments"
  88 
  89         if not item.has_image or not item.image.has_data:
  90             log.warning("Item '%s' has no image", item.id)
  91             return
  92 
  93         basedir = basedir or self._save_dir
  94         path = path or osp.join(basedir,
  95             self._make_image_filename(item, name=name, subdir=subdir))
  96         path = osp.abspath(path)
  97 
  98         item.image.save(path)
  99 
  100     def _save_point_cloud(self, item=None, path=None, *,
  101             name=None, subdir=None, basedir=None):
  102         assert not ((subdir or name or basedir) and path), \
  103             "Can't use both subdir or name or basedir and path arguments"
  104 
  105         if not item.point_cloud:
  106             log.warning("Item '%s' has no pcd", item.id)
  107             return
  108 
  109         basedir = basedir or self._save_dir
  110         path = path or osp.join(basedir,
  111             self._make_pcd_filename(item, name=name, subdir=subdir))
  112         path = osp.abspath(path)
  113 
  114         os.makedirs(osp.dirname(path), exist_ok=True)
  115         if item.point_cloud and osp.isfile(item.point_cloud):
  116             if item.point_cloud != path:
  117                 shutil.copyfile(item.point_cloud, path)
  118
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/datumaro/components/converter.py b/datumaro/components/converter.py
  --- a/datumaro/components/converter.py
  +++ b/datumaro/components/converter.py
  @@ -2,6 +2,7 @@
   #
   # SPDX-License-Identifier: MIT
   
  +from tempfile import mkdtemp
   from typing import Union
   import logging as log
   import os
  @@ -11,6 +12,7 @@
   from datumaro.components.cli_plugin import CliPlugin
   from datumaro.components.dataset import DatasetPatch
   from datumaro.components.extractor import DatasetItem
  +from datumaro.util import error_rollback, on_error_do
   from datumaro.util.image import Image
   
   
  @@ -34,8 +36,34 @@
           return converter.apply()
   
       @classmethod
  +    @error_rollback
       def patch(cls, dataset, patch, save_dir, **options):
  -        return cls.convert(dataset, save_dir, **options)
  +        # This solution is not any better in performance than just
  +        # writing a dataset, but in case of patching (i.e. writing
  +        # to the previous location), it allows to avoid many problems
  +        # with removing and replacing existing files. Surely, this
  +        # approach also has problems with removal of the given directory.
  +        # Problems can occur if we can't remove the directory,
  +        # or want to reuse the given directory. It can happen if it
  +        # is mounted or (sym-)linked.
  +        # Probably, a better solution could be to wipe directory
  +        # contents and write new data there. Note that directly doing this
  +        # also doesn't work, because images may be needed for writing.
  +
  +        if not osp.isdir(save_dir):
  +            return cls.convert(dataset, save_dir, **options)
  +
  +        tmpdir = mkdtemp(dir=osp.dirname(save_dir),
  +            prefix=osp.basename(save_dir), suffix='.tmp')
  +        on_error_do(shutil.rmtree, tmpdir, ignore_errors=True)
  +        shutil.copymode(save_dir, tmpdir)
  +
  +        retval = cls.convert(dataset, tmpdir, **options)
  +
  +        shutil.rmtree(save_dir)
  +        os.replace(tmpdir, save_dir)
  +
  +        return retval
   
       def apply(self):
           raise NotImplementedError("Should be implemented in a subclass")
  
verification_info: {"golden_diff": "diff --git a/datumaro/components/converter.py b/datumaro/components/converter.py\n--- a/datumaro/components/converter.py\n+++ b/datumaro/components/converter.py\n@@ -2,6 +2,7 @@\n #\n # SPDX-License-Identifier: MIT\n \n+from tempfile import mkdtemp\n from typing import Union\n import logging as log\n import os\n@@ -11,6 +12,7 @@\n from datumaro.components.cli_plugin import CliPlugin\n from datumaro.components.dataset import DatasetPatch\n from datumaro.components.extractor import DatasetItem\n+from datumaro.util import error_rollback, on_error_do\n from datumaro.util.image import Image\n \n \n@@ -34,8 +36,34 @@\n         return converter.apply()\n \n     @classmethod\n+    @error_rollback\n     def patch(cls, dataset, patch, save_dir, **options):\n-        return cls.convert(dataset, save_dir, **options)\n+        # This solution is not any better in performance than just\n+        # writing a dataset, but in case of patching (i.e. writing\n+        # to the previous location), it allows to avoid many problems\n+        # with removing and replacing existing files. Surely, this\n+        # approach also has problems with removal of the given directory.\n+        # Problems can occur if we can't remove the directory,\n+        # or want to reuse the given directory. It can happen if it\n+        # is mounted or (sym-)linked.\n+        # Probably, a better solution could be to wipe directory\n+        # contents and write new data there. Note that directly doing this\n+        # also doesn't work, because images may be needed for writing.\n+\n+        if not osp.isdir(save_dir):\n+            return cls.convert(dataset, save_dir, **options)\n+\n+        tmpdir = mkdtemp(dir=osp.dirname(save_dir),\n+            prefix=osp.basename(save_dir), suffix='.tmp')\n+        on_error_do(shutil.rmtree, tmpdir, ignore_errors=True)\n+        shutil.copymode(save_dir, tmpdir)\n+\n+        retval = cls.convert(dataset, tmpdir, **options)\n+\n+        shutil.rmtree(save_dir)\n+        os.replace(tmpdir, save_dir)\n+\n+        return retval\n \n     def apply(self):\n         raise NotImplementedError(\"Should be implemented in a subclass\")\n", "issue": "Dataset patches do not remove empty (e.g. renamed) subsets\nSteps to reproduce:\r\n1. Create a project\r\n2. Import a dataset\r\n3. Rename a subset (e.g. `datum transform -t random_split`)\r\n\r\nDepending on the format, the exported dataset will contain annotations from renamed-from and renamed-to subsets. This leads to duplication of annotations in different subsets, which are then found and merged together on importing.\n", "before_files": [{"content": "# Copyright (C) 2019-2021 Intel Corporation\n#\n# SPDX-License-Identifier: MIT\n\nfrom typing import Union\nimport logging as log\nimport os\nimport os.path as osp\nimport shutil\n\nfrom datumaro.components.cli_plugin import CliPlugin\nfrom datumaro.components.dataset import DatasetPatch\nfrom datumaro.components.extractor import DatasetItem\nfrom datumaro.util.image import Image\n\n\nclass Converter(CliPlugin):\n    DEFAULT_IMAGE_EXT = None\n\n    @classmethod\n    def build_cmdline_parser(cls, **kwargs):\n        parser = super().build_cmdline_parser(**kwargs)\n        parser.add_argument('--save-images', action='store_true',\n            help=\"Save images (default: %(default)s)\")\n        parser.add_argument('--image-ext', default=None,\n            help=\"Image extension (default: keep or use format default%s)\" % \\\n                (' ' + cls.DEFAULT_IMAGE_EXT if cls.DEFAULT_IMAGE_EXT else ''))\n\n        return parser\n\n    @classmethod\n    def convert(cls, extractor, save_dir, **options):\n        converter = cls(extractor, save_dir, **options)\n        return converter.apply()\n\n    @classmethod\n    def patch(cls, dataset, patch, save_dir, **options):\n        return cls.convert(dataset, save_dir, **options)\n\n    def apply(self):\n        raise NotImplementedError(\"Should be implemented in a subclass\")\n\n    def __init__(self, extractor, save_dir, save_images=False,\n            image_ext=None, default_image_ext=None):\n        default_image_ext = default_image_ext or self.DEFAULT_IMAGE_EXT\n        assert default_image_ext\n        self._default_image_ext = default_image_ext\n\n        self._save_images = save_images\n        self._image_ext = image_ext\n\n        self._extractor = extractor\n        self._save_dir = save_dir\n\n        # TODO: refactor this variable.\n        # Can be used by a subclass to store the current patch info\n        if isinstance(extractor, DatasetPatch.DatasetPatchWrapper):\n            self._patch = extractor.patch\n        else:\n            self._patch = None\n\n    def _find_image_ext(self, item: Union[DatasetItem, Image]):\n        src_ext = None\n\n        if isinstance(item, DatasetItem) and item.has_image:\n            src_ext = item.image.ext\n        elif isinstance(item, Image):\n            src_ext = item.ext\n\n        return self._image_ext or src_ext or self._default_image_ext\n\n    def _make_item_filename(self, item, *, name=None, subdir=None):\n        name = name or item.id\n        subdir = subdir or ''\n        return osp.join(subdir, name)\n\n    def _make_image_filename(self, item, *, name=None, subdir=None):\n        return self._make_item_filename(item, name=name, subdir=subdir) + \\\n            self._find_image_ext(item)\n\n    def _make_pcd_filename(self, item, *, name=None, subdir=None):\n        return self._make_item_filename(item, name=name, subdir=subdir) + '.pcd'\n\n    def _save_image(self, item, path=None, *,\n            name=None, subdir=None, basedir=None):\n        assert not ((subdir or name or basedir) and path), \\\n            \"Can't use both subdir or name or basedir and path arguments\"\n\n        if not item.has_image or not item.image.has_data:\n            log.warning(\"Item '%s' has no image\", item.id)\n            return\n\n        basedir = basedir or self._save_dir\n        path = path or osp.join(basedir,\n            self._make_image_filename(item, name=name, subdir=subdir))\n        path = osp.abspath(path)\n\n        item.image.save(path)\n\n    def _save_point_cloud(self, item=None, path=None, *,\n            name=None, subdir=None, basedir=None):\n        assert not ((subdir or name or basedir) and path), \\\n            \"Can't use both subdir or name or basedir and path arguments\"\n\n        if not item.point_cloud:\n            log.warning(\"Item '%s' has no pcd\", item.id)\n            return\n\n        basedir = basedir or self._save_dir\n        path = path or osp.join(basedir,\n            self._make_pcd_filename(item, name=name, subdir=subdir))\n        path = osp.abspath(path)\n\n        os.makedirs(osp.dirname(path), exist_ok=True)\n        if item.point_cloud and osp.isfile(item.point_cloud):\n            if item.point_cloud != path:\n                shutil.copyfile(item.point_cloud, path)\n", "path": "datumaro/components/converter.py"}], "after_files": [{"content": "# Copyright (C) 2019-2021 Intel Corporation\n#\n# SPDX-License-Identifier: MIT\n\nfrom tempfile import mkdtemp\nfrom typing import Union\nimport logging as log\nimport os\nimport os.path as osp\nimport shutil\n\nfrom datumaro.components.cli_plugin import CliPlugin\nfrom datumaro.components.dataset import DatasetPatch\nfrom datumaro.components.extractor import DatasetItem\nfrom datumaro.util import error_rollback, on_error_do\nfrom datumaro.util.image import Image\n\n\nclass Converter(CliPlugin):\n    DEFAULT_IMAGE_EXT = None\n\n    @classmethod\n    def build_cmdline_parser(cls, **kwargs):\n        parser = super().build_cmdline_parser(**kwargs)\n        parser.add_argument('--save-images', action='store_true',\n            help=\"Save images (default: %(default)s)\")\n        parser.add_argument('--image-ext', default=None,\n            help=\"Image extension (default: keep or use format default%s)\" % \\\n                (' ' + cls.DEFAULT_IMAGE_EXT if cls.DEFAULT_IMAGE_EXT else ''))\n\n        return parser\n\n    @classmethod\n    def convert(cls, extractor, save_dir, **options):\n        converter = cls(extractor, save_dir, **options)\n        return converter.apply()\n\n    @classmethod\n    @error_rollback\n    def patch(cls, dataset, patch, save_dir, **options):\n        # This solution is not any better in performance than just\n        # writing a dataset, but in case of patching (i.e. writing\n        # to the previous location), it allows to avoid many problems\n        # with removing and replacing existing files. Surely, this\n        # approach also has problems with removal of the given directory.\n        # Problems can occur if we can't remove the directory,\n        # or want to reuse the given directory. It can happen if it\n        # is mounted or (sym-)linked.\n        # Probably, a better solution could be to wipe directory\n        # contents and write new data there. Note that directly doing this\n        # also doesn't work, because images may be needed for writing.\n\n        if not osp.isdir(save_dir):\n            return cls.convert(dataset, save_dir, **options)\n\n        tmpdir = mkdtemp(dir=osp.dirname(save_dir),\n            prefix=osp.basename(save_dir), suffix='.tmp')\n        on_error_do(shutil.rmtree, tmpdir, ignore_errors=True)\n        shutil.copymode(save_dir, tmpdir)\n\n        retval = cls.convert(dataset, tmpdir, **options)\n\n        shutil.rmtree(save_dir)\n        os.replace(tmpdir, save_dir)\n\n        return retval\n\n    def apply(self):\n        raise NotImplementedError(\"Should be implemented in a subclass\")\n\n    def __init__(self, extractor, save_dir, save_images=False,\n            image_ext=None, default_image_ext=None):\n        default_image_ext = default_image_ext or self.DEFAULT_IMAGE_EXT\n        assert default_image_ext\n        self._default_image_ext = default_image_ext\n\n        self._save_images = save_images\n        self._image_ext = image_ext\n\n        self._extractor = extractor\n        self._save_dir = save_dir\n\n        # TODO: refactor this variable.\n        # Can be used by a subclass to store the current patch info\n        if isinstance(extractor, DatasetPatch.DatasetPatchWrapper):\n            self._patch = extractor.patch\n        else:\n            self._patch = None\n\n    def _find_image_ext(self, item: Union[DatasetItem, Image]):\n        src_ext = None\n\n        if isinstance(item, DatasetItem) and item.has_image:\n            src_ext = item.image.ext\n        elif isinstance(item, Image):\n            src_ext = item.ext\n\n        return self._image_ext or src_ext or self._default_image_ext\n\n    def _make_item_filename(self, item, *, name=None, subdir=None):\n        name = name or item.id\n        subdir = subdir or ''\n        return osp.join(subdir, name)\n\n    def _make_image_filename(self, item, *, name=None, subdir=None):\n        return self._make_item_filename(item, name=name, subdir=subdir) + \\\n            self._find_image_ext(item)\n\n    def _make_pcd_filename(self, item, *, name=None, subdir=None):\n        return self._make_item_filename(item, name=name, subdir=subdir) + '.pcd'\n\n    def _save_image(self, item, path=None, *,\n            name=None, subdir=None, basedir=None):\n        assert not ((subdir or name or basedir) and path), \\\n            \"Can't use both subdir or name or basedir and path arguments\"\n\n        if not item.has_image or not item.image.has_data:\n            log.warning(\"Item '%s' has no image\", item.id)\n            return\n\n        basedir = basedir or self._save_dir\n        path = path or osp.join(basedir,\n            self._make_image_filename(item, name=name, subdir=subdir))\n        path = osp.abspath(path)\n\n        item.image.save(path)\n\n    def _save_point_cloud(self, item=None, path=None, *,\n            name=None, subdir=None, basedir=None):\n        assert not ((subdir or name or basedir) and path), \\\n            \"Can't use both subdir or name or basedir and path arguments\"\n\n        if not item.point_cloud:\n            log.warning(\"Item '%s' has no pcd\", item.id)\n            return\n\n        basedir = basedir or self._save_dir\n        path = path or osp.join(basedir,\n            self._make_pcd_filename(item, name=name, subdir=subdir))\n        path = osp.abspath(path)\n\n        os.makedirs(osp.dirname(path), exist_ok=True)\n        if item.point_cloud and osp.isfile(item.point_cloud):\n            if item.point_cloud != path:\n                shutil.copyfile(item.point_cloud, path)\n", "path": "datumaro/components/converter.py"}]}