Fixes (#76)

* Prevent inplace file copying when saving images

* Add more dataset tests

* Move config tests to separate file

* Fix labelmap parameter in voc converter

* Update config class:

- Fix eq comparison with other classes
- Add parsing and dumping into a file
- Rename DefaultConfig to DictConfig

* Rename from_cmdline to parse_cmdline

Author: Maxim Zhiltsov

CHANGELOG.md

@@ -28,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Allow plugins inherited from `Extractor` (instead of only `SourceExtractor`) (<https://github.com/openvinotoolkit/datumaro/pull/70>)
 - Windows installation with `pip` for `pycocotools` (<https://github.com/openvinotoolkit/datumaro/pull/73>)
 - `YOLO` extractor path matching on Windows (<https://github.com/openvinotoolkit/datumaro/pull/73>)
+- Fixed inplace file copying when saving images (<https://github.com/openvinotoolkit/datumaro/pull/76>)
+- Fixed `labelmap` parameter type checking in `VOC` converter (<https://github.com/openvinotoolkit/datumaro/pull/76>)
 
 ### Security
 -

datumaro/cli/commands/convert.py

@@ -68,7 +68,7 @@ def convert_command(args):
     except KeyError:
         raise CliException("Converter for format '%s' is not found" % \
             args.output_format)
-    extra_args = converter.from_cmdline(args.extra_args)
+    extra_args = converter.parse_cmdline(args.extra_args)
 
     filter_args = FilterModes.make_filter_args(args.filter_mode)
 

datumaro/cli/contexts/model/__init__.py

@@ -59,12 +59,12 @@ def add_command(args):
         assert args.name not in project.config.models, args.name
 
     try:
-        launcher = project.env.launchers.get(args.launcher)
+        launcher = project.env.launchers[args.launcher]
     except KeyError:
         raise CliException("Launcher '%s' is not found" % args.launcher)
 
     cli_plugin = getattr(launcher, 'cli_plugin', launcher)
-    model_args = cli_plugin.from_cmdline(args.extra_args)
+    model_args = cli_plugin.parse_cmdline(args.extra_args)
 
     if args.copy:
         log.info("Copying model data")

datumaro/cli/contexts/project/__init__.py

@@ -200,8 +200,8 @@ def import_command(args):
             raise CliException("Unknown format '%s'. A format can be added"
                 "by providing an Extractor and Importer plugins" % fmt)
 
-        if hasattr(arg_parser, 'from_cmdline'):
-            extra_args = arg_parser.from_cmdline(args.extra_args)
+        if hasattr(arg_parser, 'parse_cmdline'):
+            extra_args = arg_parser.parse_cmdline(args.extra_args)
         else:
             raise CliException("Format '%s' does not accept "
                 "extra parameters" % fmt)
@@ -330,26 +330,24 @@ def export_command(args):
     dst_dir = osp.abspath(dst_dir)
 
     try:
-        converter = project.env.converters.get(args.format)
+        converter = project.env.converters[args.format]
     except KeyError:
         raise CliException("Converter for format '%s' is not found" % \
             args.format)
 
-    extra_args = converter.from_cmdline(args.extra_args)
-    def converter_proxy(extractor, save_dir):
-        return converter.convert(extractor, save_dir, **extra_args)
+    extra_args = converter.parse_cmdline(args.extra_args)
 
     filter_args = FilterModes.make_filter_args(args.filter_mode)
 
     log.info("Loading the project...")
     dataset = project.make_dataset()
 
     log.info("Exporting the project...")
-    dataset.export_project(
-        save_dir=dst_dir,
-        converter=converter_proxy,
-        filter_expr=args.filter,
-        **filter_args)
+
+    if args.filter:
+        dataset = dataset.filter(args.filter, **filter_args)
+    dataset.export(args.format, save_dir=dst_dir, **extra_args)
+
     log.info("Project exported to '%s' as '%s'" % \
         (dst_dir, args.format))
 
@@ -674,13 +672,13 @@ def transform_command(args):
     dst_dir = osp.abspath(dst_dir)
 
     try:
-        transform = project.env.transforms.get(args.transform)
+        transform = project.env.transforms[args.transform]
     except KeyError:
         raise CliException("Transform '%s' is not found" % args.transform)
 
     extra_args = {}
-    if hasattr(transform, 'from_cmdline'):
-        extra_args = transform.from_cmdline(args.extra_args)
+    if hasattr(transform, 'parse_cmdline'):
+        extra_args = transform.parse_cmdline(args.extra_args)
 
     log.info("Loading the project...")
     dataset = project.make_dataset()

datumaro/components/cli_plugin.py

@@ -31,7 +31,7 @@ def build_cmdline_parser(cls, **kwargs):
         return argparse.ArgumentParser(**args)
 
     @classmethod
-    def from_cmdline(cls, args=None):
+    def parse_cmdline(cls, args=None):
         if args and args[0] == '--':
             args = args[1:]
         parser = cls.build_cmdline_parser()

datumaro/components/config.py

@@ -150,14 +150,19 @@ def __setattr__(self, key, value):
 
     def __eq__(self, other):
         try:
-            for k, my_v in self.items(allow_internal=False):
+            keys = set(self.keys()) | set(other.keys())
+            for k in keys:
+                my_v = self[k]
                 other_v = other[k]
                 if my_v != other_v:
                     return False
             return True
-        except Exception:
+        except (KeyError, AttributeError):
             return False
 
+    def __repr__(self):
+        return repr(dict(self))
+
     def update(self, other):
         for k, v in other.items():
             self.set(k, v)
@@ -205,23 +210,29 @@ def set(self, key, value):
         return value
 
     @staticmethod
-    def parse(path):
-        with open(path, 'r') as f:
-            return Config(yaml.safe_load(f))
+    def parse(path, *args, **kwargs):
+        if isinstance(path, str):
+            with open(path, 'r') as f:
+                return Config(yaml.safe_load(f), *args, **kwargs)
+        else:
+            return Config(yaml.safe_load(path), *args, **kwargs)
 
     @staticmethod
     def yaml_representer(dumper, value):
         return dumper.represent_data(
             value._items(allow_internal=False, allow_fallback=False))
 
     def dump(self, path):
-        with open(path, 'w+') as f:
-            yaml.dump(self, f)
+        if isinstance(path, str):
+            with open(path, 'w') as f:
+                yaml.dump(self, f)
+        else:
+            yaml.dump(self, path)
 
 yaml.add_multi_representer(Config, Config.yaml_representer)
 
 
-class DefaultConfig(Config):
+class DictConfig(Config):
     def __init__(self, default=None):
         super().__init__()
         self.__dict__['_default'] = default

datumaro/components/config_model.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: MIT
 
 from datumaro.components.config import Config, \
-    DefaultConfig as _DefaultConfig, \
+    DictConfig as _DictConfig, \
     SchemaBuilder as _SchemaBuilder
 
 
@@ -34,9 +34,9 @@ def __init__(self, config=None):
     .add('format_version', int) \
     \
     .add('subsets', list) \
-    .add('sources', lambda: _DefaultConfig(
+    .add('sources', lambda: _DictConfig(
         lambda v=None: Source(v))) \
-    .add('models', lambda: _DefaultConfig(
+    .add('models', lambda: _DictConfig(
         lambda v=None: Model(v))) \
     \
     .add('models_dir', str, internal=True) \

datumaro/components/converter.py

@@ -68,7 +68,8 @@ def _save_image(self, item, path=None):
 
         os.makedirs(osp.dirname(path), exist_ok=True)
         if src_ext == dst_ext and osp.isfile(item.image.path):
-            shutil.copyfile(item.image.path, path)
+            if item.image.path != path:
+                shutil.copyfile(item.image.path, path)
         elif src_ext == dst_ext and isinstance(item.image, ByteImage):
             with open(path, 'wb') as f:
                 f.write(item.image.get_bytes())

datumaro/plugins/voc_format/converter.py

@@ -13,7 +13,7 @@
 from lxml import etree as ET
 
 from datumaro.components.converter import Converter
-from datumaro.components.extractor import (DEFAULT_SUBSET_NAME, AnnotationType,
+from datumaro.components.extractor import (AnnotationType,
     CompiledMask, LabelCategories)
 from datumaro.util import find, str_to_bool
 from datumaro.util.image import save_image
@@ -104,6 +104,7 @@ def __init__(self, extractor, save_dir,
 
         if label_map is None:
             label_map = LabelmapType.source.name
+        assert isinstance(label_map, (str, dict)), label_map
         self._load_categories(label_map)
 
     def apply(self):
@@ -469,9 +470,9 @@ def _load_categories(self, label_map_source):
             label_map = parse_label_map(label_map_source)
 
         else:
-            raise Exception("Wrong labelmap specified, "
+            raise Exception("Wrong labelmap specified: '%s', "
                 "expected one of %s or a file path" % \
-                ', '.join(t.name for t in LabelmapType))
+                (label_map_source, ', '.join(t.name for t in LabelmapType)))
 
         # There must always be a label with color (0, 0, 0) at index 0
         bg_label = find(label_map.items(), lambda x: x[1][0] == (0, 0, 0))

datumaro/util/os_util.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2020 - 2021 Intel Corporation
+# Copyright (C) 2020-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT