t

Author: Maxim Zhiltsov

datumaro/components/config.py

@@ -238,8 +238,14 @@ def __init__(self, default=None):
         self.__dict__['_default'] = default
 
     def set(self, key, value):
-        if key not in self.keys(allow_fallback=False):
-            value = self._default(value)
-            return super().set(key, value)
-        else:
-            return super().set(key, value)
+        if self._default is not None:
+            schema_entry_instance = self._default(value)
+            if not isinstance(value, type(schema_entry_instance)):
+                if isinstance(value, dict) and \
+                        isinstance(schema_entry_instance, Config):
+                    schema_entry_instance.update(value)
+                    value = schema_entry_instance
+                else:
+                    raise Exception("Can not set key '%s' - schema mismatch" % (key))
+
+        return super().set(key, value)

datumaro/components/config_model.py

@@ -7,11 +7,25 @@
     DictConfig as _DictConfig, \
     SchemaBuilder as _SchemaBuilder
 
+from datumaro.util import find
+
+
+REMOTE_SCHEMA = _SchemaBuilder() \
+    .add('url', str) \
+    .add('type', str) \
+    .add('options', dict) \
+    .build()
+
+class Remote(Config):
+    def __init__(self, config=None):
+        super().__init__(config, schema=REMOTE_SCHEMA)
+
 
 SOURCE_SCHEMA = _SchemaBuilder() \
     .add('url', str) \
     .add('format', str) \
     .add('options', dict) \
+    .add('remote', str) \
     .build()
 
 class Source(Config):
@@ -29,35 +43,87 @@ def __init__(self, config=None):
         super().__init__(config, schema=MODEL_SCHEMA)
 
 
+BUILDSTAGE_SCHEMA = _SchemaBuilder() \
+    .add('name', str) \
+    .add('type', str) \
+    .add('kind', str) \
+    .add('params', dict) \
+    .build()
+
+class BuildStage(Config):
+    def __init__(self, config=None):
+        super().__init__(config, schema=BUILDSTAGE_SCHEMA)
+
+BUILDTARGET_SCHEMA = _SchemaBuilder() \
+    .add('stages', list) \
+    .add('parents', list) \
+    .build()
+
+class BuildTarget(Config):
+    def __init__(self, config=None):
+        super().__init__(config, schema=BUILDTARGET_SCHEMA)
+        self.stages = [BuildStage(o) for o in self.stages]
+
+    @property
+    def root(self):
+        return self.stages[0]
+
+    @property
+    def head(self):
+        return self.stages[-1]
+
+    def find_stage(self, stage):
+        if stage == 'root':
+            return self.root
+        elif stage == 'head':
+            return self.head
+        return find(self.stages, lambda x: x.name == stage or x == stage)
+
+    def get_stage(self, stage):
+        res = self.find_stage(stage)
+        if res is None:
+            raise KeyError("Unknown stage '%s'" % stage)
+        return res
+
+
 PROJECT_SCHEMA = _SchemaBuilder() \
     .add('project_name', str) \
     .add('format_version', int) \
     \
-    .add('subsets', list) \
-    .add('sources', lambda: _DictConfig(
-        lambda v=None: Source(v))) \
-    .add('models', lambda: _DictConfig(
-        lambda v=None: Model(v))) \
+    .add('default_repo', str) \
+    .add('remotes', lambda: _DictConfig(lambda v=None: Remote(v))) \
+    .add('sources', lambda: _DictConfig(lambda v=None: Source(v))) \
+    .add('models', lambda: _DictConfig(lambda v=None: Model(v))) \
+    .add('build_targets', lambda: _DictConfig(lambda v=None: BuildTarget(v))) \
     \
     .add('models_dir', str, internal=True) \
     .add('plugins_dir', str, internal=True) \
     .add('sources_dir', str, internal=True) \
     .add('dataset_dir', str, internal=True) \
+    .add('dvc_aux_dir', str, internal=True) \
+    .add('pipelines_dir', str, internal=True) \
+    .add('build_dir', str, internal=True) \
     .add('project_filename', str, internal=True) \
     .add('project_dir', str, internal=True) \
     .add('env_dir', str, internal=True) \
+    .add('detached', bool, internal=True) \
     .build()
 
 PROJECT_DEFAULT_CONFIG = Config({
     'project_name': 'undefined',
-    'format_version': 1,
+    'format_version': 2,
 
     'sources_dir': 'sources',
     'dataset_dir': 'dataset',
     'models_dir': 'models',
     'plugins_dir': 'plugins',
+    'dvc_aux_dir': 'dvc_aux',
+    'pipelines_dir': 'dvc_pipelines',
+    'build_dir': 'build',
 
+    'default_repo': 'origin',
     'project_filename': 'config.yaml',
     'project_dir': '',
     'env_dir': '.datumaro',
+    'detached': False,
 }, mutable=False, schema=PROJECT_SCHEMA)

datumaro/components/dataset.py

@@ -452,8 +452,9 @@ def __init__(self, source: IDataset = None,
 
         self._format = DEFAULT_FORMAT
         self._source_path = None
+        self._options = {}
 
-    def define_categories(self, categories: Dict):
+    def define_categories(self, categories: CategoriesInfo):
         assert not self._data._categories and self._data._source is None
         self._data._categories = categories
 
@@ -626,8 +627,7 @@ def import_from(cls, path: str, format: str = None, env: Environment = None,
         if format in env.importers:
             importer = env.make_importer(format)
             with logging_disabled(log.INFO):
-                project = importer(path, **kwargs)
-            detected_sources = list(project.config.sources.values())
+                detected_sources = importer(path, **kwargs)
         elif format in env.extractors:
             detected_sources = [{
                 'url': path, 'format': format, 'options': kwargs

datumaro/components/environment.py

@@ -4,14 +4,12 @@
 
 from functools import partial
 from glob import glob
-import git
 import inspect
 import logging as log
 import os
 import os.path as osp
 
 from datumaro.components.config import Config
-from datumaro.components.config_model import Model, Source
 from datumaro.util.os_util import import_foreign_module
 
 
@@ -46,29 +44,6 @@ def __getitem__(self, key):
     def __contains__(self, key):
         return key in self.items
 
-
-class ModelRegistry(Registry):
-    def __init__(self, config=None):
-        super().__init__(config, item_type=Model)
-
-    def load(self, config):
-        # TODO: list default dir, insert values
-        if 'models' in config:
-            for name, model in config.models.items():
-                self.register(name, model)
-
-
-class SourceRegistry(Registry):
-    def __init__(self, config=None):
-        super().__init__(config, item_type=Source)
-
-    def load(self, config):
-        # TODO: list default dir, insert values
-        if 'sources' in config:
-            for name, source in config.sources.items():
-                self.register(name, source)
-
-
 class PluginRegistry(Registry):
     def __init__(self, config=None, builtin=None, local=None):
         super().__init__(config)
@@ -85,47 +60,6 @@ def __init__(self, config=None, builtin=None, local=None):
                 self.register(k, v)
 
 
-class GitWrapper:
-    def __init__(self, config=None):
-        self.repo = None
-
-        if config is not None and config.project_dir:
-            self.init(config.project_dir)
-
-    @staticmethod
-    def _git_dir(base_path):
-        return osp.join(base_path, '.git')
-
-    @classmethod
-    def spawn(cls, path):
-        spawn = not osp.isdir(cls._git_dir(path))
-        repo = git.Repo.init(path=path)
-        if spawn:
-            repo.config_writer().set_value("user", "name", "User") \
-                .set_value("user", "email", "user@nowhere.com") \
-                .release()
-            # gitpython does not support init, use git directly
-            repo.git.init()
-            repo.git.commit('-m', 'Initial commit', '--allow-empty')
-        return repo
-
-    def init(self, path):
-        self.repo = self.spawn(path)
-        return self.repo
-
-    def is_initialized(self):
-        return self.repo is not None
-
-    def create_submodule(self, name, dst_dir, **kwargs):
-        self.repo.create_submodule(name, dst_dir, **kwargs)
-
-    def has_submodule(self, name):
-        return name in [submodule.name for submodule in self.repo.submodules]
-
-    def remove_submodule(self, name, **kwargs):
-        return self.repo.submodule(name).remove(**kwargs)
-
-
 class Environment:
     _builtin_plugins = None
     PROJECT_EXTRACTOR_NAME = 'datumaro_project'
@@ -136,11 +70,6 @@ def __init__(self, config=None):
         config = Config(config,
             fallback=PROJECT_DEFAULT_CONFIG, schema=PROJECT_SCHEMA)
 
-        self.models = ModelRegistry(config)
-        self.sources = SourceRegistry(config)
-
-        self.git = GitWrapper(config)
-
         env_dir = osp.join(config.project_dir, config.env_dir)
         builtin = self._load_builtin_plugins()
         custom = self._load_plugins2(osp.join(env_dir, config.plugins_dir))
@@ -284,12 +213,6 @@ def make_converter(self, name, *args, **kwargs):
     def make_transform(self, name, *args, **kwargs):
         return partial(self.transforms.get(name), *args, **kwargs)
 
-    def register_model(self, name, model):
-        self.models.register(name, model)
-
-    def unregister_model(self, name):
-        self.models.unregister(name)
-
     def is_format_known(self, name):
         return name in self.importers or name in self.extractors
 

datumaro/components/extractor.py

@@ -649,22 +649,18 @@ def find_sources(cls, path) -> List[Dict]:
         raise NotImplementedError()
 
     def __call__(self, path, **extra_params):
-        from datumaro.components.project import Project # cyclic import
-        project = Project()
-
-        sources = self.find_sources(osp.normpath(path))
-        if len(sources) == 0:
+        found_sources = self.find_sources(osp.normpath(path))
+        if len(found_sources) == 0:
             raise Exception("Failed to find dataset at '%s'" % path)
 
-        for desc in sources:
+        sources = []
+        for desc in found_sources:
             params = dict(extra_params)
             params.update(desc.get('options', {}))
             desc['options'] = params
+            sources.append(desc)
 
-            source_name = osp.splitext(osp.basename(desc['url']))[0]
-            project.add_source(source_name, desc)
-
-        return project
+        return sources
 
     @classmethod
     def _find_sources_recursive(cls, path, ext, extractor_name,