Support Zarr directories

dandi/cli/base.py

@@ -11,6 +11,25 @@
 # Aux common functionality
 
 
+class IntColonInt(click.ParamType):
+    name = "int:int"
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, str):
+            v1, colon, v2 = value.partition(":")
+            try:
+                v1 = int(v1)
+                v2 = int(v2) if colon else None
+            except ValueError:
+                self.fail("Value must be of the form `N[:M]`", param, ctx)
+            return (v1, v2)
+        else:
+            return value
+
+    def get_metavar(self, param):
+        return "N[:M]"
+
+
 # ???: could make them always available but hidden
 #  via  hidden=True.
 def devel_option(*args, **kwargs):

dandi/cli/cmd_download.py

@@ -2,7 +2,7 @@
 
 import click
 
-from .base import instance_option, map_to_click_exceptions
+from .base import IntColonInt, instance_option, map_to_click_exceptions
 from ..consts import known_instances, known_instances_rev
 from ..dandiarchive import _dandi_url_parser, parse_dandi_url
 
@@ -70,8 +70,9 @@ def get_metavar(self, param):
 @click.option(
     "-J",
     "--jobs",
-    help="Number of parallel download jobs.",
-    default=6,  # TODO: come up with smart auto-scaling etc
+    type=IntColonInt(),
+    help="Number of parallel download jobs and, optionally number of subjobs per Zarr asset",
+    default="6",  # TODO: come up with smart auto-scaling etc
     show_default=True,
 )
 @click.option(
@@ -141,7 +142,8 @@ def download(
         output_dir,
         existing=existing,
         format=format,
-        jobs=jobs,
+        jobs=jobs[0],
+        jobs_per_zarr=jobs[1],
         get_metadata="dandiset.yaml" in download_types,
         get_assets="assets" in download_types,
         sync=sync,

dandi/cli/cmd_ls.py

@@ -6,6 +6,7 @@
 
 from .base import devel_option, lgr, map_to_click_exceptions
 from ..dandiarchive import DandisetURL, _dandi_url_parser, parse_dandi_url
+from ..misctypes import Digest
 from ..utils import is_url
 
 # TODO: all the recursion options etc
@@ -354,8 +355,7 @@ def fn():
                         rec = nwb2asset(
                             path,
                             schema_version=schema,
-                            digest=digest,
-                            digest_type="dandi_etag",
+                            digest=Digest.dandi_etag(digest),
                         ).json_dict()
                 else:
                     rec = get_metadata(path)

dandi/cli/cmd_upload.py

@@ -1,32 +1,14 @@
 import click
 
 from .base import (
+    IntColonInt,
     devel_debug_option,
     devel_option,
     instance_option,
     map_to_click_exceptions,
 )
 
 
-class IntColonInt(click.ParamType):
-    name = "int:int"
-
-    def convert(self, value, param, ctx):
-        if isinstance(value, str):
-            v1, colon, v2 = value.partition(":")
-            try:
-                v1 = int(v1)
-                v2 = int(v2) if colon else None
-            except ValueError:
-                self.fail("Value must be of the form `N[:M]`", param, ctx)
-            return (v1, v2)
-        else:
-            return value
-
-    def get_metavar(self, param):
-        return "N[:M]"
-
-
 @click.command()
 # @dandiset_path_option(
 #     help="Top directory (local) of the dandiset.  Files will be uploaded with "
@@ -102,9 +84,9 @@ def upload(
     Local Dandiset should pass validation.  For that, the assets should first
     be organized using the `dandi organize` command.
 
-    By default all .nwb files in the Dandiset (excluding directories starting
-    with a period) will be considered for the upload.  You can point to
-    specific files you would like to validate and have uploaded.
+    By default all .nwb, .zarr, and .ngff assets in the Dandiset (ignoring
+    directories starting with a period) will be considered for the upload.  You
+    can point to specific files you would like to validate and have uploaded.
     """
     from ..upload import upload
 

dandi/cli/cmd_validate.py

@@ -18,7 +18,7 @@
 @devel_debug_option()
 @map_to_click_exceptions
 def validate(paths, schema=None, devel_debug=False, allow_any_path=False):
-    """Validate files for NWB (and DANDI) compliance.
+    """Validate files for NWB and DANDI compliance.
 
     Exits with non-0 exit code if any file is not compliant.
     """
@@ -43,7 +43,7 @@ def validate(paths, schema=None, devel_debug=False, allow_any_path=False):
     all_files_errors = {}
     nfiles = 0
     for path, errors in validate_(
-        paths,
+        *paths,
         schema_version=schema,
         devel_debug=devel_debug,
         allow_any_path=allow_any_path,

dandi/cli/tests/test_download.py

@@ -18,6 +18,7 @@ def test_download_defaults(mocker):
         existing="error",
         format="pyout",
         jobs=6,
+        jobs_per_zarr=None,
         get_metadata=True,
         get_assets=True,
         sync=False,
@@ -34,6 +35,7 @@ def test_download_all_types(mocker):
         existing="error",
         format="pyout",
         jobs=6,
+        jobs_per_zarr=None,
         get_metadata=True,
         get_assets=True,
         sync=False,
@@ -50,6 +52,7 @@ def test_download_metadata_only(mocker):
         existing="error",
         format="pyout",
         jobs=6,
+        jobs_per_zarr=None,
         get_metadata=True,
         get_assets=False,
         sync=False,
@@ -66,6 +69,7 @@ def test_download_assets_only(mocker):
         existing="error",
         format="pyout",
         jobs=6,
+        jobs_per_zarr=None,
         get_metadata=False,
         get_assets=True,
         sync=False,
@@ -94,6 +98,7 @@ def test_download_gui_instance_in_dandiset(mocker):
         existing="error",
         format="pyout",
         jobs=6,
+        jobs_per_zarr=None,
         get_metadata=True,
         get_assets=True,
         sync=False,
@@ -113,6 +118,7 @@ def test_download_api_instance_in_dandiset(mocker):
         existing="error",
         format="pyout",
         jobs=6,
+        jobs_per_zarr=None,
         get_metadata=True,
         get_assets=True,
         sync=False,
@@ -136,6 +142,7 @@ def test_download_url_instance_match(mocker):
         existing="error",
         format="pyout",
         jobs=6,
+        jobs_per_zarr=None,
         get_metadata=True,
         get_assets=True,
         sync=False,

dandi/consts.py

@@ -142,3 +142,12 @@ class EmbargoStatus(Enum):
 #: HTTP response status codes that should always be retried (until we run out
 #: of retries)
 RETRY_STATUSES = (500, 502, 503, 504)
+
+#: Maximum allowed depth of a Zarr directory tree
+MAX_ZARR_DEPTH = 5
+
+#: MIME type assigned to & used to identify Zarr assets
+ZARR_MIME_TYPE = "application/x-zarr"
+
+#: Maximum number of Zarr directory entries to upload at once
+ZARR_UPLOAD_BATCH_SIZE = 255