From 3bdaa394383ce6607da3f2a7b7542fabc9c468f7 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 18 Mar 2025 10:40:47 -0400 Subject: [PATCH 1/2] fix: filelock when fetching zips --- autotest/mf6examples.zip.lock | 0 modflow_devtools/models.py | 27 +++++++++++++++------------ pyproject.toml | 2 ++ 3 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 autotest/mf6examples.zip.lock diff --git a/autotest/mf6examples.zip.lock b/autotest/mf6examples.zip.lock new file mode 100644 index 00000000..e69de29b diff --git a/modflow_devtools/models.py b/modflow_devtools/models.py index 05842cdd..15037c28 100644 --- a/modflow_devtools/models.py +++ b/modflow_devtools/models.py @@ -12,6 +12,7 @@ import pooch import tomli +from filelock import FileLock import modflow_devtools @@ -26,7 +27,7 @@ # set up the pooch FETCHERS = {} -REGISTRY: dict[str, str] = {} +REGISTRY: dict[str, dict] = {} MODELMAP: dict[str, list[str]] = {} EXAMPLES: dict[str, list[str]] = {} POOCH = pooch.create( @@ -42,12 +43,13 @@ def _fetch_files(): return [Path(POOCH.fetch(fname)) for fname in file_names] def _fetch_zip(zip_name): - return [ - Path(f) - for f in POOCH.fetch( - zip_name, processor=pooch.Unzip(members=MODELMAP[model_name]) - ) - ] + with FileLock(f"{zip_name}.lock"): + return [ + Path(f) + for f in POOCH.fetch( + zip_name, processor=pooch.Unzip(members=MODELMAP[model_name]) + ) + ] urls = [POOCH.registry[fname] for fname in file_names] if not any(url for url in urls) or set(urls) == {f"{BASE_URL}/{ZIP_NAME}"}: @@ -62,10 +64,11 @@ def _fetch_zip(zip_name): with pkg_resources.open_binary( REGISTRY_ANCHOR, REGISTRY_FILE_NAME ) as registry_file: - registry = tomli.load(registry_file) - urls = {k: v["url"] for k, v in registry.items() if v.get("url", None)} - registry = {k: v.get("hash", None) for k, v in registry.items()} - POOCH.registry = registry + REGISTRY = tomli.load(registry_file) + # extract urls then drop them, leaving a direct map of name to hash + urls = {k: v["url"] for k, v in REGISTRY.items() if v.get("url", None)} + REGISTRY = {k: v.get("hash", None) for k, v in REGISTRY.items()} + POOCH.registry = REGISTRY POOCH.urls = urls except: # noqa: E722 warn( @@ -105,7 +108,7 @@ def get_examples() -> dict[str, list[str]]: def get_registry() -> dict[str, str]: - return POOCH.registry + return REGISTRY def list_models() -> list[str]: diff --git a/pyproject.toml b/pyproject.toml index ae0f0c6b..e8f637ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,7 @@ dfn = [ ] models = [ "boltons", + "filelock", "pooch", "tomli", "tomli-w" @@ -120,6 +121,7 @@ dfn = [ ] models = [ "boltons", + "filelock", "pooch", "tomli", "tomli-w" From 2aee2c3ab762145d4fbc4eaba224a8f6e4a25517 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 18 Mar 2025 10:57:21 -0400 Subject: [PATCH 2/2] hints --- modflow_devtools/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modflow_devtools/models.py b/modflow_devtools/models.py index 15037c28..634cf675 100644 --- a/modflow_devtools/models.py +++ b/modflow_devtools/models.py @@ -27,9 +27,9 @@ # set up the pooch FETCHERS = {} -REGISTRY: dict[str, dict] = {} -MODELMAP: dict[str, list[str]] = {} -EXAMPLES: dict[str, list[str]] = {} +REGISTRY: dict = {} +MODELMAP: dict = {} +EXAMPLES: dict = {} POOCH = pooch.create( path=pooch.os_cache(modflow_devtools.__name__.replace("_", "-")), base_url=BASE_URL,