feat(models): support example models

.github/workflows/ci.yml

@@ -89,45 +89,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           path: modflow-devtools
-
-      - name: Checkout modflow6
-        uses: actions/checkout@v4
-        with:
-          repository: MODFLOW-ORG/modflow6
-          path: modflow6
-
-      - name: Checkout modflow6 examples
-        uses: actions/checkout@v4
-        with:
-          repository: MODFLOW-ORG/modflow6-examples
-          path: modflow6-examples
-
-      - name: Checkout modflow6 test models
-        uses: actions/checkout@v3
-        with:
-          repository: MODFLOW-ORG/modflow6-testmodels
-          path: modflow6-testmodels
-
-      - name: Checkout modflow6 large test models
-        uses: actions/checkout@v3
-        with:
-          repository: MODFLOW-ORG/modflow6-largetestmodels
-          path: modflow6-largetestmodels
-
-      - name: Install executables
-        uses: modflowpy/install-modflow-action@v1
-
-      - name: Install nightly build
-        uses: modflowpy/install-modflow-action@v1
-        with:
-          repo: modflow6-nightly-build
-
-      - name: Setup GNU Fortran ${{ env.GCC_V }}
-        uses: awvwgk/setup-fortran@main
-        with:
-          compiler: gcc
-          version: ${{ env.GCC_V }}
-
+
       - name: Setup uv
         uses: astral-sh/setup-uv@v5
         with:
@@ -138,27 +100,6 @@ jobs:
         working-directory: modflow-devtools
         run: uv sync --all-extras
 
-      - name: Cache modflow6 examples
-        id: cache-examples
-        uses: actions/cache@v3
-        with:
-          path: modflow6-examples/examples
-          key: modflow6-examples-${{ hashFiles('modflow6-examples/data/**') }}
-
-      - name: Install extra Python packages
-        working-directory: modflow6-examples/etc
-        run: |
-          uv pip install -r requirements.pip.txt
-          uv pip install -r requirements.usgs.txt
-
-      - name: Update FloPy packages
-        run: uv run python -m flopy.mf6.utils.generate_classes --ref develop --no-backup
-
-      - name: Build modflow6 example models
-        if: steps.cache-examples.outputs.cache-hit != 'true'
-        working-directory: modflow6-examples/autotest
-        run: uv run pytest -v -n auto test_scripts.py --init
-
       - name: Run local tests
         working-directory: modflow-devtools/autotest
         env:

autotest/test_build.py

@@ -7,11 +7,11 @@
 from modflow_devtools.build import meson_build
 from modflow_devtools.markers import requires_exe
 
-_repos_path = environ.get("REPOS_PATH")
-if _repos_path is None:
-    _repos_path = Path(__file__).parent.parent.parent.parent
+if from_env := environ.get("REPOS_PATH"):
+    _repos_path = Path(from_env).expanduser().absolute()
+else:
+    _repos_path = Path(__file__).parents[3]
 _repos_path = Path(_repos_path).expanduser().absolute()
-_project_root_path = Path(__file__).parent.parent.parent.parent
 _modflow6_repo_path = _repos_path / "modflow6"
 _system = platform.system()
 _exe_ext = ".exe" if _system == "Windows" else ""

autotest/test_fixtures.py

@@ -7,7 +7,9 @@
 
 system = platform.system()
 proj_root = Path(__file__).parents[1]
-module_path = Path(inspect.getmodulename(__file__))
+module_name = inspect.getmodulename(__file__)
+assert module_name is not None  # appease mypy
+module_path = Path(module_name)
 
 
 # test temporary directory fixtures
@@ -250,36 +252,6 @@ def test_meta():
     assert pytest.main(args, plugins=[TestMeta()]) == ExitCode.OK
 
 
-# test fixtures dynamically generated from examples and test models
-
-
-def test_example_scenario(example_scenario):
-    assert isinstance(example_scenario, tuple)
-    name, namefiles = example_scenario
-    assert isinstance(name, str)
-    assert isinstance(namefiles, list)
-    assert all(namefile.is_file() for namefile in namefiles)
-
-
-def test_test_model_mf6(test_model_mf6):
-    assert isinstance(test_model_mf6, Path)
-    assert test_model_mf6.is_file()
-    assert test_model_mf6.name == "mfsim.nam"
-
-
-def test_test_model_mf5to6(test_model_mf5to6):
-    assert isinstance(test_model_mf5to6, Path)
-    assert test_model_mf5to6.is_file()
-    assert test_model_mf5to6.suffix == ".nam"
-
-
-def test_large_test_model(large_test_model):
-    print(large_test_model)
-    assert isinstance(large_test_model, Path)
-    assert large_test_model.is_file()
-    assert large_test_model.name == "mfsim.nam"
-
-
 # test tabular data format fixture
 
 

autotest/test_models.py

@@ -1,68 +1,65 @@
-import random
+from itertools import islice
 from pathlib import Path
 
 import pytest
 import tomli
 
 import modflow_devtools.models as models
+from modflow_devtools.misc import is_in_ci
 
-MODELS_TOML_PATH = Path(models.DATA_PATH) / models.MODELMAP_NAME
-
-
-@pytest.fixture
-def models_toml():
-    with MODELS_TOML_PATH.open("rb") as f:
-        return tomli.load(f)
-
-
-@pytest.fixture
-def temp_cache_dir(tmpdir, monkeypatch):
-    temp_dir = tmpdir.mkdir("pooch_cache")
-    monkeypatch.setenv("MF_DATA_DIR", str(temp_dir))
-    models.FETCHER.path = temp_dir  # Update the fetcher path
-    return temp_dir
+TAKE = 5 if is_in_ci() else None
+PROJ_ROOT = Path(__file__).parents[1]
+MODELMAP_PATH = PROJ_ROOT / "modflow_devtools" / "registry" / models.MODELMAP_FILE_NAME
+with MODELMAP_PATH.open("rb") as f:
+    MODELMAP = tomli.load(f)
 
 
 def test_registry():
-    assert models.FETCHER.registry is not None, "Registry was not loaded"
-    assert len(models.FETCHER.registry) > 0, "Registry is empty"
+    registry = models.get_registry()
+    assert registry is not None, "Registry was not loaded"
+    assert registry is models.POOCH.registry
+    assert any(registry), "Registry is empty"
 
 
-def test_model_map(models_toml):
-    assert models.model_map()
-    for model_name, files in models_toml.items():
-        assert model_name in models.model_map().keys(), (
-            f"Model {model_name} not found in model map"
-        )
-        assert files == models.model_map()[model_name], (
-            f"Files for model {model_name} do not match"
-        )
+@pytest.mark.parametrize("model_name, files", MODELMAP.items())
+def test_models(model_name, files):
+    model_names = models.list_models()
+    assert model_name in model_names, f"Model {model_name} not found in model map"
+    assert files == models.MODELMAP[model_name], (
+        f"Files for model {model_name} do not match"
+    )
+    assert hasattr(models, model_name), (
+        f"Function {model_name} not found in models module"
+    )
+    if "mf6" in model_name:
+        assert any(Path(f).name == "mfsim.nam" for f in files)
 
 
-def test_generated_functions_exist(models_toml):
-    for model_name in models_toml.keys():
-        assert hasattr(models, model_name), (
-            f"Function {model_name} not found in models module"
-        )
+@pytest.mark.parametrize("model_name, files", list(islice(MODELMAP.items(), TAKE)))
+def test_copy_to(model_name, files, tmp_path):
+    workspace = models.copy_to(tmp_path, model_name)
+    assert workspace.exists(), f"Model {model_name} was not copied to {tmp_path}"
+    assert workspace.is_dir(), f"Model {model_name} is not a directory"
+    assert len(list(workspace.iterdir())) == len(files), (
+        f"Model {model_name} does not have the correct number of files"
+    )
+    if "mf6" in model_name:
+        assert any(Path(f).name == "mfsim.nam" for f in files)
 
 
-def test_generated_functions_return_files(models_toml, temp_cache_dir):
-    for model_name, files in models_toml.items():
-        model_function = getattr(models, model_name)
-        fetched_files = model_function()
-        cached_files = temp_cache_dir.listdir()
-        assert isinstance(fetched_files, list), (
-            f"Function {model_name} did not return a list"
-        )
-        assert len(fetched_files) == len(files), (
-            f"Function {model_name} did not return the correct number of files"
+@pytest.mark.parametrize("model_name, files", list(islice(MODELMAP.items(), TAKE)))
+def test_generated_functions_return_files(model_name, files):
+    model_function = getattr(models, model_name)
+    fetched_files = model_function()
+    assert isinstance(fetched_files, list), (
+        f"Function {model_name} did not return a list"
+    )
+    assert len(fetched_files) == len(files), (
+        f"Function {model_name} did not return the correct number of files"
+    )
+    if "mf6" in model_name:
+        assert any(Path(f).name == "mfsim.nam" for f in files)
+    for fetched_file in fetched_files:
+        assert Path(fetched_file).exists(), (
+            f"Fetched file {fetched_file} does not exist"
         )
-        for fetched_file in fetched_files:
-            assert Path(fetched_file).exists(), (
-                f"Fetched file {fetched_file} does not exist"
-            )
-            assert Path(temp_cache_dir) / Path(fetched_file).name in cached_files, (
-                f"Fetched file {fetched_file} is not in the temp cache directory"
-            )
-        if random.randint(0, 5) % 5 == 0:
-            break  # just the first few so we dont ddos github

docs/md/models.md

@@ -16,15 +16,15 @@ files = models.some_model()
 
 The `make_registry.py` script is responsible for generating a registry text file and a mapping between files and models. This script should be run in the CI pipeline at release time before the package is built. The generated registry file and model mapping are used to create a pooch instance for fetching model files, and should be distributed with the package.
 
-The script can be executed with `python -m modflow_devtools.make_registry` and accepts the following options:
+The script can be executed with `python -m modflow_devtools.make_registry`. It accepts a single positional argument, specifying the base directory containing model directories. It accepts two named arguments:
 
-- `--path` or `-p`: Specifies the directory containing model directories. If not provided, the default path is used.
 - `--append` or `-a`: If specified, the script will append to the existing registry file instead of overwriting it.
-- `--base-url` or `-b`: Specifies the base URL for the registry file. If not provided, the default base URL is used.
+- `--url` or `-u`: Specifies the base URL for the registry file. If not provided, the default base URL is used.
 
-For example, to create a registry of models in the MF6 test models repositories, each of which is checked out in the current working directory:
+For example, to create a registry of models in the MF6 examples and test models repositories, assuming each is checked out next to this project:
 
 ```shell
-python -m modflow_devtools.make_registry -p modflow6-testmodels -b https://github.com/MODFLOW-ORG/modflow6-testmodels/raw/master
-python -m modflow_devtools.make_registry -a -p modflow6-largetestmodels -b https://github.com/MODFLOW-ORG/modflow6-largetestmodels/raw/master
+python -m modflow_devtools.make_registry ../modflow6-examples/examples --url https://github.com/MODFLOW-ORG/modflow6-examples/releases/download/current/mf6examples.zip
+python -m modflow_devtools.make_registry ../modflow6-testmodels --append --url https://github.com/MODFLOW-ORG/modflow6-testmodels/raw/master
+python -m modflow_devtools.make_registry ../modflow6-largetestmodels --append --url https://github.com/MODFLOW-ORG/modflow6-largetestmodels/raw/master
 ```