From 0606aedcd43ec4b90fa720902cbbefda04f7caf4 Mon Sep 17 00:00:00 2001 From: JoerivanEngelen Date: Tue, 3 Dec 2024 14:58:10 +0100 Subject: [PATCH 1/8] Add CopyFiles package and tests --- imod/msw/copy_files.py | 43 ++++++++++++++++ imod/tests/test_msw/test_copy_files.py | 69 ++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 imod/msw/copy_files.py create mode 100644 imod/tests/test_msw/test_copy_files.py diff --git a/imod/msw/copy_files.py b/imod/msw/copy_files.py new file mode 100644 index 000000000..0a7a8f42a --- /dev/null +++ b/imod/msw/copy_files.py @@ -0,0 +1,43 @@ +from pathlib import Path +from shutil import copy2 +from typing import cast + +import numpy as np +import xarray as xr + +from imod.msw.pkgbase import MetaSwapPackage +from imod.typing import Imod5DataDict + + +class CopyFiles(MetaSwapPackage): + def __init__(self, paths: list[str]): + super().__init__() + paths_da = xr.DataArray( + paths, coords={"file_nr": np.arange(len(paths))}, dims=("file_nr",) + ) + self.dataset["paths"] = paths_da + + @classmethod + def from_imod5_data(cls, imod5_data: Imod5DataDict): + paths = cast(list[list[str]], imod5_data["extra"]["paths"]) + paths_unpacked = [Path(p[0]) for p in paths] + files_to_filter = ( + "mete_grid.inp", + "para_sim.inp", + "svat2precgrid.inp", + "svat2etrefgrid.inp", + ) + paths_filtered = [ + str(p) for p in paths_unpacked if p.name.lower() not in files_to_filter + ] + + return cls(paths_filtered) + + def write(self, directory: str | Path, *_): + directory = Path(directory) + + src_paths = [Path(p) for p in self.dataset["paths"].to_numpy()] + dst_paths = [directory / p.name for p in src_paths] + + for src_path, dst_path in zip(src_paths, dst_paths): + copy2(src_path, dst_path) diff --git a/imod/tests/test_msw/test_copy_files.py b/imod/tests/test_msw/test_copy_files.py new file mode 100644 index 000000000..3a21426d0 --- /dev/null +++ b/imod/tests/test_msw/test_copy_files.py @@ -0,0 +1,69 @@ +from pytest_cases import parametrize_with_cases + +from imod.msw.copy_files import CopyFiles + + +def setup_src_files(directory, filenames): + paths = [directory / filename for filename in filenames] + for p in paths: + with open(p, mode="w") as f: + f.write("test") + return paths + + +def case_simple_files(tmp_path_factory): + directory = tmp_path_factory.mktemp("simple_files") + filenames = [ + "a.inp", + "b.inp", + "c.inp", + ] + return setup_src_files(directory, filenames) + + +def case_imod5_extra_files(tmp_path_factory): + directory = tmp_path_factory.mktemp("imod5_extra_files") + filenames = [ + "a.inp", + "b.inp", + "c.inp", + "mete_grid.inp", + "para_sim.inp", + "svat2precgrid.inp", + "svat2etrefgrid.inp", + ] + return setup_src_files(directory, filenames) + + +@parametrize_with_cases("src_files", cases=".") +def test_copyfile_init(src_files): + # Act + copyfiles = CopyFiles(src_files) + # Arrange + assert "paths" in copyfiles.dataset.keys() + assert len(copyfiles.dataset["paths"]) == len(src_files) + + +@parametrize_with_cases("src_files", cases=".") +def test_copyfile_write(src_files, tmp_path): + # Arrange + expected_filenames = {f.name for f in src_files} + # Act + copyfiles = CopyFiles(src_files) + copyfiles.write(tmp_path) + # Assert + actual_filepaths = tmp_path.glob("*.inp") + actual_filenames = {f.name for f in actual_filepaths} + diff = expected_filenames ^ actual_filenames + assert len(diff) == 0 + + +@parametrize_with_cases("src_files", cases=".") +def test_from_imod5_data(src_files): + # Arrange + imod5_ls = [[p] for p in src_files] + imod5_data = {"extra": {"paths": imod5_ls}} + # Act + copyfiles = CopyFiles.from_imod5_data(imod5_data) + # Assert + len(copyfiles.dataset["paths"]) == 3 From 5585cbbc0dc77dd2ab3eb062f1af73321d5e68c7 Mon Sep 17 00:00:00 2001 From: JoerivanEngelen Date: Tue, 3 Dec 2024 15:33:07 +0100 Subject: [PATCH 2/8] Log removed files --- imod/msw/copy_files.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/imod/msw/copy_files.py b/imod/msw/copy_files.py index 0a7a8f42a..312ece96e 100644 --- a/imod/msw/copy_files.py +++ b/imod/msw/copy_files.py @@ -5,9 +5,15 @@ import numpy as np import xarray as xr +from imod.logging import logger +from imod.logging.loglevel import LogLevel from imod.msw.pkgbase import MetaSwapPackage from imod.typing import Imod5DataDict +_LOG_MESSAGE_TEMPLATE = """\ +Will not copy files {filtered}, these will be generated by iMOD Python +instead.""" + class CopyFiles(MetaSwapPackage): def __init__(self, paths: list[str]): @@ -20,7 +26,7 @@ def __init__(self, paths: list[str]): @classmethod def from_imod5_data(cls, imod5_data: Imod5DataDict): paths = cast(list[list[str]], imod5_data["extra"]["paths"]) - paths_unpacked = [Path(p[0]) for p in paths] + paths_unpacked = {Path(p[0]) for p in paths} files_to_filter = ( "mete_grid.inp", "para_sim.inp", @@ -30,7 +36,13 @@ def from_imod5_data(cls, imod5_data: Imod5DataDict): paths_filtered = [ str(p) for p in paths_unpacked if p.name.lower() not in files_to_filter ] - + paths_filtered_away = paths_unpacked - set(paths_filtered) + if paths_filtered_away: + log_message = _LOG_MESSAGE_TEMPLATE.format(filtered=paths_filtered_away) + logger.log( + loglevel=LogLevel.INFO, + message=log_message, + ) return cls(paths_filtered) def write(self, directory: str | Path, *_): From fad585128cbbf8fd7705fbc75f5b48c83b97f9e7 Mon Sep 17 00:00:00 2001 From: JoerivanEngelen Date: Tue, 3 Dec 2024 15:46:13 +0100 Subject: [PATCH 3/8] fix mypy error --- imod/msw/copy_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imod/msw/copy_files.py b/imod/msw/copy_files.py index 312ece96e..ccc9a1ab0 100644 --- a/imod/msw/copy_files.py +++ b/imod/msw/copy_files.py @@ -36,7 +36,7 @@ def from_imod5_data(cls, imod5_data: Imod5DataDict): paths_filtered = [ str(p) for p in paths_unpacked if p.name.lower() not in files_to_filter ] - paths_filtered_away = paths_unpacked - set(paths_filtered) + paths_filtered_away = {str(p) for p in paths_unpacked} - set(paths_filtered) if paths_filtered_away: log_message = _LOG_MESSAGE_TEMPLATE.format(filtered=paths_filtered_away) logger.log( From 8977eabace392a134405fc025837dadc24a562f3 Mon Sep 17 00:00:00 2001 From: JoerivanEngelen Date: Tue, 3 Dec 2024 15:51:35 +0100 Subject: [PATCH 4/8] Update changelog --- docs/api/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api/changelog.rst b/docs/api/changelog.rst index 330bf8144..a29925e41 100644 --- a/docs/api/changelog.rst +++ b/docs/api/changelog.rst @@ -14,6 +14,8 @@ Added - :class:`imod.msw.MeteoGridCopy` to copy existing `mete_grid.inp` files, so ASCII grids in large existing meteo databases do not have to be read. +- :class:`imod.msw.CopyFiles` to copy settings and lookup tables in existing + ``.inp`` files. - :meth:`imod.mf6.Recharge.from_imod5_cap_data` to construct a recharge package for coupling a MODFLOW6 model to MetaSWAP. From 7d5e03fe0ec2cb1e2eb56b63afe38bca20697510 Mon Sep 17 00:00:00 2001 From: JoerivanEngelen Date: Tue, 3 Dec 2024 16:08:49 +0100 Subject: [PATCH 5/8] Make CopyFiles available under imod.msw --- imod/msw/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imod/msw/__init__.py b/imod/msw/__init__.py index 83f8df90b..91f145c6e 100644 --- a/imod/msw/__init__.py +++ b/imod/msw/__init__.py @@ -1,3 +1,4 @@ +from imod.msw.copy_files import CopyFiles from imod.msw.coupler_mapping import CouplerMapping from imod.msw.grid_data import GridData from imod.msw.idf_mapping import IdfMapping From e38befdd1fb27e0b4ec5241331f20eaa7619a960 Mon Sep 17 00:00:00 2001 From: JoerivanEngelen Date: Wed, 4 Dec 2024 12:03:12 +0100 Subject: [PATCH 6/8] Rename setup function --- imod/tests/test_msw/test_copy_files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imod/tests/test_msw/test_copy_files.py b/imod/tests/test_msw/test_copy_files.py index 3a21426d0..622333194 100644 --- a/imod/tests/test_msw/test_copy_files.py +++ b/imod/tests/test_msw/test_copy_files.py @@ -3,7 +3,7 @@ from imod.msw.copy_files import CopyFiles -def setup_src_files(directory, filenames): +def write_test_files(directory, filenames): paths = [directory / filename for filename in filenames] for p in paths: with open(p, mode="w") as f: @@ -18,7 +18,7 @@ def case_simple_files(tmp_path_factory): "b.inp", "c.inp", ] - return setup_src_files(directory, filenames) + return write_test_files(directory, filenames) def case_imod5_extra_files(tmp_path_factory): @@ -32,7 +32,7 @@ def case_imod5_extra_files(tmp_path_factory): "svat2precgrid.inp", "svat2etrefgrid.inp", ] - return setup_src_files(directory, filenames) + return write_test_files(directory, filenames) @parametrize_with_cases("src_files", cases=".") From 404ec6ef97dd1d7ecebff238a4f53b7169aa1875 Mon Sep 17 00:00:00 2001 From: JoerivanEngelen Date: Wed, 4 Dec 2024 12:03:37 +0100 Subject: [PATCH 7/8] Better package name --- imod/msw/__init__.py | 2 +- imod/msw/copy_files.py | 2 +- imod/tests/test_msw/test_copy_files.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/imod/msw/__init__.py b/imod/msw/__init__.py index 91f145c6e..d8d053c49 100644 --- a/imod/msw/__init__.py +++ b/imod/msw/__init__.py @@ -1,4 +1,4 @@ -from imod.msw.copy_files import CopyFiles +from imod.msw.copy_files import FileCopier from imod.msw.coupler_mapping import CouplerMapping from imod.msw.grid_data import GridData from imod.msw.idf_mapping import IdfMapping diff --git a/imod/msw/copy_files.py b/imod/msw/copy_files.py index ccc9a1ab0..21b89ae40 100644 --- a/imod/msw/copy_files.py +++ b/imod/msw/copy_files.py @@ -15,7 +15,7 @@ instead.""" -class CopyFiles(MetaSwapPackage): +class FileCopier(MetaSwapPackage): def __init__(self, paths: list[str]): super().__init__() paths_da = xr.DataArray( diff --git a/imod/tests/test_msw/test_copy_files.py b/imod/tests/test_msw/test_copy_files.py index 622333194..80d8ceb08 100644 --- a/imod/tests/test_msw/test_copy_files.py +++ b/imod/tests/test_msw/test_copy_files.py @@ -1,6 +1,6 @@ from pytest_cases import parametrize_with_cases -from imod.msw.copy_files import CopyFiles +from imod.msw.copy_files import FileCopier def write_test_files(directory, filenames): @@ -38,7 +38,7 @@ def case_imod5_extra_files(tmp_path_factory): @parametrize_with_cases("src_files", cases=".") def test_copyfile_init(src_files): # Act - copyfiles = CopyFiles(src_files) + copyfiles = FileCopier(src_files) # Arrange assert "paths" in copyfiles.dataset.keys() assert len(copyfiles.dataset["paths"]) == len(src_files) @@ -49,7 +49,7 @@ def test_copyfile_write(src_files, tmp_path): # Arrange expected_filenames = {f.name for f in src_files} # Act - copyfiles = CopyFiles(src_files) + copyfiles = FileCopier(src_files) copyfiles.write(tmp_path) # Assert actual_filepaths = tmp_path.glob("*.inp") @@ -64,6 +64,6 @@ def test_from_imod5_data(src_files): imod5_ls = [[p] for p in src_files] imod5_data = {"extra": {"paths": imod5_ls}} # Act - copyfiles = CopyFiles.from_imod5_data(imod5_data) + copyfiles = FileCopier.from_imod5_data(imod5_data) # Assert len(copyfiles.dataset["paths"]) == 3 From 8c899449dce3a9514029e97cc32e94f1292c1f38 Mon Sep 17 00:00:00 2001 From: JoerivanEngelen Date: Wed, 4 Dec 2024 12:09:50 +0100 Subject: [PATCH 8/8] Clearer varnames --- imod/msw/copy_files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/imod/msw/copy_files.py b/imod/msw/copy_files.py index 21b89ae40..c5615b5fb 100644 --- a/imod/msw/copy_files.py +++ b/imod/msw/copy_files.py @@ -33,17 +33,17 @@ def from_imod5_data(cls, imod5_data: Imod5DataDict): "svat2precgrid.inp", "svat2etrefgrid.inp", ) - paths_filtered = [ + paths_included = [ str(p) for p in paths_unpacked if p.name.lower() not in files_to_filter ] - paths_filtered_away = {str(p) for p in paths_unpacked} - set(paths_filtered) - if paths_filtered_away: - log_message = _LOG_MESSAGE_TEMPLATE.format(filtered=paths_filtered_away) + paths_excluded = {str(p) for p in paths_unpacked} - set(paths_included) + if paths_excluded: + log_message = _LOG_MESSAGE_TEMPLATE.format(filtered=paths_excluded) logger.log( loglevel=LogLevel.INFO, message=log_message, ) - return cls(paths_filtered) + return cls(paths_included) def write(self, directory: str | Path, *_): directory = Path(directory)