diff --git a/CHANGELOG.md b/CHANGELOG.md index 1502a5d11..090438345 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Improved tracking of metadata changes with `resample_spatial` and `resample_cube_spatial` ([#690](https://github.com/Open-EO/openeo-python-client/issues/690)) + ### Removed ### Fixed diff --git a/openeo/_version.py b/openeo/_version.py index 61538ac93..5a81e4a70 100644 --- a/openeo/_version.py +++ b/openeo/_version.py @@ -1 +1 @@ -__version__ = "0.38.0a1" +__version__ = "0.38.0a2" diff --git a/openeo/metadata.py b/openeo/metadata.py index 0ded32e84..9a6f2d75a 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -12,6 +12,7 @@ from openeo.internal.jupyter import render_component from openeo.util import Rfc3339, deep_get +from openeo.utils.normalize import normalize_resample_resolution _log = logging.getLogger(__name__) @@ -25,6 +26,8 @@ class DimensionAlreadyExistsException(MetadataException): # TODO: make these dimension classes immutable data classes +# TODO: align better with STAC datacube extension +# TODO: align/adapt/integrate with pystac's datacube extension implementation? class Dimension: """Base class for dimensions.""" @@ -58,6 +61,8 @@ def rename_labels(self, target, source) -> Dimension: class SpatialDimension(Dimension): + # TODO: align better with STAC datacube extension: e.g. support "axis" (x or y) + DEFAULT_CRS = 4326 def __init__( @@ -257,6 +262,10 @@ def __init__(self, dimensions: Optional[List[Dimension]] = None): def __eq__(self, o: Any) -> bool: return isinstance(o, type(self)) and self._dimensions == o._dimensions + def __str__(self) -> str: + bands = self.band_names if self.has_band_dimension() else "no bands dimension" + return f"CubeMetadata({bands} - {self.dimension_names()})" + def _clone_and_update(self, dimensions: Optional[List[Dimension]] = None, **kwargs) -> CubeMetadata: """Create a new instance (of same class) with copied/updated fields.""" cls = type(self) @@ -411,10 +420,39 @@ def drop_dimension(self, name: str = None) -> CubeMetadata: raise ValueError("No dimension named {n!r} (valid names: {ns!r})".format(n=name, ns=dimension_names)) return self._clone_and_update(dimensions=[d for d in self._dimensions if not d.name == name]) - def __str__(self) -> str: - bands = self.band_names if self.has_band_dimension() else "no bands dimension" - return f"CubeMetadata({bands} - {self.dimension_names()})" + def resample_spatial( + self, + resolution: Union[float, Tuple[float, float], List[float]] = 0.0, + projection: Union[int, str, None] = None, + ) -> CubeMetadata: + resolution = normalize_resample_resolution(resolution) + if self._dimensions is None: + # Best-effort fallback to work with + dimensions = [ + SpatialDimension(name="x", extent=[None, None]), + SpatialDimension(name="y", extent=[None, None]), + ] + else: + # Make sure to work with a copy (to edit in-place) + dimensions = list(self._dimensions) + + # Find and replace spatial dimensions + spatial_indixes = [i for i, d in enumerate(dimensions) if isinstance(d, SpatialDimension)] + if len(spatial_indixes) != 2: + raise MetadataException(f"Expected two spatial resolutions but found {spatial_indixes=}") + for i in spatial_indixes: + dim: SpatialDimension = dimensions[i] + dimensions[i] = SpatialDimension( + name=dim.name, + extent=dim.extent, + crs=projection or dim.crs, + step=resolution[i] if resolution[i] else dim.step, + ) + + return self._clone_and_update(dimensions=dimensions) + def resample_cube_spatial(self, target: CubeMetadata) -> CubeMetadata: + return self._clone_and_update(dimensions=list(target._dimensions)) class CollectionMetadata(CubeMetadata): """ diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 1f5400f8c..a7975284a 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -40,6 +40,7 @@ Band, BandDimension, CollectionMetadata, + CubeMetadata, SpatialDimension, TemporalDimension, metadata_from_stac, @@ -69,6 +70,10 @@ from openeo.udf import XarrayDataCube +# Sentinel value for arguments that are unset (when `None` has a different meaning) +_UNSET = object() + + log = logging.getLogger(__name__) @@ -97,7 +102,7 @@ def process( self, process_id: str, arguments: Optional[dict] = None, - metadata: Optional[CollectionMetadata] = None, + metadata: Optional[CubeMetadata] = _UNSET, namespace: Optional[str] = None, **kwargs, ) -> DataCube: @@ -111,7 +116,11 @@ def process( :return: new DataCube instance """ pg = self._build_pgnode(process_id=process_id, arguments=arguments, namespace=namespace, **kwargs) - return DataCube(graph=pg, connection=self._connection, metadata=metadata or self.metadata) + return DataCube( + graph=pg, + connection=self._connection, + metadata=self.metadata if metadata is _UNSET else metadata, + ) graph_add_node = legacy_alias(process, "graph_add_node", since="0.1.1") @@ -749,16 +758,24 @@ def band(self, band: Union[str, int]) -> DataCube: @openeo_process def resample_spatial( - self, resolution: Union[float, Tuple[float, float]], projection: Union[int, str] = None, - method: str = 'near', align: str = 'upper-left' + self, + resolution: Union[float, Tuple[float, float], List[float]] = 0.0, + projection: Union[int, str, None] = None, + method: str = "near", + align: str = "upper-left", ) -> DataCube: - return self.process('resample_spatial', { - 'data': THIS, - 'resolution': resolution, - 'projection': projection, - 'method': method, - 'align': align - }) + metadata = (self.metadata or CubeMetadata()).resample_spatial(resolution=resolution, projection=projection) + return self.process( + process_id="resample_spatial", + arguments={ + "data": THIS, + "resolution": resolution, + "projection": projection, + "method": method, + "align": align, + }, + metadata=metadata, + ) def resample_cube_spatial(self, target: DataCube, method: str = "near") -> DataCube: """ @@ -773,7 +790,15 @@ def resample_cube_spatial(self, target: DataCube, method: str = "near") -> DataC :param method: Resampling method to use. :return: """ - return self.process("resample_cube_spatial", {"data": self, "target": target, "method": method}) + if target.metadata: + metadata = (self.metadata or CubeMetadata()).resample_cube_spatial(target=target.metadata) + else: + metadata = None + return self.process( + process_id="resample_cube_spatial", + arguments={"data": self, "target": target, "method": method}, + metadata=metadata, + ) @openeo_process def resample_cube_temporal( diff --git a/openeo/utils/__init__.py b/openeo/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openeo/utils/normalize.py b/openeo/utils/normalize.py new file mode 100644 index 000000000..518db7a58 --- /dev/null +++ b/openeo/utils/normalize.py @@ -0,0 +1,16 @@ +from typing import Tuple, Union + + +def normalize_resample_resolution( + resolution: Union[int, float, Tuple[float, float], Tuple[int, int]] +) -> Tuple[Union[int, float], Union[int, float]]: + """Normalize a resolution value, as used in the `resample_spatial` process to a two-element tuple.""" + if isinstance(resolution, (int, float)): + return (resolution, resolution) + elif ( + isinstance(resolution, (list, tuple)) + and len(resolution) == 2 + and all(isinstance(r, (int, float)) for r in resolution) + ): + return tuple(resolution) + raise ValueError(f"Invalid resolution {resolution!r}") diff --git a/tests/rest/datacube/test_datacube.py b/tests/rest/datacube/test_datacube.py index f353b4d42..6acb1c7b6 100644 --- a/tests/rest/datacube/test_datacube.py +++ b/tests/rest/datacube/test_datacube.py @@ -20,6 +20,7 @@ from openeo import collection_property from openeo.api.process import Parameter +from openeo.metadata import SpatialDimension from openeo.rest import BandMathException, OpenEoClientException from openeo.rest._testing import build_capabilities from openeo.rest.connection import Connection @@ -730,12 +731,144 @@ def test_apply_kernel(s2cube): def test_resample_spatial(s2cube): - im = s2cube.resample_spatial(resolution=[2.0, 3.0], projection=4578) - graph = _get_leaf_node(im) - assert graph["process_id"] == "resample_spatial" - assert "data" in graph["arguments"] - assert graph["arguments"]["resolution"] == [2.0, 3.0] - assert graph["arguments"]["projection"] == 4578 + cube = s2cube.resample_spatial(resolution=[2.0, 3.0], projection=4578) + assert get_download_graph(cube, drop_load_collection=True, drop_save_result=True) == { + "resamplespatial1": { + "process_id": "resample_spatial", + "arguments": { + "data": {"from_node": "loadcollection1"}, + "resolution": [2.0, 3.0], + "projection": 4578, + "method": "near", + "align": "upper-left", + }, + } + } + + assert cube.metadata.spatial_dimensions == [ + SpatialDimension(name="x", extent=None, crs=4578, step=2.0), + SpatialDimension(name="y", extent=None, crs=4578, step=3.0), + ] + + +def test_resample_spatial_no_metadata(s2cube_without_metadata): + cube = s2cube_without_metadata.resample_spatial(resolution=(3, 5), projection=4578) + assert get_download_graph(cube, drop_load_collection=True, drop_save_result=True) == { + "resamplespatial1": { + "process_id": "resample_spatial", + "arguments": { + "data": {"from_node": "loadcollection1"}, + "resolution": [3, 5], + "projection": 4578, + "method": "near", + "align": "upper-left", + }, + } + } + assert cube.metadata.spatial_dimensions == [ + SpatialDimension(name="x", extent=[None, None], crs=4578, step=3.0), + SpatialDimension(name="y", extent=[None, None], crs=4578, step=5.0), + ] + + +def test_resample_cube_spatial(s2cube): + cube1 = s2cube.resample_spatial(resolution=[2.0, 3.0], projection=4578) + cube2 = s2cube.resample_spatial(resolution=10, projection=32631) + + cube12 = cube1.resample_cube_spatial(target=cube2) + assert get_download_graph(cube12, drop_load_collection=True, drop_save_result=True) == { + "resamplespatial1": { + "process_id": "resample_spatial", + "arguments": { + "align": "upper-left", + "data": {"from_node": "loadcollection1"}, + "method": "near", + "projection": 4578, + "resolution": [2.0, 3.0], + }, + }, + "resamplespatial2": { + "process_id": "resample_spatial", + "arguments": { + "align": "upper-left", + "data": {"from_node": "loadcollection1"}, + "method": "near", + "projection": 32631, + "resolution": 10, + }, + }, + "resamplecubespatial1": { + "arguments": { + "data": {"from_node": "resamplespatial1"}, + "method": "near", + "target": {"from_node": "resamplespatial2"}, + }, + "process_id": "resample_cube_spatial", + }, + } + assert cube12.metadata.spatial_dimensions == [ + SpatialDimension(name="x", extent=None, crs=32631, step=10), + SpatialDimension(name="y", extent=None, crs=32631, step=10), + ] + + cube21 = cube2.resample_cube_spatial(target=cube1) + assert get_download_graph(cube21, drop_load_collection=True, drop_save_result=True) == { + "resamplespatial1": { + "process_id": "resample_spatial", + "arguments": { + "align": "upper-left", + "data": {"from_node": "loadcollection1"}, + "method": "near", + "projection": 32631, + "resolution": 10, + }, + }, + "resamplespatial2": { + "process_id": "resample_spatial", + "arguments": { + "align": "upper-left", + "data": {"from_node": "loadcollection1"}, + "method": "near", + "projection": 4578, + "resolution": [2.0, 3.0], + }, + }, + "resamplecubespatial1": { + "arguments": { + "data": {"from_node": "resamplespatial1"}, + "method": "near", + "target": {"from_node": "resamplespatial2"}, + }, + "process_id": "resample_cube_spatial", + }, + } + assert cube21.metadata.spatial_dimensions == [ + SpatialDimension(name="x", extent=None, crs=4578, step=2.0), + SpatialDimension(name="y", extent=None, crs=4578, step=3.0), + ] + + +def test_resample_cube_spatial_no_source_metadata(s2cube, s2cube_without_metadata): + cube = s2cube_without_metadata + target = s2cube.resample_spatial(resolution=10, projection=32631) + assert cube.metadata is None + assert target.metadata is not None + + result = cube.resample_cube_spatial(target=target) + assert result.metadata.spatial_dimensions == [ + SpatialDimension(name="x", extent=None, crs=32631, step=10), + SpatialDimension(name="y", extent=None, crs=32631, step=10), + ] + + +def test_resample_cube_spatial_no_target_metadata(s2cube, s2cube_without_metadata): + cube = s2cube.resample_spatial(resolution=10, projection=32631) + target = s2cube_without_metadata + assert cube.metadata is not None + assert target.metadata is None + + result = cube.resample_cube_spatial(target=target) + assert result.metadata is None def test_merge(s2cube, api_version, test_data): diff --git a/tests/test_metadata.py b/tests/test_metadata.py index da9280ed6..5d531ae79 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -22,6 +22,19 @@ from openeo.testing.stac import StacDummyBuilder +@pytest.fixture +def xytb_cube_metadata() -> CubeMetadata: + """Generic 4D cube (x, y, temporal, band) metadata""" + return CubeMetadata( + dimensions=[ + SpatialDimension(name="x", extent=[2, 7], crs=4326, step=0.1), + SpatialDimension(name="y", extent=[49, 52], crs=4326, step=0.1), + TemporalDimension(name="t", extent=["2024-09-01", "2024-12-01"]), + BandDimension(name="bands", bands=[Band("B2"), Band("B3")]), + ] + ) + + def test_metadata_get(): metadata = CollectionMetadata({"foo": "bar", "very": {"deeply": {"nested": {"path": {"to": "somewhere"}}}}}) assert metadata.get("foo") == "bar" @@ -915,3 +928,51 @@ def test_metadata_from_stac_temporal_dimension(tmp_path, stac_dict, expected): assert (dim.name, dim.extent) == expected else: assert not metadata.has_temporal_dimension() + + +@pytest.mark.parametrize( + ["kwargs", "expected_x", "expected_y"], + [ + ({}, {"crs": 4326, "step": 0.1}, {"crs": 4326, "step": 0.1}), + ({"resolution": 2}, {"crs": 4326, "step": 2}, {"crs": 4326, "step": 2}), + ({"resolution": [0.5, 2]}, {"crs": 4326, "step": 0.5}, {"crs": 4326, "step": 2}), + ({"projection": 32631}, {"crs": 32631, "step": 0.1}, {"crs": 32631, "step": 0.1}), + ({"resolution": 10, "projection": 32631}, {"crs": 32631, "step": 10}, {"crs": 32631, "step": 10}), + ({"resolution": [11, 22], "projection": 32631}, {"crs": 32631, "step": 11}, {"crs": 32631, "step": 22}), + ], +) +def test_metadata_resample_spatial(xytb_cube_metadata, kwargs, expected_x, expected_y): + metadata = xytb_cube_metadata.resample_spatial(**kwargs) + assert isinstance(metadata, CubeMetadata) + assert metadata.spatial_dimensions == [ + SpatialDimension(name="x", extent=[2, 7], **expected_x), + SpatialDimension(name="y", extent=[49, 52], **expected_y), + ] + assert metadata.temporal_dimension == xytb_cube_metadata.temporal_dimension + assert metadata.band_dimension == xytb_cube_metadata.band_dimension + + +def test_metadata_resample_cube_spatial(xytb_cube_metadata): + metadata1 = xytb_cube_metadata.resample_spatial(resolution=(11, 22), projection=32631) + metadata2 = xytb_cube_metadata.resample_spatial(resolution=0.5) + + assert metadata1.spatial_dimensions == [ + SpatialDimension(name="x", extent=[2, 7], crs=32631, step=11), + SpatialDimension(name="y", extent=[49, 52], crs=32631, step=22), + ] + assert metadata2.spatial_dimensions == [ + SpatialDimension(name="x", extent=[2, 7], crs=4326, step=0.5), + SpatialDimension(name="y", extent=[49, 52], crs=4326, step=0.5), + ] + + metadata12 = metadata1.resample_cube_spatial(target=metadata2) + assert metadata12.spatial_dimensions == [ + SpatialDimension(name="x", extent=[2, 7], crs=4326, step=0.5), + SpatialDimension(name="y", extent=[49, 52], crs=4326, step=0.5), + ] + + metadata21 = metadata2.resample_cube_spatial(target=metadata1) + assert metadata21.spatial_dimensions == [ + SpatialDimension(name="x", extent=[2, 7], crs=32631, step=11), + SpatialDimension(name="y", extent=[49, 52], crs=32631, step=22), + ] diff --git a/tests/utills/test_nomalize.py b/tests/utills/test_nomalize.py new file mode 100644 index 000000000..bec5568a9 --- /dev/null +++ b/tests/utills/test_nomalize.py @@ -0,0 +1,29 @@ +import pytest + +from openeo.utils.normalize import normalize_resample_resolution + + +@pytest.mark.parametrize( + ["resolution", "expected"], + [ + (1, (1, 1)), + (1.23, (1.23, 1.23)), + ([1, 2], (1, 2)), + ((1.23, 2.34), (1.23, 2.34)), + ], +) +def test_normalize_resample_resolution(resolution, expected): + assert normalize_resample_resolution(resolution) == expected + + +@pytest.mark.parametrize( + "resolution", + [ + "0123", + [1, 2, 3], + {"x": 2, "y": 5}, + ], +) +def test_normalize_resample_resolution(resolution): + with pytest.raises(ValueError, match="Invalid resolution"): + normalize_resample_resolution(resolution)