diff --git a/changelog/770.feature.md b/changelog/770.feature.md new file mode 100644 index 000000000..4ca92aab1 --- /dev/null +++ b/changelog/770.feature.md @@ -0,0 +1,7 @@ +The CI coupling gate now couples each regression baseline to its diagnostic's `Diagnostic.version`. +A test case fails the gate when the diagnostic's in-code version no longer matches the version recorded +when its baseline was minted, so an execution-affecting change can no longer slip through as a green replay +against a stale committed bundle. +The new author-declared `diagnostic_version` field lives in the test case `manifest.json` (schema version 2), +and the new `ref test-cases migrate-manifests` command backfills it across existing baselines. +A legitimate revert is still permitted: lower the diagnostic version and re-mint the baseline. diff --git a/docs/background/regression-baselines.md b/docs/background/regression-baselines.md index 8adff92fd..a5067a32b 100644 --- a/docs/background/regression-baselines.md +++ b/docs/background/regression-baselines.md @@ -27,14 +27,22 @@ A baseline is split into two layers with very different trust and portability pr The two layers are bound by a **`manifest.json`** alongside the bundle, which records: -- `schema` — integer schema version for the manifest format itself (currently `1`). +- `schema` — integer schema version for the manifest format itself (currently `2`). The loader rejects manifests whose `schema` does not match the current `SCHEMA_VERSION`, so format migrations are detected immediately rather than silently misread. - `test_case_version` — a monotonic, author-bumped integer that *authorises* a new baseline. +- `diagnostic_version` — the author-declared `Diagnostic.version` captured at mint time, + coupling the baseline to the diagnostic code that produced it. - `committed` — sha256 digests of the committed JSON artefacts, over the exact placeholder-substituted bytes on disk. - `native` — sha256 + size of each curated native file (empty `{}` until minted). - `catalog_hash` — the hash of the test case's input `catalog.yaml`, coupling the baseline to the inputs that produced it. +These give the manifest **two orthogonal version axes**: +`test_case_version` is per-case and mint-authored (it authorises a new committed baseline), +while `diagnostic_version` is per-diagnostic and author-authored (it declares that the +diagnostic's *results* changed). `test_case_version` remains monotonic, and `diagnostic_version` +may move backwards only for an authorised revert. The gate consults each for a different purpose. + !!! note "An empty native set is a permanent, valid state" Fork contributors cannot mint (minting needs object-store write credentials that never run on untrusted pull-request code). @@ -67,6 +75,7 @@ flowchart LR | `sync` | public read | Fetch the native blobs referenced by the manifest into the local store cache. | | `replay` | public read | Materialise the native baseline into a slot, re-run only `build_execution_result`, and tolerantly compare to the committed bundle. | | `check-store` | write | Preflight the writable native store (credentials + bucket) without uploading anything. Run this before a slow `mint` to confirm credentials are correct. | +| `migrate-manifests` | none | One-shot, idempotent maintenance command that rewrites every committed `manifest.json` at the current `SCHEMA_VERSION`, stamping each case's `diagnostic_version` from the diagnostic's in-code `Diagnostic.version`. | ### Output slots @@ -172,7 +181,7 @@ because an empty native set is legitimate, a missing native baseline downgrades - **`skip`** — nothing relevant to this case changed, or the case is not under regression management. - **`replay`** — cheap, anonymous re-check against the cached native baseline (only when native blobs exist). This also verifies a `test_case_version` bump that ships native blobs. - **`execute`** — full end-to-end re-run with tolerant compare; required when `test_case_version` was bumped to authorise a new baseline *and* no native baseline exists to replay against. -- **`fail`** — an unauthorised or unverifiable change (committed bundle edited without a version bump, version moved backwards, bundle drifted from its manifest, or catalog changed without regenerating the baseline). +- **`fail`** — an unauthorised or unverifiable change: the committed bundle or catalog changed without a version bump, a version moved backwards, the bundle drifted from its manifest, or the in-code `Diagnostic.version` no longer matches the recorded `diagnostic_version`. ### Decision flow @@ -190,17 +199,24 @@ flowchart TD committedOk -->|yes| catalogOk{catalog.yaml
matches manifest
catalog_hash?} catalogOk -->|no| failCatalog[["FAIL
inputs changed without
regenerating baseline"]] - catalogOk -->|yes| seeding{newly added
this branch?} + catalogOk -->|yes| staleVersion{code Diagnostic.version
vs baseline
diagnostic_version} + staleVersion -->|"code > baseline"| failStale[["FAIL
stale baseline
re-mint required"]] + staleVersion -->|"code < baseline"| failDowngrade[["FAIL
illegal downgrade
lower code AND re-mint"]] + + staleVersion -->|equal| seeding{newly added
this branch?} seeding -->|yes| seedNative{native
non-empty?} seedNative -->|yes| replaySeed[["REPLAY
seed against committed"]] seedNative -->|no| skipSeed[["SKIP
committed-only baseline"]] - seeding -->|no| versionCmp{version vs base} + seeding -->|no| versionCmp{test_case_version
vs base} versionCmp -->|decreased| failVersion[["FAIL
version not monotonic"]] - versionCmp -->|bumped| bumpNative{native
present?} + versionCmp -->|"unchanged or bumped"| diagVersionCmp{diagnostic_version
decreased vs base?} + diagVersionCmp -->|"yes, committed unchanged"| failBareDowngrade[["FAIL
bare downgrade
re-mint required"]] + diagVersionCmp -->|"no, or committed re-minted"| versionBumped{test_case_version
bumped?} + versionBumped -->|yes| bumpNative{native
present?} bumpNative -->|yes| replayBump[["REPLAY
verify new baseline
reproduces from native"]] bumpNative -->|no| execute[["EXECUTE
full re-run
(no native to replay)"]] - versionCmp -->|unchanged| committedChanged{committed
changed?} + versionBumped -->|no| committedChanged{committed
changed?} committedChanged -->|yes| failUnauthorised[["FAIL
baseline changed without
version bump"]] committedChanged -->|no| nativeChanged{native
changed?} @@ -224,6 +240,16 @@ and maps the changed-file list onto each case: - **`extraction_changed`** is coarse and errs toward `replay` (cheap, credential-free): any change under the diagnostic's provider package, or under the core regression package, counts for every case in that provider. + It is a *path-based* heuristic, so it cannot tell an execution change from an extraction change. +- **`diagnostic_version`** is the *author-declared* execution-change signal: when a diagnostic's + results change the author bumps `Diagnostic.version`, and the gate fails any baseline whose + recorded `diagnostic_version` no longer matches the in-code value until it is re-minted. + This closes the **replay blind-spot** that `extraction_changed` alone leaves open: + an execution-affecting change with no extraction-path edit (the #671 ECS case) would otherwise + route to a green replay against a stale bundle. + A `diagnostic_version` decrease is permitted only as an **authorised revert**: the author lowers + the code version *and* re-mints, so the committed bundle changes and the gate recognises the + revert structurally rather than via a flag. - **`committed_integrity_ok`** recomputes the committed digests from the working tree and compares them to `manifest.committed`. - **`catalog_integrity_ok`** recomputes the input catalog hash diff --git a/packages/climate-ref-core/src/climate_ref_core/regression/gate.py b/packages/climate-ref-core/src/climate_ref_core/regression/gate.py index 5c59dbc88..25741526f 100644 --- a/packages/climate-ref-core/src/climate_ref_core/regression/gate.py +++ b/packages/climate-ref-core/src/climate_ref_core/regression/gate.py @@ -1,22 +1,29 @@ """ CI coupling gate for test case regression bundles. -For each test case, CI must decide *how* to verify the regression baseline in a pull request, +For each test case, CI decides how to verify the regression baseline in a pull request, based on what changed relative to the base branch: -- **EXECUTE** — re-run the diagnostic end-to-end and compare the regenerated committed - bundle to the in-repo copy (tolerant compare). - Required when the author has bumped ``test_case_version`` to authorise a new baseline - but no native baseline exists to replay against. -- **REPLAY** — materialise the cached native baseline and re-run only - ``build_execution_result`` against it, comparing to the committed bundle. - Cheap and anonymous; used when extraction code changed but the committed bundle did - not, to seed a newly added manifest, and to verify a ``test_case_version`` bump that ships native blobs - — but only when native blobs exist to replay. -- **FAIL** — the committed bundle (or its input catalog) changed without a - ``test_case_version`` bump (an unauthorised baseline change), or the version moved backwards. -- **SKIP** — nothing relevant to this test case changed, or the case is not yet under - regression-baseline management. +- **EXECUTE**: re-run the diagnostic end-to-end and compare the regenerated bundle + to the committed copy. + Used when ``test_case_version`` was bumped but no native baseline exists to replay. +- **REPLAY**: re-run ``build_execution_result`` against the cached native baseline + and compare to the committed bundle. + Used when extraction code changed but the committed bundle did not, + to seed a newly added manifest, + and to verify a ``test_case_version`` bump that ships native blobs. +- **FAIL**: the committed bundle or its catalog changed without a ``test_case_version`` bump, + a version moved backwards, + or the in-code ``Diagnostic.version`` no longer matches the recorded ``diagnostic_version``. +- **SKIP**: nothing relevant changed, or the case is not under regression-baseline management. + +The gate tracks two orthogonal version axes: +``test_case_version`` (per-case, mint-authored) +and ``diagnostic_version`` (per-diagnostic, author-authored). +``test_case_version`` is monotonic. +``diagnostic_version`` may decrease only for an authorised revert: +the code version is lowered and the baseline re-minted so the committed bundle changes, +which the gate recognises structurally rather than via a flag. See [Regression Baselines](https://climate-ref.readthedocs.io/en/latest/background/regression-baselines/) for more information about the motivation and workflow for regression baselines. @@ -59,13 +66,14 @@ class GateDecision: """A human-readable explanation, surfaced in CI logs.""" -def decide_coupling( # noqa: PLR0911, PLR0912 +def decide_coupling( # noqa: PLR0911, PLR0912, PLR0913 manifest: Manifest | None, base_manifest: Manifest | None, *, extraction_changed: bool = False, committed_integrity_ok: bool = True, catalog_integrity_ok: bool = True, + code_diagnostic_version: int | None = None, ) -> GateDecision: """ Decide how CI should verify a single test case's regression baseline. @@ -118,6 +126,12 @@ def decide_coupling( # noqa: PLR0911, PLR0912 so the committed bundle no longer reflects its inputs — a hard failure. Always ``True`` when the manifest carries no ``catalog_hash`` + code_diagnostic_version + The diagnostic's current in-code ``Diagnostic.version``, injected by the caller + (the pure function does no I/O). When provided, a code version that exceeds + ``manifest.diagnostic_version`` fails the case as a stale baseline that must be + re-minted, and a code version below it fails as an illegal downgrade with no re-mint. + ``None`` (the default) skips the staleness comparison entirely, preserving legacy behaviour. Returns ------- @@ -156,6 +170,28 @@ def decide_coupling( # noqa: PLR0911, PLR0912 "regenerate the baseline with `ref test-cases run` after changing the inputs", ) + # Runs before seeding so a newly added manifest whose author bumped the code + # without re-minting still fails. + # Equality must fall through; only the > and < arms return. + if manifest is not None and code_diagnostic_version is not None: + if code_diagnostic_version > manifest.diagnostic_version: + return GateDecision( + Action.FAIL, + f"diagnostic Diagnostic.version is {code_diagnostic_version} but the baseline " + f"manifest records diagnostic_version {manifest.diagnostic_version}; the " + "diagnostic's results changed; re-mint this baseline (`ref test-cases mint`) " + "so the committed bundle reflects the new version", + ) + if code_diagnostic_version < manifest.diagnostic_version: + return GateDecision( + Action.FAIL, + f"diagnostic Diagnostic.version ({code_diagnostic_version}) is below the " + f"baseline's diagnostic_version ({manifest.diagnostic_version}); " + "Diagnostic.version is append-only. To revert legitimately, lower the code " + "version AND re-mint (`ref test-cases mint`) so the bundle is re-authored at " + "the lower version.", + ) + if base_manifest is None: # Seeding a newly added manifest. # Replay only verifies something when native blobs exist @@ -177,6 +213,20 @@ def decide_coupling( # noqa: PLR0911, PLR0912 f"{manifest.test_case_version}); version must be monotonic", ) + # A diagnostic_version decrease only fails when the committed bundle is unchanged: + # a bare downgrade with no re-mint. + # A decrease that ships a re-minted bundle is an authorised revert and falls through, + # where the committed-changed logic below re-verifies it. + if manifest.diagnostic_version < base_manifest.diagnostic_version: + if manifest.committed == base_manifest.committed: + return GateDecision( + Action.FAIL, + f"diagnostic_version decreased ({base_manifest.diagnostic_version} -> " + f"{manifest.diagnostic_version}) with the committed bundle unchanged; " + "a bare downgrade is not permitted; re-mint to re-author the baseline at " + "the lower version, or restore the version", + ) + version_bumped = manifest.test_case_version > base_manifest.test_case_version committed_changed = manifest.committed != base_manifest.committed native_changed = manifest.native != base_manifest.native diff --git a/packages/climate-ref-core/src/climate_ref_core/regression/manifest.py b/packages/climate-ref-core/src/climate_ref_core/regression/manifest.py index 13a95cef2..56ae53ecc 100644 --- a/packages/climate-ref-core/src/climate_ref_core/regression/manifest.py +++ b/packages/climate-ref-core/src/climate_ref_core/regression/manifest.py @@ -6,6 +6,8 @@ - ``schema``: the manifest schema version (``SCHEMA_VERSION``). - ``test_case_version``: a monotonic, author-bumped integer coupling the committed bundle to its native outputs. +- ``diagnostic_version``: the author-declared ``Diagnostic.version`` captured at mint + time, coupling the baseline to the diagnostic code that produced it. - ``committed``: sha256 digests of the committed regression JSON artefacts, computed over the exact placeholder-substituted bytes as they sit on disk, so that a CI recompute is deterministic. @@ -26,7 +28,7 @@ from climate_ref_core.paths import safe_path -SCHEMA_VERSION: int = 1 +SCHEMA_VERSION: int = 2 """Current manifest schema version.""" _DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") @@ -133,6 +135,14 @@ class Manifest: test_case_version: int """Monotonic, author-bumped version coupling the bundle to its native outputs.""" + diagnostic_version: int + """Author-declared ``Diagnostic.version`` at mint time. + + Monotonic and append-only (with an authorised-revert carve-out in the CI gate). + The gate fails a case whose in-code ``Diagnostic.version`` exceeds this value + (a stale baseline that must be re-minted). + """ + committed: dict[str, str] """Digests of committed regression JSON artefacts: ``{relpath: sha256}``.""" @@ -195,7 +205,11 @@ def loads(cls, text: str, *, source: str = "") -> Manifest: (e.g. hand-edited or written by an incompatible version). """ data = json.loads(text) - missing = [key for key in ("schema", "test_case_version", "committed", "native") if key not in data] + missing = [ + key + for key in ("schema", "test_case_version", "diagnostic_version", "committed", "native") + if key not in data + ] if missing: raise ValueError( f"Invalid manifest {source}: missing required keys {missing}. " @@ -209,6 +223,16 @@ def loads(cls, text: str, *, source: str = "") -> Manifest: f"expected {SCHEMA_VERSION}. The manifest was written by an incompatible " "version; regenerate it with `ref test-cases run --force-regen`." ) + diagnostic_version = data["diagnostic_version"] + if ( + isinstance(diagnostic_version, bool) + or not isinstance(diagnostic_version, int) + or diagnostic_version < 1 + ): + raise ValueError( + f"Invalid manifest {source}: diagnostic_version {diagnostic_version!r} " + "is invalid; expected an integer >= 1." + ) try: native = { relpath: NativeEntry(sha256=entry["sha256"], size=entry["size"]) @@ -235,6 +259,7 @@ def loads(cls, text: str, *, source: str = "") -> Manifest: return cls( schema=data["schema"], test_case_version=data["test_case_version"], + diagnostic_version=diagnostic_version, committed=dict(data["committed"]), native=native, # TODO: remove optonality when all manifests have this field. @@ -256,6 +281,7 @@ def dump(self, path: Path) -> None: payload = { "schema": self.schema, "test_case_version": self.test_case_version, + "diagnostic_version": self.diagnostic_version, "catalog_hash": self.catalog_hash, "committed": self.committed, "native": {relpath: asdict(entry) for relpath, entry in self.native.items()}, @@ -264,7 +290,12 @@ def dump(self, path: Path) -> None: path.write_text(text, encoding="utf-8") @classmethod - def seed_v1(cls, committed_digests: dict[str, str], catalog_hash: str | None = None) -> Manifest: + def seed_v1( + cls, + committed_digests: dict[str, str], + catalog_hash: str | None = None, + diagnostic_version: int = 1, + ) -> Manifest: """ Create an initial manifest at ``test_case_version == 1`` with no native outputs. @@ -274,6 +305,9 @@ def seed_v1(cls, committed_digests: dict[str, str], catalog_hash: str | None = N Digests of the committed regression JSON artefacts. catalog_hash Hash of the input ``catalog.yaml`` that produced the baseline, if known. + diagnostic_version + Author-declared ``Diagnostic.version`` at seed time. Defaults to ``1``, + the version of a brand-new baseline. Returns ------- @@ -283,6 +317,7 @@ def seed_v1(cls, committed_digests: dict[str, str], catalog_hash: str | None = N return cls( schema=SCHEMA_VERSION, test_case_version=1, + diagnostic_version=diagnostic_version, committed=dict(committed_digests), catalog_hash=catalog_hash, native={}, diff --git a/packages/climate-ref-core/tests/unit/regression/test_gate.py b/packages/climate-ref-core/tests/unit/regression/test_gate.py index c0fbea039..2a03eb26f 100644 --- a/packages/climate-ref-core/tests/unit/regression/test_gate.py +++ b/packages/climate-ref-core/tests/unit/regression/test_gate.py @@ -21,11 +21,13 @@ def make_manifest( version: int, committed: dict[str, str] | None = None, native: dict[str, NativeEntry] | None = None, + diagnostic_version: int = 1, ) -> Manifest: """Build a manifest with a given version and committed/native digests.""" return Manifest( schema=SCHEMA_VERSION, test_case_version=version, + diagnostic_version=diagnostic_version, committed=committed if committed is not None else {"output.json": "a" * 64}, native=native if native is not None else {}, ) @@ -158,6 +160,84 @@ def test_no_change_skips(self) -> None: assert "nothing to verify" in decision.reason +class TestDiagnosticVersionGate: + """The staleness (2a) and conditioned-monotonic (2b) diagnostic_version branches.""" + + def test_code_ahead_of_manifest_fails(self) -> None: + # The diagnostic was bumped but the baseline was not re-minted: stale -> FAIL. + manifest = make_manifest(1, diagnostic_version=1) + decision = decide_coupling(manifest, None, code_diagnostic_version=2) + assert decision.action is Action.FAIL + assert "re-mint" in decision.reason + + def test_code_below_manifest_fails(self) -> None: + # The code version dropped below the recorded baseline with no re-mint: FAIL. + manifest = make_manifest(1, diagnostic_version=2) + decision = decide_coupling(manifest, None, code_diagnostic_version=1) + assert decision.action is Action.FAIL + assert "append-only" in decision.reason + + def test_code_equals_manifest_falls_through(self) -> None: + # Equality must NOT early-return: a matched baseline proceeds to the normal logic. + manifest = make_manifest(1, native={"data.nc": NativeEntry("a" * 64, 10)}, diagnostic_version=2) + decision = decide_coupling(manifest, None, code_diagnostic_version=2) + assert decision.action is Action.REPLAY + assert "seeding" in decision.reason + + def test_staleness_outranks_seeding(self) -> None: + # A newly added manifest (no base) whose code is ahead is stale -> FAIL, not REPLAY/SKIP. + manifest = make_manifest(1, native={"data.nc": NativeEntry("a" * 64, 10)}, diagnostic_version=1) + decision = decide_coupling(manifest, None, code_diagnostic_version=2) + assert decision.action is Action.FAIL + + def test_staleness_below_integrity(self) -> None: + # An integrity failure outranks the staleness check. + manifest = make_manifest(1, diagnostic_version=1) + decision = decide_coupling(manifest, None, committed_integrity_ok=False, code_diagnostic_version=2) + assert decision.action is Action.FAIL + assert "does not match manifest.json digests" in decision.reason + + def test_diagnostic_version_decrease_bare_downgrade_fails(self) -> None: + # A diagnostic_version decrease with the committed bundle unchanged is a bare downgrade. + base = make_manifest(1, {"output.json": "a" * 64}, diagnostic_version=2) + current = make_manifest(1, {"output.json": "a" * 64}, diagnostic_version=1) + decision = decide_coupling(current, base, code_diagnostic_version=1) + assert decision.action is Action.FAIL + assert "bare downgrade" in decision.reason + + def test_diagnostic_version_decrease_with_reminted_bundle_falls_through(self) -> None: + # A decrease that ships a re-minted bundle (committed changed) + a tcv bump is authorised. + native = {"data.nc": NativeEntry("a" * 64, 10)} + base = make_manifest(1, {"output.json": "a" * 64}, native, diagnostic_version=2) + current = make_manifest(2, {"output.json": "b" * 64}, native, diagnostic_version=1) + decision = decide_coupling(current, base, code_diagnostic_version=1) + # 2b falls through; the tcv bump with native present routes to REPLAY. + assert decision.action is Action.REPLAY + assert "1 -> 2" in decision.reason + + def test_diagnostic_version_increase_proceeds(self) -> None: + # An increase (with matching code) is not a 2b failure; normal logic applies. + base = make_manifest(1, {"output.json": "a" * 64}, diagnostic_version=1) + current = make_manifest(1, {"output.json": "a" * 64}, diagnostic_version=2) + decision = decide_coupling(current, base, code_diagnostic_version=2) + assert decision.action is Action.SKIP + assert "nothing to verify" in decision.reason + + def test_legacy_none_skips_all_new_branches(self) -> None: + # code_diagnostic_version=None preserves legacy behaviour: no staleness check at all. + manifest = make_manifest(1, diagnostic_version=1) + decision = decide_coupling(manifest, None, code_diagnostic_version=None) + assert decision.action is Action.SKIP + + def test_tcv_bump_with_matched_diagnostic_version_still_executes(self) -> None: + # A test_case_version bump with code == manifest.diagnostic_version behaves as before. + base = make_manifest(1, diagnostic_version=2) + current = make_manifest(2, diagnostic_version=2) + decision = decide_coupling(current, base, code_diagnostic_version=2) + assert decision.action is Action.EXECUTE + assert "1 -> 2" in decision.reason + + class TestPathsUnder: def test_file_directly_under_root_matches(self) -> None: assert paths_under(["pkg/src/mod.py"], ["pkg/src"]) diff --git a/packages/climate-ref-core/tests/unit/regression/test_manifest.py b/packages/climate-ref-core/tests/unit/regression/test_manifest.py index 90700bcb6..33f3fe1df 100644 --- a/packages/climate-ref-core/tests/unit/regression/test_manifest.py +++ b/packages/climate-ref-core/tests/unit/regression/test_manifest.py @@ -83,6 +83,14 @@ def test_committed_digests_stored(self) -> None: m = Manifest.seed_v1(digests) assert m.committed == digests + def test_diagnostic_version_defaults_to_1(self) -> None: + m = Manifest.seed_v1({}) + assert m.diagnostic_version == 1 + + def test_diagnostic_version_override(self) -> None: + m = Manifest.seed_v1({}, diagnostic_version=3) + assert m.diagnostic_version == 3 + class TestDumpLoad: """dump then load must be byte-identical and stable.""" @@ -91,6 +99,7 @@ def _make_manifest(self) -> Manifest: return Manifest( schema=SCHEMA_VERSION, test_case_version=2, + diagnostic_version=3, committed={"output.json": "cc" * 32, "series.json": "dd" * 32}, native={"foo/bar.nc": NativeEntry(sha256="ee" * 32, size=1024)}, ) @@ -102,6 +111,7 @@ def test_round_trip_values(self, tmp_path: Path) -> None: loaded = Manifest.load(p) assert loaded.schema == original.schema assert loaded.test_case_version == original.test_case_version + assert loaded.diagnostic_version == original.diagnostic_version assert loaded.committed == original.committed assert loaded.native == original.native @@ -133,6 +143,7 @@ def test_native_entry_round_trip(self, tmp_path: Path) -> None: m = Manifest( schema=SCHEMA_VERSION, test_case_version=1, + diagnostic_version=1, committed={}, native={"path/to/file.nc": entry}, ) @@ -141,17 +152,67 @@ def test_native_entry_round_trip(self, tmp_path: Path) -> None: loaded = Manifest.load(p) assert loaded.native["path/to/file.nc"] == entry + def test_dump_emits_unconditional_null_catalog_hash(self, tmp_path: Path) -> None: + # dump always serialises catalog_hash (as null when unset) for byte-stability. + p = tmp_path / "manifest.json" + self._make_manifest().dump(p) + data = json.loads(p.read_text(encoding="utf-8")) + assert "catalog_hash" in data + assert data["catalog_hash"] is None + assert data["diagnostic_version"] == 3 + def test_load_missing_keys_raises_value_error(self, tmp_path: Path) -> None: p = tmp_path / "manifest.json" p.write_text(json.dumps({"schema": SCHEMA_VERSION, "committed": {}}), encoding="utf-8") - with pytest.raises(ValueError, match=r"missing required keys \['test_case_version', 'native'\]"): + with pytest.raises( + ValueError, + match=r"missing required keys \['test_case_version', 'diagnostic_version', 'native'\]", + ): + Manifest.load(p) + + def test_load_missing_diagnostic_version_raises_value_error(self, tmp_path: Path) -> None: + # A schema-2 payload missing diagnostic_version is rejected post-bump. + p = tmp_path / "manifest.json" + payload = { + "schema": SCHEMA_VERSION, + "test_case_version": 1, + "committed": {}, + "native": {}, + } + p.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="missing required keys"): Manifest.load(p) + @pytest.mark.parametrize("bad_value", [0, -1, "1", True, 1.0]) + def test_load_rejects_invalid_diagnostic_version(self, bad_value: object) -> None: + payload = { + "schema": SCHEMA_VERSION, + "test_case_version": 1, + "diagnostic_version": bad_value, + "committed": {}, + "native": {}, + } + with pytest.raises(ValueError, match="diagnostic_version"): + Manifest.loads(json.dumps(payload)) + + def test_load_rejects_schema_1_text(self) -> None: + # A schema-1 manifest carrying diagnostic_version is still rejected by the schema check. + payload = { + "schema": 1, + "test_case_version": 1, + "diagnostic_version": 1, + "committed": {}, + "native": {}, + } + with pytest.raises(ValueError, match="unsupported schema"): + Manifest.loads(json.dumps(payload)) + def test_load_malformed_native_entry_raises_value_error(self, tmp_path: Path) -> None: p = tmp_path / "manifest.json" payload = { "schema": SCHEMA_VERSION, "test_case_version": 1, + "diagnostic_version": 1, "committed": {}, "native": {"file.nc": {"sha256": "aa" * 32}}, # missing "size" } @@ -168,6 +229,7 @@ def test_load_rejects_unsafe_native_path(self, tmp_path: Path, bad_relpath: str) payload = { "schema": SCHEMA_VERSION, "test_case_version": 1, + "diagnostic_version": 1, "committed": {}, "native": {bad_relpath: {"sha256": "aa" * 32, "size": 1}}, } @@ -184,6 +246,7 @@ def test_load_rejects_bad_native_digest(self, tmp_path: Path, bad_digest: str) - payload = { "schema": SCHEMA_VERSION, "test_case_version": 1, + "diagnostic_version": 1, "committed": {}, "native": {"file.nc": {"sha256": bad_digest, "size": 1}}, } @@ -191,11 +254,12 @@ def test_load_rejects_bad_native_digest(self, tmp_path: Path, bad_digest: str) - with pytest.raises(ValueError, match="Invalid sha256 digest"): Manifest.load(p) - @pytest.mark.parametrize("bad_schema", [2, 0, "1", True, 1.0]) + @pytest.mark.parametrize("bad_schema", [3, 0, "1", True, 1.0]) def test_load_rejects_unknown_schema(self, bad_schema: object) -> None: payload = { "schema": bad_schema, "test_case_version": 1, + "diagnostic_version": 1, "committed": {}, "native": {}, } @@ -207,6 +271,7 @@ def test_load_rejects_invalid_native_size(self, bad_size: object) -> None: payload = { "schema": SCHEMA_VERSION, "test_case_version": 1, + "diagnostic_version": 1, "committed": {}, "native": {"file.nc": {"sha256": "aa" * 32, "size": bad_size}}, } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/climate-at-global-warming-levels/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/climate-at-global-warming-levels/cmip6/manifest.json index 5391330fa..6cd9776ec 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/climate-at-global-warming-levels/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/climate-at-global-warming-levels/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "e0dd8f0902fcc0d6b032e6ef1e57b785c776aa18847ef60805d4b388166a2daa", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "20a2c7e604440a3e14b33de2306b0deee39187a7bebf626af2dab1cebdc6ab92", @@ -103,6 +104,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/climate-drivers-for-fire/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/climate-drivers-for-fire/cmip6/manifest.json index 38732230c..74f79e554 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/climate-drivers-for-fire/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/climate-drivers-for-fire/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "da1aeaaa1b0a99771b91fdddaefcd3769a8ef6d7183396cfd0d0f97eb7e359dd", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/climate-drivers-for-fire/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/climate-drivers-for-fire/cmip7/manifest.json index 31bf2d55a..82323eb17 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/climate-drivers-for-fire/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/climate-drivers-for-fire/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "c7a2dd809d8e30636ec7b2410764b4f44a799c911bb10b4ee323a57f8c94b751", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/cloud-radiative-effects/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/cloud-radiative-effects/cmip6/manifest.json index 617f68e09..e08cba786 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/cloud-radiative-effects/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/cloud-radiative-effects/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "4b9ebaf83c65b8e0718a84951502617a545fce6386cabe640de7ec406afe6509", "series.json": "4ec8ab11924089ef7cba8bd16de237ad8431399caaef74c0a66e81fefee69ef3" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -79,6 +80,6 @@ "size": 60160 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-cli-ta/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-cli-ta/cmip6/manifest.json index e813145c7..7b06d2b2d 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-cli-ta/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-cli-ta/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "454335b7796afd5d5c214c24501fad57b9dbaf21c997177209da025d24b3b6aa", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clivi-lwcre/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clivi-lwcre/cmip6/manifest.json index ef89dead6..ef499c2ed 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clivi-lwcre/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clivi-lwcre/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "312cf9003669edd2a808aa35983c77e987dba3a4f0ceb234b63345539e6a36a3", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clivi-lwcre/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clivi-lwcre/cmip7/manifest.json index 906c842a3..4bc7cde6f 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clivi-lwcre/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clivi-lwcre/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "add694adb3b029792e8706dc1ec646e90978d30b0e5ff3be7ce86d67715a3368", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clt-swcre/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clt-swcre/cmip6/manifest.json index 3e841cf4b..e4c700d20 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clt-swcre/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clt-swcre/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "cd6d53a6375230ea611d2938b3f8af535b24f3045ce8f520dd0edde98c54ec96", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clt-swcre/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clt-swcre/cmip7/manifest.json index 35e09cf11..b872e914e 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clt-swcre/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clt-swcre/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "958937d8c54124c46bdada306cccec6401a7d6a8397b2b003151e278ee38f169", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clwvi-pr/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clwvi-pr/cmip6/manifest.json index 089975392..e83045134 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clwvi-pr/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clwvi-pr/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "c95e6545f3873b3627b496aa9245e12c7b7e3834f073d89b6dfc9e0d553691af", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clwvi-pr/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clwvi-pr/cmip7/manifest.json index 501b27344..6b6e5b7a2 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clwvi-pr/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/cloud-scatterplots-clwvi-pr/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "7342b55c0338ca15ab3fc43f366fe1be04a68922ee8566cd24f77b9fe2caf6bc", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/enso-basic-climatology/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/enso-basic-climatology/cmip6/manifest.json index cf3e5eb44..eff37af87 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/enso-basic-climatology/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/enso-basic-climatology/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "f1a62fdcea7c346688eb476587778df31bec3e0b457b8fb5b4aeb7d57d2bb63e", "series.json": "2e7f6ee7162116378d21f0f405d639244ea8e53b74c2dfe2260ad0e1665b3ec9" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -147,6 +148,6 @@ "size": 93534 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/enso-characteristics/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/enso-characteristics/cmip6/manifest.json index 7db55d5b2..9870c424b 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/enso-characteristics/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/enso-characteristics/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "334cac04275b2581ab1129356f1242164d3bb5eaade4fc3de27fb9b89772f1a5", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "63803d2c8776cb1a2d869405aaac5513e512a08d0c36a92d1e7eba4351091024", @@ -59,6 +60,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/equilibrium-climate-sensitivity/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/equilibrium-climate-sensitivity/cmip6/manifest.json index 657b6d01a..746042b56 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/equilibrium-climate-sensitivity/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/equilibrium-climate-sensitivity/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "dd8c208f2134be1289606e1b12be998d327f4ae329318a5a83cfd82ac2325219", "series.json": "cac18de7ea3575ad34111d4e3402545ba358959e2037e5bc8c48fdf22f264267" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "4253b68b6116fc9afc5970a9186b20c82be9975330842a8e61efe812b1c2d0aa", @@ -43,6 +44,6 @@ "size": 8877 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/global-mean-timeseries/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/global-mean-timeseries/cmip6/manifest.json index 4e4a2c2af..c0b1136a5 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/global-mean-timeseries/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/global-mean-timeseries/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "ec46b24f66ebe4116471f65b6da7b1b141cd5fdc6683baba7b09c5f9579c207d", "series.json": "6e2a7bfbf5eb29f4380344a81b7022c772e2d5b22620266af47b739f6c92e5d6" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -35,6 +36,6 @@ "size": 9827 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/global-mean-timeseries/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/global-mean-timeseries/cmip7/manifest.json index ed2b6bb45..a07b297ce 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/global-mean-timeseries/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/global-mean-timeseries/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "9a8ff885fbf13c06d1974834210da71f48f83fa1700f335fce5813bf5c0535b2", "series.json": "30b664467549cef4af8a5cb11736fa8aa7b8c1042ea0401192bf6d7c03a9ac0b" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/ozone-annual-cycle/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/ozone-annual-cycle/cmip6/manifest.json index 6f7d01342..8ab2d212b 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/ozone-annual-cycle/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/ozone-annual-cycle/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "978fd3b10d7e7ff386b4c39dbdc1f33ef5509fcb3afd3759132eee6524fc1b60", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/ozone-annual-cycle/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/ozone-annual-cycle/cmip7/manifest.json index 08075cd11..def58c196 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/ozone-annual-cycle/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/ozone-annual-cycle/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "6f71c80892ecb575a43c0ee25a77769ad779162ae4da22fc1585ee869a1d3857", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/ozone-lat-time/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/ozone-lat-time/cmip6/manifest.json index 6764b53f4..a96abd484 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/ozone-lat-time/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/ozone-lat-time/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "806aec53b4c2ce0a9d9ec21cd2c0360d9b433ef2392f04053eead0f4e9e30a0e", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/ozone-lat-time/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/ozone-lat-time/cmip7/manifest.json index 85923cd86..509b065a8 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/ozone-lat-time/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/ozone-lat-time/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "758c03396e0b6b00a89ace6651712a6f9dc268f5d207fe3bbb35365351b68205", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/ozone-nh-mar/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/ozone-nh-mar/cmip6/manifest.json index d0192fbc6..2001df54e 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/ozone-nh-mar/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/ozone-nh-mar/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "a8589880fc34c8a12ab98fdccf5c893dc58c8d6ba76a72338c8297eb375af1f4", "series.json": "9cd508338685d5054387550f19f9b008ddc8a255bd6f28ba4bc909cc242a43ec" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -35,6 +36,6 @@ "size": 4422 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/ozone-nh-mar/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/ozone-nh-mar/cmip7/manifest.json index 65aae7f8d..5765aae25 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/ozone-nh-mar/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/ozone-nh-mar/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "a8589880fc34c8a12ab98fdccf5c893dc58c8d6ba76a72338c8297eb375af1f4", "series.json": "d8bf2a7bd65802c545927b8293bfda7ca47bc08c9160aa387291624b32c08571" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/ozone-sh-oct/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/ozone-sh-oct/cmip6/manifest.json index cfbbed55c..0e8e4fb1d 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/ozone-sh-oct/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/ozone-sh-oct/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "7474e7e327a830c53b821d5320e05bff2f55c24068a4926fb49c76c484ed2c9a", "series.json": "4b803facd29ee24afb14f81e2de5d5ae5dc3997a28f58a61eec90d27525d7c1d" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -35,6 +36,6 @@ "size": 4412 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/ozone-sh-oct/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/ozone-sh-oct/cmip7/manifest.json index b957db73a..e8cf34cc7 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/ozone-sh-oct/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/ozone-sh-oct/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "7474e7e327a830c53b821d5320e05bff2f55c24068a4926fb49c76c484ed2c9a", "series.json": "f40c1da8ac81e1ef2317fad9c17dbd78e04f336b226a9313307dbdfadbeb3fc0" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/ozone-zonal/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/ozone-zonal/cmip6/manifest.json index 246376dbc..f5b2dc46b 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/ozone-zonal/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/ozone-zonal/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "c3ad14429dfae8653c4ee2a86e24c5c8b49fa7f5ec4e7b13f3183aaf719fb75a", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-annual-cycle/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-annual-cycle/cmip6/manifest.json index f4cc21678..2228236ac 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-annual-cycle/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-annual-cycle/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "4face38eb05e1815739e9dd290ddffbb13989edd30ae4483927b3902915abc31", "series.json": "f4da9c7dbf8f6fdcc0ac6dd86043e6163a788034c8076e865bf2cf9703626b63" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -2363,6 +2364,6 @@ "size": 295674 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-annual-cycle/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-annual-cycle/cmip7/manifest.json index be373c4c6..145aa7558 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-annual-cycle/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-annual-cycle/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "4face38eb05e1815739e9dd290ddffbb13989edd30ae4483927b3902915abc31", "series.json": "8442bce313fac12b2c0e1dd66e7ed6702455600d02aabf319986780731dbec78" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-timeseries/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-timeseries/cmip6/manifest.json index 42c5a0947..086d4c72d 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-timeseries/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-timeseries/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "f4161df1ae5efcf92d271fe6f6f50b65d04efd73b503659479b6a7b242254ca0", "series.json": "0882a4bc2e3007cc5451574124079b9621f4e1fe07b48f3bf952ec6e83c8aa48" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -135,6 +136,6 @@ "size": 291052 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-timeseries/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-timeseries/cmip7/manifest.json index 9367c0df8..70d1de404 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-timeseries/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-timeseries/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "f4161df1ae5efcf92d271fe6f6f50b65d04efd73b503659479b6a7b242254ca0", "series.json": "4391f54d28471a6197c14749ea569075e179c0d69c4a15b461da36822b501e8c" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-trend/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-trend/cmip6/manifest.json index 9158b5f78..70b5e2f94 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-trend/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-trend/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "51aa6644a84d7632889f588d83bf8c7680caecdcae2c5af9c4f1f38a699f9e53", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "16da698a382486c5e22437b79581eb7effbb8308f0539b077b077348c0eb9491", @@ -83,6 +84,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-trend/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-trend/cmip7/manifest.json index 7cb785d65..b9c994852 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-trend/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/regional-historical-trend/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "51aa6644a84d7632889f588d83bf8c7680caecdcae2c5af9c4f1f38a699f9e53", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-area-basic/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-area-basic/cmip6/manifest.json index a901c5114..3a964ba37 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-area-basic/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-area-basic/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "ece9f19183b1bede25cda85779d3eb905f51bc24615559dbcc2ac6688835cd92", "series.json": "8ad040e4747b2fc954d9166da0e6a11749cf5f97efc51ea73e49b220b2062f8b" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84", @@ -63,6 +64,6 @@ "size": 17179 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-area-basic/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-area-basic/cmip7/manifest.json index c12d51d00..e5f0e2ea6 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-area-basic/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-area-basic/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "ece9f19183b1bede25cda85779d3eb905f51bc24615559dbcc2ac6688835cd92", "series.json": "1535dab04428841a3ac32a30acaa7c78ce57d8035cd3671518c01ab1db9e52eb" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-sensitivity/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-sensitivity/cmip6/manifest.json index 599b24ffb..d64db0bcf 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-sensitivity/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-sensitivity/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "0addb1511d0f758554aa7caac92ee3c254d279e2cb7da71e326dcdc191169939", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "a9fce0fa39fbba0cdd17aca8264f0d272f6e2b3efb0412cf851b958a4a6a08e2", @@ -55,6 +56,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-sensitivity/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-sensitivity/cmip7/manifest.json index f7cf844db..f237e17b1 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-sensitivity/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/sea-ice-sensitivity/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "0addb1511d0f758554aa7caac92ee3c254d279e2cb7da71e326dcdc191169939", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response-emissions/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response-emissions/cmip6/manifest.json index 594529a10..98b884f3a 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response-emissions/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response-emissions/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "666d10cf64bf489e34d726b6cad954dbd05b92cff0cb61b054b3a06d036c7558", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "0afbce145a1ba1dfe5fba69d3c49102d63d9b859722644a449accd7ea4f57fb6", @@ -35,6 +36,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response-emissions/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response-emissions/cmip7/manifest.json index f89855f77..7945b5bf5 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response-emissions/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response-emissions/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "666d10cf64bf489e34d726b6cad954dbd05b92cff0cb61b054b3a06d036c7558", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response/cmip6/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response/cmip6/manifest.json index 26ad6f8d2..0f8697103 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response/cmip6/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "fd052cef9fed3553114ac7840c1a3771f1e0e0c626fe439e5248739390278d69", "series.json": "1a798f01f18c15b28d20af2397d80304bbe116a7d4d8b83ba4a26baaad1732aa" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "7be2123937a4bda58e7ed538ecea9da9cf442636c3b1fa92c9d5e4e6296f5dd8", @@ -39,6 +40,6 @@ "size": 8670 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response/cmip7/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response/cmip7/manifest.json index 6cfb0155d..e454ca0ea 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response/cmip7/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/transient-climate-response/cmip7/manifest.json @@ -5,7 +5,8 @@ "output.json": "fd052cef9fed3553114ac7840c1a3771f1e0e0c626fe439e5248739390278d69", "series.json": "222fba950bba928affc9ec3d4c0267e71e5dba9e2ce6edae04289cc693034d27" }, + "diagnostic_version": 1, "native": {}, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/zero-emission-commitment/cmip6-1pctCO2-parent/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/zero-emission-commitment/cmip6-1pctCO2-parent/manifest.json index 6125d8c18..bbe52567d 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/zero-emission-commitment/cmip6-1pctCO2-parent/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/zero-emission-commitment/cmip6-1pctCO2-parent/manifest.json @@ -5,6 +5,7 @@ "output.json": "1ee3bdfdd9c227d6a61d02862d9c0d10c0534b7dda424faf9296fb7447fb47d5", "series.json": "b7be3eed74de5ab386270f0c741ffd6007908297d50670c20e14af0dcfcc1db6" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "7fe21a4c956b49714c373bf74135e6a9ebe99abdd92e1e32a7f7a33b3f523f7c", @@ -43,6 +44,6 @@ "size": 3884 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-esmvaltool/tests/test-data/zero-emission-commitment/cmip6-esm-1pctCO2-parent/manifest.json b/packages/climate-ref-esmvaltool/tests/test-data/zero-emission-commitment/cmip6-esm-1pctCO2-parent/manifest.json index 46e024676..b7ba00667 100644 --- a/packages/climate-ref-esmvaltool/tests/test-data/zero-emission-commitment/cmip6-esm-1pctCO2-parent/manifest.json +++ b/packages/climate-ref-esmvaltool/tests/test-data/zero-emission-commitment/cmip6-esm-1pctCO2-parent/manifest.json @@ -5,6 +5,7 @@ "output.json": "1ee3bdfdd9c227d6a61d02862d9c0d10c0534b7dda424faf9296fb7447fb47d5", "series.json": "51756d25e3969e66d38e2371879c1f2241e0e31f2826a6e9a4f9d5075e3fa341" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "b695021bb6d656c1e8c9051d3a50369cbbbc492787a04b5b0f94fca72fd39df3", @@ -43,6 +44,6 @@ "size": 3970 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-example/tests/test-data/global-mean-timeseries/cmip7/manifest.json b/packages/climate-ref-example/tests/test-data/global-mean-timeseries/cmip7/manifest.json index 91d02cecf..ecbea23ef 100644 --- a/packages/climate-ref-example/tests/test-data/global-mean-timeseries/cmip7/manifest.json +++ b/packages/climate-ref-example/tests/test-data/global-mean-timeseries/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "e937c8b787585d0c2f4025eb93a4effe8f4065ba4575285655d5be6cffa5db55", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "annual_mean_global_mean_timeseries.nc": { "sha256": "68f789157617604344470133d5749a94805934abb8624caa96d80866e28196b9", @@ -23,6 +24,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 2 } diff --git a/packages/climate-ref-example/tests/test-data/global-mean-timeseries/default/manifest.json b/packages/climate-ref-example/tests/test-data/global-mean-timeseries/default/manifest.json index 5ecb21bb3..573d1a904 100644 --- a/packages/climate-ref-example/tests/test-data/global-mean-timeseries/default/manifest.json +++ b/packages/climate-ref-example/tests/test-data/global-mean-timeseries/default/manifest.json @@ -5,6 +5,7 @@ "output.json": "e937c8b787585d0c2f4025eb93a4effe8f4065ba4575285655d5be6cffa5db55", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "annual_mean_global_mean_timeseries.nc": { "sha256": "acd6202a8fd46d5e1a8d5bef1650b20ab1a93452193aba56a0d05a173cacc62b", @@ -23,6 +24,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 2 } diff --git a/packages/climate-ref-example/tests/test-data/global-sst-bias/cmip7/manifest.json b/packages/climate-ref-example/tests/test-data/global-sst-bias/cmip7/manifest.json index b2c31944e..a76c2898e 100644 --- a/packages/climate-ref-example/tests/test-data/global-sst-bias/cmip7/manifest.json +++ b/packages/climate-ref-example/tests/test-data/global-sst-bias/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "3fe2b84a2d7c0b7c5332657ccd49b6dfdbb6634d8d48902e883da35cd74ad9d4", "series.json": "9749b54c192218ac34f9930145ed98d98f49f4903d89ab885009504d20f3d5e3" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "ef55f03a0716a1233d541e95c2b5957de28622223ef7a37831fb2bb4c29086ce", @@ -31,6 +32,6 @@ "size": 67440 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-example/tests/test-data/global-sst-bias/default/manifest.json b/packages/climate-ref-example/tests/test-data/global-sst-bias/default/manifest.json index 9080e509a..b2d31e00e 100644 --- a/packages/climate-ref-example/tests/test-data/global-sst-bias/default/manifest.json +++ b/packages/climate-ref-example/tests/test-data/global-sst-bias/default/manifest.json @@ -5,6 +5,7 @@ "output.json": "3fe2b84a2d7c0b7c5332657ccd49b6dfdbb6634d8d48902e883da35cd74ad9d4", "series.json": "3428ff5e5721babeff3dc23cbaec3ed9c3e3fa3960168d86e4d6e973cea3e4c3" }, + "diagnostic_version": 1, "native": { "diagnostic.json": { "sha256": "73ba96882f0ceffbd0ece0d7f1ce75275ab57e8b0ebab530ec2442bbdf35d43b", @@ -31,6 +32,6 @@ "size": 67393 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/burntfractionall-gfed/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/burntfractionall-gfed/cmip6/manifest.json index 4e0ae0049..07fc3a894 100644 --- a/packages/climate-ref-ilamb/tests/test-data/burntfractionall-gfed/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/burntfractionall-gfed/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "bea6d4be3d825f3ebca24eb6a164409d05cbce234eeab4378ad21c57a9d199d1", "series.json": "bfc11736e025968323c015e182d4473fc1e03726d7d2d2a241fcd8f0002cdc71" }, + "diagnostic_version": 2, "native": { "CESM2.csv": { "sha256": "d139d936df5f7a3c1b28572ae40058a6072b8decc0dc3f61eecef405ca2f5577", @@ -99,6 +100,6 @@ "size": 49856 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/burntfractionall-gfed/cmip7/manifest.json b/packages/climate-ref-ilamb/tests/test-data/burntfractionall-gfed/cmip7/manifest.json index 7b2c7be14..d21466cdc 100644 --- a/packages/climate-ref-ilamb/tests/test-data/burntfractionall-gfed/cmip7/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/burntfractionall-gfed/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "35453fd1efc52369b562391777b76c794c5a568a5b44054b006721bbec626412", "series.json": "52380eae3bfe7d7ee3c28ee18cd1342834a576eb6a7eaea36799c1b614cc16aa" }, + "diagnostic_version": 2, "native": { "CESM2.csv": { "sha256": "6dc152ee20eb5f59bb3d5b78461108d797077d6bbe3c2b8c283438aee44fd999", @@ -99,6 +100,6 @@ "size": 49877 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/csoil-hwsd2/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/csoil-hwsd2/cmip6/manifest.json index 38ebb13fa..96065d4f2 100644 --- a/packages/climate-ref-ilamb/tests/test-data/csoil-hwsd2/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/csoil-hwsd2/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "012cc1766c43d0ee125a4ae63af7b286d087a8590546fdcf83e312a5e7623c7f", "series.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "971b53c31db23387c9082d6e1c0b155e606bc1b745daa714b409016c3dd7b982", @@ -75,6 +76,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/csoil-hwsd2/cmip7/manifest.json b/packages/climate-ref-ilamb/tests/test-data/csoil-hwsd2/cmip7/manifest.json index b716ba321..21a73b14b 100644 --- a/packages/climate-ref-ilamb/tests/test-data/csoil-hwsd2/cmip7/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/csoil-hwsd2/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "95d0c169733834bf9256cc1bd42df6f7dacfc4e723c8c8dc9056b260ee513b69", "series.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "d4a40c7f4f710e7b3f46873a97efeb0dfd955c517a1a25639c47b888bd00b446", @@ -75,6 +76,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/emp-gleamgpcp2.3/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/emp-gleamgpcp2.3/cmip6/manifest.json index 9f2ad698f..4da728c26 100644 --- a/packages/climate-ref-ilamb/tests/test-data/emp-gleamgpcp2.3/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/emp-gleamgpcp2.3/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "9f24beceaf62c5265b854c5229d02f2f76f2e469e74c59c2d59593d554574c15", "series.json": "5960d070338f35c135b28478aab5597fd14fc420b7ee1bed1ec55c5caf165cda" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "1585c638ad594ea0de3dae17ae93e2d55433bf0d760230605b4bff6efb19c642", @@ -99,6 +100,6 @@ "size": 44378 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/gpp-fluxnet2015/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/gpp-fluxnet2015/cmip6/manifest.json index 14d514b64..5a7a7a110 100644 --- a/packages/climate-ref-ilamb/tests/test-data/gpp-fluxnet2015/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/gpp-fluxnet2015/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "b104e5cc5836bcb053adb84d751fb26bfb3553f8b9cb400794b36a06b4d3520d", "series.json": "b82eb7a34e7d53776efa5336e147f7968bdd293a8b11296e579fab2ac599b729" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "206554db9e8637be7853c93ca2ad280ff2589e52085c51fdc3a27f2b74fccfa8", @@ -99,6 +100,6 @@ "size": 61980 } }, - "schema": 1, + "schema": 2, "test_case_version": 2 } diff --git a/packages/climate-ref-ilamb/tests/test-data/gpp-wecann/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/gpp-wecann/cmip6/manifest.json index 243881ca0..0711db279 100644 --- a/packages/climate-ref-ilamb/tests/test-data/gpp-wecann/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/gpp-wecann/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "557db81cbf8d4bdc3a58345c983282d88a24a26e6fd1cdbdefc605d274690383", "series.json": "feb5c66a954730372ebfe6d281745a019320deb1849c6403a10333fae1ee1fe2" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "5a1752f4e0b3ad5b14fd9ea98929a07269bebb053d2e467a88af4ab0d711d3d3", @@ -147,6 +148,6 @@ "size": 22777 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/lai-avh15c1/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/lai-avh15c1/cmip6/manifest.json index 412960f53..4476f8df2 100644 --- a/packages/climate-ref-ilamb/tests/test-data/lai-avh15c1/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/lai-avh15c1/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "b22e04642705c74adb00f34c5c7986aa4f7d10a116c19a5fa9c284cc45148b75", "series.json": "b859a2496b255e4a69c312d3eef5e26679321c3f7a32734caa220ee2940b3c11" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "5db250961063212284500d6fc395aeaf2148f49f91a1cdceb12454dbc528f3f1", @@ -99,6 +100,6 @@ "size": 90705 } }, - "schema": 1, + "schema": 2, "test_case_version": 3 } diff --git a/packages/climate-ref-ilamb/tests/test-data/lai-avh15c1/cmip7/manifest.json b/packages/climate-ref-ilamb/tests/test-data/lai-avh15c1/cmip7/manifest.json index 094a2d8dd..875105ca1 100644 --- a/packages/climate-ref-ilamb/tests/test-data/lai-avh15c1/cmip7/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/lai-avh15c1/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "1b3fa881a32d26c1979e948b3bf9193c98c41b31218876e086a94f89a182e9a1", "series.json": "5e0aa42e83231b0df6d5b519f201a926789f7a500277da0a4a8b7d355eecc761" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "0c292eb9e484514e5bfea36a24978d33421b45515d5290a9369188c6db3d56a5", @@ -99,6 +100,6 @@ "size": 90678 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/mrro-lora/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/mrro-lora/cmip6/manifest.json index 2871eb2bb..a783c3816 100644 --- a/packages/climate-ref-ilamb/tests/test-data/mrro-lora/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/mrro-lora/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "b37d6fe9dcf6fad97714c8f8db6dd2ed0ac40c6d5a1975d332c3d35079c2fe14", "series.json": "7cb1f72133e2d08ff8b899457c8f126cc9e03a1f6e81b8b1d871a5a4cc21f84f" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "5f4351cbd895ec20cb654ee11068e6f5cdf22935e60c046f3a9418ebdf80cc25", @@ -107,6 +108,6 @@ "size": 94421 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/mrro-lora/cmip7/manifest.json b/packages/climate-ref-ilamb/tests/test-data/mrro-lora/cmip7/manifest.json index 79ffda5cf..116572712 100644 --- a/packages/climate-ref-ilamb/tests/test-data/mrro-lora/cmip7/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/mrro-lora/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "a3cf3e85975587406105d3ec6761d998be05751bbccd38ed03177f6dcfffae71", "series.json": "6a08c2da20d84c58bc87a6bbf37ca8104dca0e4f1993a6715bbe6f8ca834ae0a" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "52860806df8bf7004f034b1581d1bf38fdcde1703c3a4ff1a72d836dc1b49a3c", @@ -107,6 +108,6 @@ "size": 82149 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/mrsos-wangmao/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/mrsos-wangmao/cmip6/manifest.json index e7b78d815..c9dd6d0ee 100644 --- a/packages/climate-ref-ilamb/tests/test-data/mrsos-wangmao/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/mrsos-wangmao/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "37c11f0e73b8c5a48b3bab66fc04182156fa295aaadb27407333afc3511739f1", "series.json": "fc219c04c6cd29c93bb76bffa13334e095463fd14b432fe787a0298e29e507a6" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "7345d12369d41ec48f38a2a20aaf448e6d56fee3ae97482cdc75ae40c25d7f6a", @@ -99,6 +100,6 @@ "size": 121768 } }, - "schema": 1, + "schema": 2, "test_case_version": 3 } diff --git a/packages/climate-ref-ilamb/tests/test-data/nbp-hoffman/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/nbp-hoffman/cmip6/manifest.json index def687610..04cd8e8a1 100644 --- a/packages/climate-ref-ilamb/tests/test-data/nbp-hoffman/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/nbp-hoffman/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "3379523a4d505187c19788bda64ac220431caf342ecc0a5b14e11a35c36a4095", "series.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "dc2ea571e9648800cb740bb0bd9eb1fa3b509da3184530a1f4d4676c39faa87e", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/nbp-hoffman/cmip7/manifest.json b/packages/climate-ref-ilamb/tests/test-data/nbp-hoffman/cmip7/manifest.json index 819c6f3e6..96905c31a 100644 --- a/packages/climate-ref-ilamb/tests/test-data/nbp-hoffman/cmip7/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/nbp-hoffman/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "c9b572c25ee1b1df3d992cf9b54fa4886522c73b321da58eafc69486e134e0af", "series.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "091ae717f48776c03ec8a6d6c630b3fd85906366cee75f49cf65d5bfb2315173", @@ -43,6 +44,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/snc-esacci/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/snc-esacci/cmip6/manifest.json index 041d75310..042963fff 100644 --- a/packages/climate-ref-ilamb/tests/test-data/snc-esacci/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/snc-esacci/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "d8da1405ed69e80bacc72e5c6d5321b6a127d3fbc07a0f1d018358ab7f4af751", "series.json": "87dc060af17150527b07f6694ad31b9ea8101c1886bd56adf9c5eb9e90c22444" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "763f5be9446e2ee29a09e9d3e3e56e1ccc8d6d5fc99e9ebd00eb52b6c9545c24", @@ -99,6 +100,6 @@ "size": 83479 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/snc-esacci/cmip7/manifest.json b/packages/climate-ref-ilamb/tests/test-data/snc-esacci/cmip7/manifest.json index 8d61fc1b3..cf7339af5 100644 --- a/packages/climate-ref-ilamb/tests/test-data/snc-esacci/cmip7/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/snc-esacci/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "b9bd57f224a6a7a1847bceb0ca31ee70d9c6da56b7f9424f65ce210ce2a6efc0", "series.json": "e4014a3532f46ec3e3d93ec5d814aec685d5208b2c77a26812e84f188d33a8c9" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "9225bc53ab9b21e430ac00ee9aeb26b47cf661a98497d5ea5a3f4f85c38ead7d", @@ -99,6 +100,6 @@ "size": 83440 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/so-woa2023-surface/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/so-woa2023-surface/cmip6/manifest.json index 908e3a108..4a52aec40 100644 --- a/packages/climate-ref-ilamb/tests/test-data/so-woa2023-surface/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/so-woa2023-surface/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "ab674e556c95adf33dcc6b8235d7eb1cc38a2a5927f25fd096dc9b5250f82dd1", "series.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "5484012b34693af33960ae1047aa4032806b1eb01e8a98446654c664a15f4c0e", @@ -55,6 +56,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/so-woa2023-surface/cmip7/manifest.json b/packages/climate-ref-ilamb/tests/test-data/so-woa2023-surface/cmip7/manifest.json index e81ac0229..bebbd7714 100644 --- a/packages/climate-ref-ilamb/tests/test-data/so-woa2023-surface/cmip7/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/so-woa2023-surface/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "6005ba9d11745d6ecbe4dcf0f8085639522a67ec7d06f375854b665f705b258c", "series.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "8a8b75b692dcd1d70554ed1a1968b540c99f7e4228090cb48229172a0a2cfb03", @@ -55,6 +56,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/thetao-woa2023-surface/cmip6/manifest.json b/packages/climate-ref-ilamb/tests/test-data/thetao-woa2023-surface/cmip6/manifest.json index 3564f8c19..355c32ee7 100644 --- a/packages/climate-ref-ilamb/tests/test-data/thetao-woa2023-surface/cmip6/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/thetao-woa2023-surface/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "ab674e556c95adf33dcc6b8235d7eb1cc38a2a5927f25fd096dc9b5250f82dd1", "series.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "d79bfa94aaedcaef94c3e353c2e00cd6139d6a612d617b65c23625236b4775a5", @@ -55,6 +56,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-ilamb/tests/test-data/thetao-woa2023-surface/cmip7/manifest.json b/packages/climate-ref-ilamb/tests/test-data/thetao-woa2023-surface/cmip7/manifest.json index 7adcfa94c..ac4027aef 100644 --- a/packages/climate-ref-ilamb/tests/test-data/thetao-woa2023-surface/cmip7/manifest.json +++ b/packages/climate-ref-ilamb/tests/test-data/thetao-woa2023-surface/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "6005ba9d11745d6ecbe4dcf0f8085639522a67ec7d06f375854b665f705b258c", "series.json": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, + "diagnostic_version": 2, "native": { "CanESM5.csv": { "sha256": "1ee9108bec059150e53c195c4d41bb08dbd4283c94130c0eb4ddabecb9f84f33", @@ -55,6 +56,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-pr/manifest.json b/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-pr/manifest.json index 1774f62a8..727d29c3a 100644 --- a/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-pr/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-pr/manifest.json @@ -5,6 +5,7 @@ "output.json": "62f8d92c40e3a93b6f4feb4b4de84784a5feff3faf3279e4f5decb661543674a", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 2, "native": { "diagnostic.json": { "sha256": "f9bc138ec6090e97f4f42a5cc6e83bac2a2fb022178437698604b18721d027c9", @@ -63,6 +64,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-ts/manifest.json b/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-ts/manifest.json index a8f25382f..adedae40b 100644 --- a/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-ts/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-ts/manifest.json @@ -5,6 +5,7 @@ "output.json": "e46ca7fa84827e863165dc002b50f0a4a476fdc2ae4c228cbaeddbad86c5d9f7", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 2, "native": { "diagnostic.json": { "sha256": "c4d890859f86fb60560a013fc3327d8e05762e88c4a3e51cab4e1378f4220e8a", @@ -43,6 +44,6 @@ "size": 12567 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip7-ts/manifest.json b/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip7-ts/manifest.json index ee24e8442..804f83009 100644 --- a/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip7-ts/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip7-ts/manifest.json @@ -5,6 +5,7 @@ "output.json": "e46ca7fa84827e863165dc002b50f0a4a476fdc2ae4c228cbaeddbad86c5d9f7", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 2, "native": { "diagnostic.json": { "sha256": "644722f87a8eaaf6c39449d1cce2963222d4547a37007e84941aab75057b53da", @@ -43,6 +44,6 @@ "size": 12567 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/enso_proc/cmip6/manifest.json b/packages/climate-ref-pmp/tests/test-data/enso_proc/cmip6/manifest.json index 8cf705dc4..50ef7fe84 100644 --- a/packages/climate-ref-pmp/tests/test-data/enso_proc/cmip6/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/enso_proc/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "e5f199fbf35da8f9f4e2608474e8c711e93a8b2b429abfafc9e3c5b04a221efd", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "ENSO_proc_ACCESS-ESM1-5_historical_r1i1p1f1_BiasSstLonRmse.nc": { "sha256": "bb8f6c971556094b55f6b1adcc111cee0f504ed7c1564364c431dcd73aca7462", @@ -143,6 +144,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/enso_proc/cmip7/manifest.json b/packages/climate-ref-pmp/tests/test-data/enso_proc/cmip7/manifest.json index e219b523c..2a56a5f5a 100644 --- a/packages/climate-ref-pmp/tests/test-data/enso_proc/cmip7/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/enso_proc/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "e5f199fbf35da8f9f4e2608474e8c711e93a8b2b429abfafc9e3c5b04a221efd", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "ENSO_proc_ACCESS-ESM1-5_historical_r1i1p1f1_BiasSstLonRmse.nc": { "sha256": "9af94d6d15996e558e57986efa127381f413354b832c694236fb74f18fad77cb", @@ -143,6 +144,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/enso_tel/cmip6/manifest.json b/packages/climate-ref-pmp/tests/test-data/enso_tel/cmip6/manifest.json index 74fc7e002..59a54632b 100644 --- a/packages/climate-ref-pmp/tests/test-data/enso_tel/cmip6/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/enso_tel/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "c62a83a4a1fd9a05e038666a26220d0f8dbaa25bd24d31445073fd21e01706d8", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "ENSO_tel_ACCESS-ESM1-5_historical_r1i1p1f1_EnsoAmpl.nc": { "sha256": "e9eff2c8b11fa104237eb92d0df592878eba1dde1d338343fdad17d395b23a12", @@ -187,6 +188,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/enso_tel/cmip7/manifest.json b/packages/climate-ref-pmp/tests/test-data/enso_tel/cmip7/manifest.json index 15b0a59cf..76ed63bc8 100644 --- a/packages/climate-ref-pmp/tests/test-data/enso_tel/cmip7/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/enso_tel/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "c62a83a4a1fd9a05e038666a26220d0f8dbaa25bd24d31445073fd21e01706d8", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "ENSO_tel_ACCESS-ESM1-5_historical_r1i1p1f1_EnsoAmpl.nc": { "sha256": "cc2458e45806d117169a1b96f523daf5a1a8d56489eefacdf118379d855db5f7", @@ -187,6 +188,6 @@ "size": 2 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nam/cmip6/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nam/cmip6/manifest.json index 515865e16..f582ff6d6 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nam/cmip6/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nam/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "1904facfa2f658600751747cfab2d2e6fbcc603ecb40de975d5e6af793c44300", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_NAM_DJF_20CR_1900-2005.png": { "sha256": "14de2e777b2c8b9c46c9743c6e8ae3273612e1c95aab68d05ab59d6f550f9e7d", @@ -215,6 +216,6 @@ "size": 12335 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nam/cmip7/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nam/cmip7/manifest.json index 717292fcb..ffbb3d9af 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nam/cmip7/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nam/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "dd8293b4698a4332c2137e87b6f3550eb6fb388aa60c5f06fbc662b4ec7f6edd", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_NAM_DJF_20CR_1900-2005.png": { "sha256": "14de2e777b2c8b9c46c9743c6e8ae3273612e1c95aab68d05ab59d6f550f9e7d", @@ -215,6 +216,6 @@ "size": 12335 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nao/cmip6/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nao/cmip6/manifest.json index ed7694db8..f9ee6e7ea 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nao/cmip6/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nao/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "bb89fc63c1a99abc12eb1a558ac088a1671620d3db6b132af7add1bf502e3ea8", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_NAO_DJF_20CR_1900-2005.png": { "sha256": "474dc87d541ce4787d56ac7b7a93adc6f337607a759f62c0902efa3992a433f8", @@ -215,6 +216,6 @@ "size": 12344 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nao/cmip7/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nao/cmip7/manifest.json index 669e952ab..a35d5c27a 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nao/cmip7/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-nao/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "5ce2d3227e91bdc787fd693e2c57eb4638e179df493045f0af032448f3592246", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_NAO_DJF_20CR_1900-2005.png": { "sha256": "474dc87d541ce4787d56ac7b7a93adc6f337607a759f62c0902efa3992a433f8", @@ -215,6 +216,6 @@ "size": 12344 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npgo/cmip6/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npgo/cmip6/manifest.json index 92d7c1e20..890a7dc2e 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npgo/cmip6/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npgo/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "e36bf1d06d3635f2047609530f9da8d0dc048f352df424f0168d5bb913145079", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_NPGO_monthly_HadISST-1-1_1900-2005.png": { "sha256": "f7e720e3115fd6cd39e57144a4aaf3cc427944c761436b87e96f752a351c9ddc", @@ -83,6 +84,6 @@ "size": 5457 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npgo/cmip7/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npgo/cmip7/manifest.json index 40e1b0b37..32f02faf7 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npgo/cmip7/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npgo/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "000fe71a4613185d864720121db10eaadb1a312e5e20199346421223a804999e", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_NPGO_monthly_HadISST-1-1_1900-2005.png": { "sha256": "f7e720e3115fd6cd39e57144a4aaf3cc427944c761436b87e96f752a351c9ddc", @@ -83,6 +84,6 @@ "size": 5456 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npo/cmip6/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npo/cmip6/manifest.json index 3e1b67f82..f8ab355f8 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npo/cmip6/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npo/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "bf0dedc8a6242551d92ae91c1bcc8424341c46cec150edb7600f58783ec49108", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_NPO_DJF_20CR_1900-2005.png": { "sha256": "cd7d18a767f9e3563783b655ab9a248b6d61bc3ac25e7a075bb631fca29d75bf", @@ -263,6 +264,6 @@ "size": 16243 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npo/cmip7/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npo/cmip7/manifest.json index a93f62a40..83f01f96c 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npo/cmip7/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-npo/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "de79a3ddc42c9739cc69e39a77348f8d119b871ac46a161e199663f8280a501e", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_NPO_DJF_20CR_1900-2005.png": { "sha256": "cd7d18a767f9e3563783b655ab9a248b6d61bc3ac25e7a075bb631fca29d75bf", @@ -263,6 +264,6 @@ "size": 16243 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pdo/cmip6/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pdo/cmip6/manifest.json index 189f23bc0..fa1f08bd8 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pdo/cmip6/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pdo/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "a882dd976926ecf89e7fc7efbea9e0fd2ddd2280c45760a4cff284d6bbe73f62", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_PDO_monthly_HadISST-1-1_1900-2005.png": { "sha256": "b39032778d59e47a7c3269e66aed9aadab04c887de0c19de65317350df53fe19", @@ -71,6 +72,6 @@ "size": 4464 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pdo/cmip7/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pdo/cmip7/manifest.json index 75bff0902..0b40b1c06 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pdo/cmip7/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pdo/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "37f0eed421f73813e27df652aeab97a28a142adda89d3f5e26862b345c92168c", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_PDO_monthly_HadISST-1-1_1900-2005.png": { "sha256": "b39032778d59e47a7c3269e66aed9aadab04c887de0c19de65317350df53fe19", @@ -71,6 +72,6 @@ "size": 4459 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pna/cmip6/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pna/cmip6/manifest.json index 63c27adc1..1de10b91c 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pna/cmip6/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pna/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "e20535bfc6e337eb4f36b90f75494a192878dc04b6a83cc334bc7595aafb05b4", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_PNA_DJF_20CR_1900-2005.png": { "sha256": "507f5faee65bd25a8d517604e236fae5c6fa0759c040dbfa96d1be6a1275a88b", @@ -215,6 +216,6 @@ "size": 12338 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pna/cmip7/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pna/cmip7/manifest.json index a7f3f246a..6794bbce2 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pna/cmip7/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-pna/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "341b4e6788ae59388c9bd54805e5340e72358b0491b825a7f0d93518f3725eb6", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_PNA_DJF_20CR_1900-2005.png": { "sha256": "507f5faee65bd25a8d517604e236fae5c6fa0759c040dbfa96d1be6a1275a88b", @@ -215,6 +216,6 @@ "size": 12338 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-sam/cmip6/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-sam/cmip6/manifest.json index 0d8a2509c..779a02bc3 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-sam/cmip6/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-sam/cmip6/manifest.json @@ -5,6 +5,7 @@ "output.json": "e12259f127181727494f357ae5480074a7c3bff706bacc087fd6c53b02641587", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_SAM_DJF_20CR_1950-2005.png": { "sha256": "c7ef152390fa88c2cd263a03855103fe895e10e1be04497faf284e7ce93d4220", @@ -215,6 +216,6 @@ "size": 12341 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-sam/cmip7/manifest.json b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-sam/cmip7/manifest.json index 7a5c7a7e5..67c3b5a18 100644 --- a/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-sam/cmip7/manifest.json +++ b/packages/climate-ref-pmp/tests/test-data/extratropical-modes-of-variability-sam/cmip7/manifest.json @@ -5,6 +5,7 @@ "output.json": "9eeab48e41a1de571f02f399dc4bd66e4f8e6406b498c108e3baaa99041a6486", "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570" }, + "diagnostic_version": 1, "native": { "EG_Spec_North_test_SAM_DJF_20CR_1950-2005.png": { "sha256": "c7ef152390fa88c2cd263a03855103fe895e10e1be04497faf284e7ce93d4220", @@ -215,6 +216,6 @@ "size": 12341 } }, - "schema": 1, + "schema": 2, "test_case_version": 1 } diff --git a/packages/climate-ref/src/climate_ref/cli/test_cases/__init__.py b/packages/climate-ref/src/climate_ref/cli/test_cases/__init__.py index 06d14cdde..2f65a199f 100644 --- a/packages/climate-ref/src/climate_ref/cli/test_cases/__init__.py +++ b/packages/climate-ref/src/climate_ref/cli/test_cases/__init__.py @@ -11,6 +11,7 @@ - :mod:`~climate_ref.cli.test_cases.baselines` -- ``replay`` / ``mint`` / ``build`` - :mod:`~climate_ref.cli.test_cases.store` -- ``sync`` / ``check-store`` - :mod:`~climate_ref.cli.test_cases.ci_gate` -- ``ci-gate`` +- :mod:`~climate_ref.cli.test_cases.migrate` -- ``migrate-manifests`` """ from importlib import import_module @@ -26,7 +27,7 @@ # Import each command module for its registration side effect on ``app``. # The sequence below is the order the verbs appear in ``ref test-cases --help`` # a dynamic import keeps it explicit, where a plain ``import`` block would be alphabetised by ruff. -for _command_module in ("discovery", "run", "baselines", "store", "ci_gate"): +for _command_module in ("discovery", "run", "baselines", "store", "ci_gate", "migrate"): import_module(f"{__name__}.{_command_module}") __all__ = [ diff --git a/packages/climate-ref/src/climate_ref/cli/test_cases/_common.py b/packages/climate-ref/src/climate_ref/cli/test_cases/_common.py index 8dbddb538..2c05a0b50 100644 --- a/packages/climate-ref/src/climate_ref/cli/test_cases/_common.py +++ b/packages/climate-ref/src/climate_ref/cli/test_cases/_common.py @@ -100,10 +100,11 @@ def _validate_requested_filters( raise typer.Exit(code=1) -def _write_test_case_manifest( +def _write_test_case_manifest( # noqa: PLR0913 paths: TestCasePaths, *, test_case_version: int, + diagnostic_version: int, committed: dict[str, str], native: dict[str, NativeEntry], schema: int | None = None, @@ -114,6 +115,8 @@ def _write_test_case_manifest( Shared by ``run`` (which preserves the existing version and native block) and ``mint`` (which authors the native block and may bump the version); the two callers differ only in the ``test_case_version`` and ``native`` they supply. + Only ``mint`` advances ``diagnostic_version`` to the diagnostic's current + ``Diagnostic.version``; ``run`` / ``build`` preserve the value already recorded. The ``catalog_hash`` is always (re)derived from the current ``catalog.yaml`` so the manifest stays coupled to the inputs that produced the committed bundle. """ @@ -123,6 +126,7 @@ def _write_test_case_manifest( manifest = Manifest( schema=SCHEMA_VERSION if schema is None else schema, test_case_version=test_case_version, + diagnostic_version=diagnostic_version, committed=dict(committed), native=native, catalog_hash=get_catalog_hash(paths.catalog), diff --git a/packages/climate-ref/src/climate_ref/cli/test_cases/baselines.py b/packages/climate-ref/src/climate_ref/cli/test_cases/baselines.py index 2e52c4798..891604ed5 100644 --- a/packages/climate-ref/src/climate_ref/cli/test_cases/baselines.py +++ b/packages/climate-ref/src/climate_ref/cli/test_cases/baselines.py @@ -384,7 +384,13 @@ def mint_native( # noqa: PLR0912, PLR0913, PLR0915 version = previous.test_case_version + 1 if bump_version else previous.test_case_version else: version = 1 - _write_test_case_manifest(paths, test_case_version=version, committed=committed, native=native) + _write_test_case_manifest( + paths, + test_case_version=version, + diagnostic_version=diag.version, + committed=committed, + native=native, + ) write_source_stamp(slot, label=label, verb="mint", source=source_kind, test_case_version=version) minted += 1 @@ -497,6 +503,7 @@ def build_test_case( # noqa: PLR0912, PLR0913, PLR0915 _write_test_case_manifest( paths, test_case_version=previous.test_case_version, + diagnostic_version=previous.diagnostic_version, committed=committed, native=previous.native, schema=previous.schema, @@ -507,7 +514,13 @@ def build_test_case( # noqa: PLR0912, PLR0913, PLR0915 "re-mint with `ref test-cases mint`" ) else: - _write_test_case_manifest(paths, test_case_version=1, committed=committed, native={}) + _write_test_case_manifest( + paths, + test_case_version=1, + diagnostic_version=diag.version, + committed=committed, + native={}, + ) logger.info(f"Promoted rebuilt bundle to regression baseline: {paths.regression}") else: logger.info( diff --git a/packages/climate-ref/src/climate_ref/cli/test_cases/ci_gate.py b/packages/climate-ref/src/climate_ref/cli/test_cases/ci_gate.py index b914fa875..d0f2f5161 100644 --- a/packages/climate-ref/src/climate_ref/cli/test_cases/ci_gate.py +++ b/packages/climate-ref/src/climate_ref/cli/test_cases/ci_gate.py @@ -275,6 +275,7 @@ def record(case: str, action: Action, reason: str) -> None: extraction_changed=extraction_changed, committed_integrity_ok=committed_integrity_ok, catalog_integrity_ok=catalog_integrity_ok, + code_diagnostic_version=diag.version, ) record(case_id, decision.action, decision.reason) diff --git a/packages/climate-ref/src/climate_ref/cli/test_cases/migrate.py b/packages/climate-ref/src/climate_ref/cli/test_cases/migrate.py new file mode 100644 index 000000000..c057ba47c --- /dev/null +++ b/packages/climate-ref/src/climate_ref/cli/test_cases/migrate.py @@ -0,0 +1,118 @@ +""" +``ref test-cases migrate-manifests``. + +A one-shot, idempotent maintenance command that backfills every committed +``manifest.json`` to the current ``SCHEMA_VERSION``, stamping each case's +``diagnostic_version`` from the diagnostic's in-code ``Diagnostic.version`` only +when the field is absent. An already-recorded ``diagnostic_version`` is preserved, +so re-running never re-stamps a stale bundle and subverts the gate's staleness check. + +It iterates the same ``_iter_test_cases`` the CI gate uses, so its coverage is +definitionally aligned with the gate (iterator parity). A pre-bump manifest is +schema-1 and ``Manifest.load`` would reject it, so the raw JSON is read directly +and re-serialised through ``Manifest(...).dump`` to keep the output byte-identical +to what a later ``mint`` would write. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Annotated + +import typer +from loguru import logger + +from climate_ref.cli.test_cases._app import app +from climate_ref.cli.test_cases._common import ( + _iter_test_cases, + _validate_provider_in_registry, + _validate_requested_filters, +) +from climate_ref.config import Config + +if TYPE_CHECKING: + from rich.console import Console + + +@app.command(name="migrate-manifests") +def migrate_manifests( + ctx: typer.Context, + provider: Annotated[ + str | None, + typer.Option(help="Limit the migration to a single provider slug"), + ] = None, + diagnostic: Annotated[ + str | None, + typer.Option(help="Limit the migration to a single diagnostic slug"), + ] = None, + test_case: Annotated[ + str | None, + typer.Option(help="Limit the migration to a single test case name"), + ] = None, +) -> None: + """ + Backfill every committed ``manifest.json`` to the current schema. + + For each test case with an existing manifest, backfills ``diagnostic_version`` + from the diagnostic's in-code ``Diagnostic.version`` when it is absent (an + already-recorded value is preserved) and rewrites the manifest at the current + ``SCHEMA_VERSION``. The operation is idempotent: re-running on an already-migrated + manifest preserves the recorded value and produces identical bytes. + + Examples + -------- + ref test-cases migrate-manifests + ref test-cases migrate-manifests --provider ilamb + """ + from climate_ref.provider_registry import ProviderRegistry + from climate_ref_core.regression.manifest import SCHEMA_VERSION, Manifest, NativeEntry + from climate_ref_core.testing import TestCasePaths + + config: Config = ctx.obj.config + db = ctx.obj.database + console: Console = ctx.obj.console + + registry = ProviderRegistry.build_from_config(config, db) + _validate_provider_in_registry(registry, provider) + _validate_requested_filters(registry, provider=provider, diagnostic=diagnostic, test_case=test_case) + cases = list(_iter_test_cases(registry, provider=provider, diagnostic=diagnostic, test_case=test_case)) + + migrated = 0 + skipped = 0 + + for diag, tc in cases: + case_id = f"{diag.provider.slug}/{diag.slug}/{tc.name}" + paths = TestCasePaths.from_diagnostic(diag, tc.name) + if paths is None or not paths.manifest.exists(): + skipped += 1 + continue + + # Manifest.load rejects a schema-1 manifest, so read raw JSON and re-construct. + data = json.loads(paths.manifest.read_text(encoding="utf-8")) + native = { + relpath: NativeEntry(sha256=entry["sha256"], size=entry["size"]) + for relpath, entry in data["native"].items() + } + # Preserve an already-recorded value so this stays a pure backfill. + # After an authorised bump the manifest lags the in-code version until a re-mint, + # and re-stamping it would mask the gate's staleness check. + diagnostic_version = data.get("diagnostic_version", diag.version) + manifest = Manifest( + schema=SCHEMA_VERSION, + test_case_version=data["test_case_version"], + diagnostic_version=diagnostic_version, + committed=dict(data["committed"]), + native=native, + catalog_hash=data.get("catalog_hash"), + ) + # Serialise via dump so the bytes (sort_keys + unconditional `catalog_hash: null`) + # match exactly what a later mint would write. + manifest.dump(paths.manifest) + migrated += 1 + logger.info(f"Migrated {case_id} -> schema {SCHEMA_VERSION}, diagnostic_version {diagnostic_version}") + + console.print() + console.print( + f"[green]Migrated {migrated} manifest(s)[/green]" + + (f"; skipped {skipped} case(s) with no manifest" if skipped else "") + ) diff --git a/packages/climate-ref/src/climate_ref/cli/test_cases/run.py b/packages/climate-ref/src/climate_ref/cli/test_cases/run.py index bcb22cfa8..7cf3dcf7e 100644 --- a/packages/climate-ref/src/climate_ref/cli/test_cases/run.py +++ b/packages/climate-ref/src/climate_ref/cli/test_cases/run.py @@ -222,6 +222,7 @@ def _run_single_test_case( # noqa: PLR0911, PLR0912, PLR0913, PLR0915 _write_test_case_manifest( paths, test_case_version=previous.test_case_version, + diagnostic_version=previous.diagnostic_version, committed=committed, native=previous.native, schema=previous.schema, @@ -232,7 +233,13 @@ def _run_single_test_case( # noqa: PLR0911, PLR0912, PLR0913, PLR0915 "re-mint with `ref test-cases mint` (or `mint --from-replay`)" ) else: - _write_test_case_manifest(paths, test_case_version=1, committed=committed, native={}) + _write_test_case_manifest( + paths, + test_case_version=1, + diagnostic_version=diag.version, + committed=committed, + native={}, + ) logger.info(f"Updated regression baseline: {paths.regression}") _print_regression_summary(console, paths.regression, size_threshold) else: diff --git a/packages/climate-ref/tests/integration/test_native_roundtrip.py b/packages/climate-ref/tests/integration/test_native_roundtrip.py index 84c032719..7e03c4d29 100644 --- a/packages/climate-ref/tests/integration/test_native_roundtrip.py +++ b/packages/climate-ref/tests/integration/test_native_roundtrip.py @@ -28,7 +28,9 @@ import pandas as pd import pytest +from climate_ref.cli.test_cases._common import _iter_test_cases from climate_ref.config import Config +from climate_ref.provider_registry import ProviderRegistry from climate_ref.testing import TEST_DATA_DIR, TestCaseRunner from climate_ref_core.datasets import ( DatasetCollection, @@ -54,6 +56,8 @@ capture_execution, materialise_native, ) +from climate_ref_core.regression.gate import Action, decide_coupling +from climate_ref_core.regression.manifest import SCHEMA_VERSION from climate_ref_core.testing import ( TestCase, TestCasePaths, @@ -368,8 +372,9 @@ def test_run_mint_sync_replay(self, tmp_path: Path) -> None: assert digest == entry.sha256, "store digest must equal the captured digest" manifest = Manifest( - schema=1, + schema=2, test_case_version=1, + diagnostic_version=1, committed=committed_digests, native=native, ) @@ -575,8 +580,9 @@ def test_example_roundtrip(self, tmp_path: Path) -> None: # noqa: PLR0915 assert digest == entry.sha256 manifest = Manifest( - schema=1, + schema=2, test_case_version=1, + diagnostic_version=1, committed=committed_digests, native=native, ) @@ -723,8 +729,9 @@ def _mint(self, tmp_path: Path) -> tuple[LocalFilesystemStore, Manifest, dict[st store.put(base_dir / relpath) manifest = Manifest( - schema=1, + schema=2, test_case_version=1, + diagnostic_version=1, committed=committed_digests, native=native, ) @@ -1141,3 +1148,63 @@ def test_replay_fails_when_reconstruction_input_not_captured(self, tmp_path: Pat ) with pytest.raises((FileNotFoundError, OSError)): diag.build_execution_result(replay_definition) + + +class TestManifestSchemaMigration: + """Every committed manifest is migrated to the current schema and stamped correctly.""" + + def test_all_committed_manifests_are_schema_2(self, config, db) -> None: + """For every iterated case with a manifest, it parses at schema 2 with the code's version. + + The case universe is derived from the same ``_iter_test_cases`` the gate and the + migration verb use (iterator parity), never a hardcoded count. A schema-1 manifest left + behind by a partial backfill would raise on load and red this test. + """ + registry = ProviderRegistry.build_from_config(config, db) + checked = 0 + for diag, tc in _iter_test_cases(registry): + paths = TestCasePaths.from_diagnostic(diag, tc.name) + if paths is None or not paths.manifest.exists(): + continue + manifest = Manifest.load(paths.manifest) + assert manifest.schema == SCHEMA_VERSION, ( + f"{diag.provider.slug}/{diag.slug}/{tc.name} is not at schema {SCHEMA_VERSION}" + ) + assert manifest.diagnostic_version == diag.version, ( + f"{diag.provider.slug}/{diag.slug}/{tc.name} records diagnostic_version " + f"{manifest.diagnostic_version} but the code declares {diag.version}" + ) + checked += 1 + # The branch ships committed manifests, so the iterator must have reached at least one. + assert checked > 0 + + def test_schema_1_base_manifest_is_treated_as_seeding(self) -> None: + """A schema-1 base manifest is rejected by ``loads`` and falls back to seeding. + + This pins the one-cycle transition window: the schema-2 ``loads`` deliberately + rejects the base-branch copy (suppressed via the ci_gate ``try/except``), so the case + is treated as newly added rather than crashing the gate. + """ + # A full schema-1 base manifest text (no diagnostic_version) is rejected by loads(). + schema_1_text = json.dumps( + { + "schema": 1, + "test_case_version": 1, + "committed": {"output.json": "a" * 64}, + "native": {}, + } + ) + with pytest.raises(ValueError): + Manifest.loads(schema_1_text) + + # The ci_gate path catches that ValueError and passes base_manifest=None (seeding). + current = Manifest( + schema=2, + test_case_version=1, + diagnostic_version=1, + committed={"output.json": "a" * 64}, + native={"data.nc": NativeEntry("a" * 64, 10)}, + ) + decision = decide_coupling(current, None, code_diagnostic_version=1) + assert decision.action is Action.REPLAY + assert "seeding" in decision.reason diff --git a/packages/climate-ref/tests/unit/cli/test_test_cases.py b/packages/climate-ref/tests/unit/cli/test_test_cases.py index 221bba311..2a7fd8796 100644 --- a/packages/climate-ref/tests/unit/cli/test_test_cases.py +++ b/packages/climate-ref/tests/unit/cli/test_test_cases.py @@ -818,6 +818,7 @@ def test_run_with_force_regen(self, invoke_cli, mocker, tmp_path): mock_spec = MagicMock() mock_spec.test_cases = [mock_tc] mock_diag = MagicMock(slug="test-diag", test_data_spec=mock_spec) + mock_diag.version = 1 mock_diag.provider = MagicMock(slug="example") mock_provider = MagicMock(slug="example") mock_provider.diagnostics.return_value = [mock_diag] @@ -1042,6 +1043,7 @@ def test_run_warns_when_native_stale(self, invoke_cli, mocker, tmp_path): Manifest( schema=SCHEMA_VERSION, test_case_version=3, + diagnostic_version=4, committed={}, native={"diagnostic.json": NativeEntry(sha256="00" * 32, size=1)}, ).dump(paths.manifest) @@ -1054,6 +1056,8 @@ def test_run_warns_when_native_stale(self, invoke_cli, mocker, tmp_path): # run preserves the mint-owned native block and version (it never authors native). manifest = Manifest.load(paths.manifest) assert manifest.test_case_version == 3 + # run preserves the recorded diagnostic_version (only mint advances it). + assert manifest.diagnostic_version == 4 assert set(manifest.native) == {"diagnostic.json"} @@ -1400,6 +1404,7 @@ def _make_case_mocks(): spec = MagicMock() spec.test_cases = [tc] diag = MagicMock(slug="test-diag", test_data_spec=spec) + diag.version = 1 diag.provider = MagicMock(slug="example") provider = MagicMock(slug="example") provider.diagnostics.return_value = [diag] @@ -1515,8 +1520,9 @@ def test_sync_fetches_missing_blob(self, invoke_cli, mocker, tmp_path): case_dir.mkdir(parents=True) paths = TestCasePaths(root=case_dir) Manifest( - schema=1, + schema=2, test_case_version=1, + diagnostic_version=1, committed={}, native={"out.nc": NativeEntry(sha256=digest, size=len(b"native-data"))}, ).dump(paths.manifest) @@ -1543,8 +1549,9 @@ def test_sync_hard_fails_on_unservable_blob(self, invoke_cli, mocker, tmp_path): case_dir.mkdir(parents=True) paths = TestCasePaths(root=case_dir) Manifest( - schema=1, + schema=2, test_case_version=1, + diagnostic_version=1, committed={}, native={"out.nc": NativeEntry(sha256="ab" * 32, size=1)}, ).dump(paths.manifest) @@ -1779,7 +1786,9 @@ def test_replay_empty_native_is_hard_failure(self, invoke_cli, mocker, tmp_path) (regression_dir / "diagnostic.json").write_text("{}") digests = compute_committed_digests(regression_dir) paths = TestCasePaths(root=case_dir) - Manifest(schema=1, test_case_version=1, committed=digests, native={}).dump(paths.manifest) + Manifest(schema=2, test_case_version=1, diagnostic_version=1, committed=digests, native={}).dump( + paths.manifest + ) store = LocalFilesystemStore(root=tmp_path / "store") mocker.patch( @@ -1864,8 +1873,9 @@ def test_replay_integrity_mismatch_warns_and_continues(self, invoke_cli, mocker, paths = TestCasePaths(root=case_dir) # Committed digest deliberately wrong -> integrity mismatch (now a warning, not a gate). Manifest( - schema=1, + schema=2, test_case_version=1, + diagnostic_version=1, committed={"diagnostic.json": "00" * 32}, native={}, ).dump(paths.manifest) @@ -1915,8 +1925,9 @@ def test_replay_reconciles_integrity_mismatch_within_tolerance(self, invoke_cli, paths = TestCasePaths(root=case_dir) # Wrong committed digest -> integrity warns; native present -> passes the mint guard. Manifest( - schema=1, + schema=2, test_case_version=1, + diagnostic_version=1, committed={"series.json": "00" * 32}, native={"out.nc": NativeEntry(sha256="ab" * 32, size=1)}, ).dump(paths.manifest) @@ -2126,6 +2137,8 @@ def test_mint_writes_blobs_and_manifest(self, invoke_cli, mocker, tmp_path): manifest = Manifest.load(paths.manifest) assert manifest.test_case_version == 1 + # mint stamps the diagnostic's in-code Diagnostic.version. + assert manifest.diagnostic_version == 1 # mint authored the native block from the captured snapshot. assert set(manifest.native) == {"diagnostic.json", "output.json", "series.json"} # Each native blob was PUT into the store. @@ -2198,8 +2211,9 @@ def test_mint_from_replay_preserves_stored_native(self, invoke_cli, mocker, tmp_ paths = TestCasePaths(root=case_dir) stored_native = {"output.json": NativeEntry(sha256="aa" * 32, size=10)} Manifest( - schema=1, + schema=2, test_case_version=1, + diagnostic_version=1, committed={"diagnostic.json": "00" * 32}, native=stored_native, ).dump(paths.manifest) @@ -2239,6 +2253,60 @@ def test_mint_from_replay_preserves_stored_native(self, invoke_cli, mocker, tmp_ # The from-replay path must not snapshot the hydrated slot at all. snapshot.assert_not_called() + def test_mint_from_replay_advances_diagnostic_version(self, invoke_cli, mocker, tmp_path): + """`mint --from-replay` re-stamps the manifest's diagnostic_version from the code version. + + A version bump followed by `mint --from-replay` re-couples the unchanged native bundle to + the new declared Diagnostic.version even though the native blobs are reused verbatim. + """ + from climate_ref_core.regression.manifest import Manifest, NativeEntry + from climate_ref_core.testing import TestCasePaths + + registry, diag, _tc = _make_case_mocks() + # The diagnostic code has been bumped past the recorded baseline. + diag.version = 2 + + case_dir = tmp_path / "td" / "test-diag" / "default" + case_dir.mkdir(parents=True) + (case_dir / "catalog.yaml").touch() + paths = TestCasePaths(root=case_dir) + stored_native = {"output.json": NativeEntry(sha256="aa" * 32, size=10)} + Manifest( + schema=2, + test_case_version=1, + diagnostic_version=1, + committed={"diagnostic.json": "00" * 32}, + native=stored_native, + ).dump(paths.manifest) + + store = MagicMock() + store.has.return_value = True + mocker.patch( + "climate_ref.provider_registry.ProviderRegistry.build_from_config", + return_value=registry, + ) + mocker.patch("climate_ref_core.testing.TestCasePaths.from_diagnostic", return_value=paths) + mocker.patch("climate_ref_core.testing.get_catalog_hash", return_value=None) + mocker.patch("climate_ref_core.regression.store.build_native_store", return_value=store) + mocker.patch("climate_ref.cli.test_cases.baselines.stage_materialise") + mocker.patch( + "climate_ref.cli.test_cases.baselines.stage_build", + return_value={"diagnostic.json": "00" * 32}, + ) + mocker.patch("climate_ref.cli.test_cases.baselines.promote_to_baseline") + + result = invoke_cli( + ["test-cases", "mint", "--provider", "example", "--diagnostic", "test-diag", "--from-replay"] + ) + assert result.exit_code == 0 + + written = Manifest.load(paths.manifest) + # The native is preserved, but diagnostic_version advances to the in-code version. + assert {k: v.sha256 for k, v in written.native.items()} == { + k: v.sha256 for k, v in stored_native.items() + } + assert written.diagnostic_version == 2 + def test_mint_from_replay_requires_minted_manifest(self, invoke_cli, mocker, tmp_path): """`mint --from-replay` fails when there is no existing minted native to replay from.""" from climate_ref_core.regression.store import LocalFilesystemStore @@ -2403,6 +2471,8 @@ def _setup( current_version=None, committed_content=None, native=None, + diagnostic_version=1, + code_diagnostic_version=1, ): """ Wire a single example test case for the gate. @@ -2444,6 +2514,7 @@ def _setup( mock_tc.name = "default" mock_spec = MagicMock(test_cases=[mock_tc]) mock_diag = MagicMock(slug="test-diag", test_data_spec=mock_spec) + mock_diag.version = code_diagnostic_version mock_diag.provider = MagicMock(slug="example") mock_provider = MagicMock(slug="example") mock_provider.diagnostics.return_value = [mock_diag] @@ -2463,6 +2534,7 @@ def _setup( Manifest( schema=SCHEMA_VERSION, test_case_version=current_version, + diagnostic_version=diagnostic_version, committed=committed_digests, native=native or {}, ).dump(paths.manifest) @@ -2482,7 +2554,7 @@ def _setup( return repo, paths, committed_digests @staticmethod - def _set_base(repo, version, committed, native=None): + def _set_base(repo, version, committed, native=None, diagnostic_version=1): """Configure ``repo.git.show`` to return a base manifest with these fields.""" import json as _json @@ -2493,6 +2565,7 @@ def _set_base(repo, version, committed, native=None): payload = { "schema": SCHEMA_VERSION, "test_case_version": version, + "diagnostic_version": diagnostic_version, "committed": committed, "native": {relpath: asdict(entry) for relpath, entry in (native or {}).items()}, } @@ -2679,6 +2752,28 @@ def test_corrupt_base_manifest_treated_as_seeding(self, invoke_cli, mocker, tmp_ assert result.exit_code == 0 assert "replay" in result.output + def test_schema_1_base_manifest_treated_as_seeding(self, invoke_cli, mocker, tmp_path): + # The transition window: git show returns a *valid-JSON* schema-1 base manifest (no + # diagnostic_version). The schema-2 loader rejects it, and the real ci_gate try/except + # must convert that into base_manifest=None (seeding) rather than crashing the gate. + import json as _json + + from climate_ref_core.regression.manifest import NativeEntry + + repo, _, digests = self._setup( + mocker, tmp_path, current_version=1, native={"data.nc": NativeEntry("a" * 64, 10)} + ) + repo.git.show.side_effect = None + repo.git.show.return_value = _json.dumps( + {"schema": 1, "test_case_version": 1, "committed": digests, "native": {}}, + indent=2, + sort_keys=True, + ) + + result = invoke_cli(["test-cases", "ci-gate"]) + assert result.exit_code == 0 + assert "replay" in result.output + def test_catalog_drift_fails(self, invoke_cli, mocker, tmp_path): # The manifest records a catalog_hash that the on-disk catalog no longer matches. from climate_ref_core.regression.manifest import Manifest @@ -2688,6 +2783,7 @@ def test_catalog_drift_fails(self, invoke_cli, mocker, tmp_path): Manifest( schema=manifest.schema, test_case_version=manifest.test_case_version, + diagnostic_version=manifest.diagnostic_version, committed=manifest.committed, native=manifest.native, catalog_hash="expected_hash", @@ -2697,3 +2793,202 @@ def test_catalog_drift_fails(self, invoke_cli, mocker, tmp_path): result = invoke_cli(["test-cases", "ci-gate"], expected_exit_code=1) assert result.exit_code == 1 assert "fail" in result.output + + def test_stale_diagnostic_version_fails(self, invoke_cli, mocker, tmp_path): + # The in-code Diagnostic.version is ahead of the recorded baseline: stale -> FAIL. + repo, _, digests = self._setup( + mocker, tmp_path, current_version=1, diagnostic_version=1, code_diagnostic_version=2 + ) + self._set_base(repo, 1, digests, diagnostic_version=1) + + result = invoke_cli(["test-cases", "ci-gate"], expected_exit_code=1) + assert result.exit_code == 1 + assert "fail" in result.output + + def test_matched_diagnostic_version_does_not_fail(self, invoke_cli, mocker, tmp_path): + # code == manifest.diagnostic_version: no new failure, falls through to normal logic. + repo, _, digests = self._setup( + mocker, tmp_path, current_version=1, diagnostic_version=2, code_diagnostic_version=2 + ) + self._set_base(repo, 1, digests, diagnostic_version=2) + + result = invoke_cli(["test-cases", "ci-gate"]) + assert result.exit_code == 0 + assert "skip" in result.output + + def test_ecs_blind_spot_closed_then_remint_recovers(self, invoke_cli, mocker, tmp_path): + """#671: an execution change (code version bumped) FAILs until the baseline is re-minted.""" + from climate_ref_core.regression.manifest import Manifest, NativeEntry + + native = {"data.nc": NativeEntry("a" * 64, 10)} + + # Code bumped to 2, baseline still records 1: stale -> FAIL. + repo, paths, digests = self._setup( + mocker, + tmp_path, + current_version=1, + diagnostic_version=1, + code_diagnostic_version=2, + native=native, + ) + self._set_base(repo, 1, digests, diagnostic_version=1, native=native) + stale = invoke_cli(["test-cases", "ci-gate"], expected_exit_code=1) + assert stale.exit_code == 1 + assert "fail" in stale.output + + # Simulate the re-mint: rewrite the HEAD manifest at diagnostic_version 2 (== code). + current = Manifest.load(paths.manifest) + Manifest( + schema=current.schema, + test_case_version=current.test_case_version, + diagnostic_version=2, + committed=current.committed, + native=current.native, + catalog_hash=current.catalog_hash, + ).dump(paths.manifest) + recovered = invoke_cli(["test-cases", "ci-gate"]) + assert recovered.exit_code == 0 + assert "fail" not in recovered.output + + def test_revert_green_path(self, invoke_cli, mocker, tmp_path): + """A legitimate revert (code v2->v1 + re-mint + tcv bump) has a GREEN gate exit.""" + from climate_ref_core.regression.manifest import NativeEntry + + native = {"data.nc": NativeEntry("a" * 64, 10)} + # HEAD: re-minted at diagnostic_version 1, tcv bumped to 2, committed re-authored. + repo, _, _ = self._setup( + mocker, + tmp_path, + current_version=2, + diagnostic_version=1, + code_diagnostic_version=1, + committed_content={"output.json": '{"reauthored": 1}\n'}, + native=native, + ) + # Base: the pre-revert baseline at diagnostic_version 2 with the old committed bundle. + self._set_base(repo, 1, {"output.json": "f" * 64}, native=native, diagnostic_version=2) + + result = invoke_cli(["test-cases", "ci-gate"]) + assert result.exit_code == 0 + # 2a passes on equality, 2b's carve-out lets the decrease through, the tcv bump REPLAYs. + assert "replay" in result.output + assert "fail" not in result.output + + +class TestMigrateManifestsCommand: + """Tests for the ``ref test-cases migrate-manifests`` verb.""" + + def _wire(self, mocker, tmp_path, *, code_version=1): + from climate_ref_core.testing import TestCasePaths + + registry, diag, _tc = _make_case_mocks() + diag.version = code_version + + case_dir = tmp_path / "td" / "test-diag" / "default" + case_dir.mkdir(parents=True) + paths = TestCasePaths(root=case_dir) + + mocker.patch( + "climate_ref.provider_registry.ProviderRegistry.build_from_config", + return_value=registry, + ) + mocker.patch("climate_ref_core.testing.TestCasePaths.from_diagnostic", return_value=paths) + return paths + + def test_migrate_help(self, invoke_cli): + result = invoke_cli(["test-cases", "migrate-manifests", "--help"]) + assert "diagnostic_version" in result.output or "schema" in result.output + + def test_migrate_stamps_code_version_and_bumps_schema(self, invoke_cli, mocker, tmp_path): + import json as _json + + paths = self._wire(mocker, tmp_path, code_version=2) + # A schema-1 manifest (no diagnostic_version) on disk, as it would exist pre-migration. + paths.manifest.write_text( + _json.dumps( + { + "schema": 1, + "test_case_version": 3, + "catalog_hash": "abc", + "committed": {"output.json": "a" * 64}, + "native": {}, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + result = invoke_cli(["test-cases", "migrate-manifests", "--provider", "example"]) + assert result.exit_code == 0 + + from climate_ref_core.regression.manifest import Manifest + + migrated = Manifest.load(paths.manifest) + assert migrated.schema == 2 + assert migrated.diagnostic_version == 2 # stamped from diag.version + assert migrated.test_case_version == 3 + assert migrated.catalog_hash == "abc" + assert migrated.committed == {"output.json": "a" * 64} + + def test_migrate_preserves_recorded_diagnostic_version(self, invoke_cli, mocker, tmp_path): + import json as _json + + # Code has advanced to version 3 but the committed manifest still records version 2: + # an authorised bump not yet re-minted, which the gate's staleness check fails until + # a re-mint. migrate-manifests must not paper over that by re-stamping the recorded + # value to the in-code version. + paths = self._wire(mocker, tmp_path, code_version=3) + paths.manifest.write_text( + _json.dumps( + { + "schema": 2, + "test_case_version": 1, + "diagnostic_version": 2, + "catalog_hash": None, + "committed": {"output.json": "a" * 64}, + "native": {}, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + result = invoke_cli(["test-cases", "migrate-manifests", "--provider", "example"]) + assert result.exit_code == 0 + + from climate_ref_core.regression.manifest import Manifest + + migrated = Manifest.load(paths.manifest) + assert migrated.diagnostic_version == 2 # preserved, NOT re-stamped to 3 + + def test_migrate_is_idempotent_and_byte_stable(self, invoke_cli, mocker, tmp_path): + import json as _json + + paths = self._wire(mocker, tmp_path, code_version=1) + paths.manifest.write_text( + _json.dumps( + { + "schema": 1, + "test_case_version": 1, + "committed": {}, + "native": {}, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + first = invoke_cli(["test-cases", "migrate-manifests", "--provider", "example"]) + assert first.exit_code == 0 + after_first = paths.manifest.read_bytes() + + # Re-running on an already-migrated manifest is a no-op (byte-identical). + second = invoke_cli(["test-cases", "migrate-manifests", "--provider", "example"]) + assert second.exit_code == 0 + assert paths.manifest.read_bytes() == after_first