From 6046bbe7c4be03f35f1a0e4864fb88b572be631d Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Sat, 25 Jul 2026 14:41:48 -0700 Subject: [PATCH 1/6] =?UTF-8?q?feat(analytics/eval):=20factor-evaluation?= =?UTF-8?q?=20contract=20v1.0=20=E2=80=94=20identity=20fields=20+=20basis-?= =?UTF-8?q?column=20guard=20(D5=20C3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adjudication core is upgraded, not rewritten. v0.9 could describe a factor and one evaluation of it but could not say WHICH information set the values were taken under — survivable while every run used one basis, not survivable since PR #79 started scoring the same factor on two, because two reports then carried the same verdict word for two different statistical claims. v1.0 adds exactly two identity fields and renders four declarations that were already mandatory on the factor side: * EvalConfig.view / EvalConfig.return_basis, validated as a LEGAL PAIRING at construction through data.availability_policy.require_legal_pairing (the D0 single source — the rule is delegated, not restated). A close-view factor scored on exec_to_exec is now a readable error instead of a convention. * the provenance box states requires / adjustment / overnight_boundary / lookback_depth as named rows rather than leaving them to vars(spec)'s reprs, and an EMPTY requires declaration is marked rather than rendered as a blank row. * analytics/eval/summary.py: any cross-basis verdict table REFUSES to render without the identity columns, and refuses a declared-but-blank one too (R24). Filling a missing basis in would be the silent degradation the project forbids; guessing it from the majority of rows is worse than refusing. WHAT DID NOT MOVE, and is now pinned by value rather than by prose: every VerdictThresholds default (min_abs_icir=0.30, min_incremental_abs_icir=0.15, min_monotonicity_spearman=0.0, the three-part sample gate). Design v3.2 §十 R24 forbids re-calibrating a bar inside the refactor; a threshold that moved here would make every verdict in this cycle uninterpretable, and would do it silently. The artifact drift this causes (2 keys in eval_config, 1 top-level key, 4 Markdown provenance rows — all additions, no deletions, no metric change) is registered in docs/factors/d5_runner_difference_catalogue.md §七之二 BEFORE the C5 reconciliation runs, so it cannot later be chased as a D5 regression. That section also states the consequence the plan did not: byte-for-byte equality of the 22 md/json artifacts is NOT available as a C5 pass condition, and treating it as one would read this upgrade as a regression. Mutation evidence (run, not claimed): * remove the _check_view_basis call -> 4 tests FAIL (rc=1), restored -> pass; * narrow REQUIRED_SUMMARY_COLUMNS to ("view",) -> the return_basis-omitted case and the blank-cell case FAIL, restored -> pass. Gates: pytest 2210 passed / ruff clean / 31 configs / phase0 ic 0.9600 annual 0.8408 unchanged / exec baseline --verify 77/77 at head 45c14aa / frozen panel mtimes untouched. --- analytics/eval/__init__.py | 16 ++ analytics/eval/config.py | 30 +++ analytics/eval/contract.py | 71 +++++++ analytics/eval/render.py | 31 +++ analytics/eval/summary.py | 97 +++++++++ .../factors/d5_runner_difference_catalogue.md | 22 ++ tests/test_eval_contract_v1.py | 199 ++++++++++++++++++ 7 files changed, 466 insertions(+) create mode 100644 analytics/eval/contract.py create mode 100644 analytics/eval/summary.py create mode 100644 tests/test_eval_contract_v1.py diff --git a/analytics/eval/__init__.py b/analytics/eval/__init__.py index 65e3b3c..9ca2a3b 100644 --- a/analytics/eval/__init__.py +++ b/analytics/eval/__init__.py @@ -19,6 +19,11 @@ """ from analytics.eval.config import EvalConfig +from analytics.eval.contract import ( + EVAL_CONTRACT_VERSION, + IDENTITY_FIELDS, + basis_identity_phrase, +) from analytics.eval.evaluator import EvalIR, FactorEvaluator from analytics.eval.ir import EvalContext, StandardEvalIR, build_eval_ir from analytics.eval.report import ( @@ -27,6 +32,11 @@ extract_verdict_inputs, ) from analytics.eval.standard import StandardFactorEvaluator +from analytics.eval.summary import ( + REQUIRED_SUMMARY_COLUMNS, + render_verdict_summary, + require_basis_columns, +) from analytics.eval.sections import ( MANDATORY_SECTIONS, VERDICT_KEYS, @@ -56,6 +66,12 @@ __all__ = [ "ADOPT", "AXIS_FAIL", + "EVAL_CONTRACT_VERSION", + "IDENTITY_FIELDS", + "REQUIRED_SUMMARY_COLUMNS", + "basis_identity_phrase", + "render_verdict_summary", + "require_basis_columns", "AXIS_INSUFFICIENT_DATA", "AXIS_NAMES", "AXIS_NOT_ASSESSED", diff --git a/analytics/eval/config.py b/analytics/eval/config.py index c9171c4..38e5e31 100644 --- a/analytics/eval/config.py +++ b/analytics/eval/config.py @@ -18,6 +18,7 @@ from dataclasses import dataclass from analytics.eval.verdict import VerdictThresholds +from data.availability_policy import ReturnBasis, View, require_legal_pairing # Tolerance for spotting the base (multiplier 1.0) cost scenario among floats. _BASE_COST = 1.0 @@ -65,6 +66,16 @@ class EvalConfig: n_factors_screened : multiple-testing background (how many factors were looked at to find this one). data_snapshot_id : data/cache version, for reproducibility. + view : (contract v1.0) the INFORMATION-SET view the factor values were taken + under — ``"decision"`` (14:50) or ``"close"``. Not decoration: a verdict + computed on one view is a different statistical claim from the same + verdict on the other, and before v1.0 a report could not say which it was. + return_basis : (contract v1.0) the forward-return basis the factor was scored + on — ``"exec_to_exec"`` (the 14:51 execution anchor) or + ``"close_to_close"``. VALIDATED AS A PAIR with ``view``: the two legal + pairings are decision<->exec_to_exec and close<->close_to_close, and + anything else raises here rather than being a doc convention (design + §1.4 mechanism 1, enforced through ``data.availability_policy``). success_criteria : the PRE-REGISTERED verdict bar (design §6, v0.6). A frozen :class:`~analytics.eval.verdict.VerdictThresholds` declared HERE, before ``evaluate`` runs, IS pre-registered by construction: you cannot tune the @@ -97,6 +108,9 @@ class EvalConfig: tuned: bool = False n_factors_screened: int | None = None data_snapshot_id: str | None = None + # -- contract v1.0 identity (see analytics/eval/contract.py) -------------- + view: str = View.DECISION.value + return_basis: str = ReturnBasis.EXEC_TO_EXEC.value success_criteria: VerdictThresholds | None = None def __post_init__(self) -> None: @@ -106,6 +120,7 @@ def __post_init__(self) -> None: self._check_costs() self._check_capacity() self._check_declared_sequences() + self._check_view_basis() self._check_success_criteria() # -- validators (enforcement layer #1) -------------------------------- @@ -247,6 +262,21 @@ def _check_capacity(self) -> None: f"{self.limit_feasibility!r}." ) + def _check_view_basis(self) -> None: + """Contract v1.0: the view x return-basis pairing, enforced at construction. + + Delegated to :func:`data.availability_policy.require_legal_pairing` — the + SINGLE source of the legal pairs (D0). Re-stating the rule here would be + the describe-the-check drift the project keeps paying for. The normalized + enum VALUES are written back so the exported record carries the canonical + spelling whatever the caller passed (enum member or string). + """ + resolved_view, resolved_basis = require_legal_pairing( + self.view, self.return_basis + ) + object.__setattr__(self, "view", resolved_view.value) + object.__setattr__(self, "return_basis", resolved_basis.value) + def _check_success_criteria(self) -> None: """The pre-registered bar, if supplied, must be a real VerdictThresholds. diff --git a/analytics/eval/contract.py b/analytics/eval/contract.py new file mode 100644 index 0000000..27e1408 --- /dev/null +++ b/analytics/eval/contract.py @@ -0,0 +1,71 @@ +"""The factor-evaluation contract's VERSION and its declared identity fields. + +The statistical adjudication core (three axes / asymmetric gate / "unknown never +convicts" / N_eff CI / exploratory capped at Watch) is NOT rewritten by the +factor-layer refactor — it is UPGRADED, and this module is the explicit version +statement that upgrade owes (the PR #74 precedent: a change to a frozen contract +is stated, not slipped in). + +CONTRACT v1.0 — WHAT CHANGED AND WHY +------------------------------------ +v0.9 (the eleven-factor loop's frozen contract) described a factor and one +evaluation of it, but it could not say WHICH INFORMATION SET the factor values +were taken under. That was survivable while every run used one basis; it stopped +being survivable when the same factor started being evaluated on two +(``close_to_close`` and ``exec_to_exec``, PR #79), because two reports then +carried the same verdict word for two different statistical claims. + +v1.0 adds exactly two identity fields to ``EvalConfig`` and renders four more off +the ``FactorSpec`` that D1 already made mandatory: + +* ``EvalConfig.view`` / ``EvalConfig.return_basis`` — the information-set view and + the forward-return basis, validated as a LEGAL PAIRING at construction time via + :func:`data.availability_policy.require_legal_pairing`. A close-view factor + scored on ``exec_to_exec`` returns is now a readable error, not a convention. +* the provenance box renders ``FactorSpec.requires`` / ``adjustment`` / + ``overnight_boundary`` / ``lookback_depth`` (contract v1.0/v1.1 of the FACTOR + spec, PR #86 / #91) as named rows instead of leaving them to the JSON's + ``vars(spec)`` dump. + +WHAT DID NOT CHANGE (and must not, in this step) +----------------------------------------------- +Every decision rule and every threshold. ``VerdictThresholds`` defaults +(``min_abs_icir=0.30``, ``min_incremental_abs_icir=0.15``, +``min_monotonicity_spearman=0.0``, the three-part sample gate) are carried over +BYTE-FOR-BYTE as the UNVALIDATED close-era defaults they have always been +(design v3.2 §十 R24). Re-calibrating a bar is a separate pre-registered run; a +refactor that quietly moved one would make every verdict in this cycle +uninterpretable. + +IMPACT SURFACE +-------------- +Two new keys in the exported ``eval_config`` block and one new top-level key +(``eval_contract_version``) in every report JSON, plus four new provenance rows in +the Markdown. No metric, no axis input and no threshold moves. The D5 C5 +reconciliation registers this as a named, expected artifact drift. +""" + +from __future__ import annotations + +#: The evaluation contract version this package implements. Bumped ONLY with a +#: written statement of what changed (this module's docstring is that statement). +EVAL_CONTRACT_VERSION = "1.0" + +#: The identity fields a v1.0 report must be able to state about itself. Named +#: here once so the config validator, the renderer and the cross-basis summary +#: guard all refer to ONE list instead of three spellings of it. +IDENTITY_FIELDS: tuple[str, ...] = ("view", "return_basis") + + +def basis_identity_phrase(view: str, return_basis: str) -> str: + """The ONE sentence that states a report's information set + return basis. + + Author-once (#76/#78/#82): the provenance row, the cross-basis summary header + and any prose that needs to say which basis a number belongs to COMPOSE this + string rather than each writing their own. A regex cannot assert that no other + sentence says this; "there is no other sentence" can. + """ + return f"view={view} x return_basis={return_basis}" + + +__all__ = ["EVAL_CONTRACT_VERSION", "IDENTITY_FIELDS", "basis_identity_phrase"] diff --git a/analytics/eval/render.py b/analytics/eval/render.py index a0d53a1..e461f40 100644 --- a/analytics/eval/render.py +++ b/analytics/eval/render.py @@ -21,6 +21,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING +from analytics.eval.contract import EVAL_CONTRACT_VERSION, basis_identity_phrase from analytics.eval.sections import MANDATORY_SECTIONS, Skipped from data.quality.report import clean_value, sanitize_text @@ -150,6 +151,10 @@ def report_to_dict(report: FactorEvalReport) -> dict: sections.append(dict(sorted(entry.items()))) return { "criteria_source": report.criteria_source, + # Contract v1.0 (analytics/eval/contract.py): stated in the machine-readable + # record so a stored verdict can be matched to the contract that produced it + # — the #74 lesson that a verdict is only interpretable against its contract. + "eval_contract_version": EVAL_CONTRACT_VERSION, "eval_config": sanitize_payload(vars(report.cfg)), "schema_version": report.SCHEMA_VERSION, "sections": sections, @@ -219,6 +224,21 @@ def render_report(report: FactorEvalReport, mandatory: tuple[str, ...]) -> str: return "\n".join(lines).rstrip() + "\n" +def _requires_row(spec) -> str: + """``spec.requires`` as ``source.field`` pairs, sorted, or an explicit marker. + + The JSON's ``vars(spec)`` dump renders these through ``clean_value``'s ``str()`` + fallback, i.e. as dataclass REPRs — readable by accident, not by design. The + provenance box states them as data (contract v1.0), and an EMPTY declaration is + marked rather than rendered as a blank row: "this factor declares no endpoint + input" is a fact a reader must not have to infer from whitespace. + """ + fields = tuple(spec.requires or ()) + if not fields: + return "(none declared)" + return ", ".join(sorted(f"{f.source}.{f.field}" for f in fields)) + + def _provenance_rows(report: FactorEvalReport) -> list[tuple[str, str]]: spec, cfg = report.spec, report.cfg rows: list[tuple[str, object]] = [ @@ -227,6 +247,17 @@ def _provenance_rows(report: FactorEvalReport) -> list[tuple[str, str]]: ("family", spec.family), ("hypothesis (expected IC sign, fixed pre-run)", f"{spec.expected_ic_sign:+d}"), ("horizon / return basis", f"h={spec.forward_return_horizon} / {spec.return_basis}"), + # -- contract v1.0 identity + factor declarations (D1 contract v1.0/v1.1) -- + ( + "evaluation contract", + f"v{EVAL_CONTRACT_VERSION}, {basis_identity_phrase(cfg.view, cfg.return_basis)}", + ), + ("requires (endpoint inputs)", _requires_row(spec)), + ( + "adjustment / overnight boundary", + f"{spec.adjustment} / {spec.overnight_boundary}", + ), + ("lookback depth (trailing trading days)", spec.lookback_depth), ("input fields", ", ".join(spec.input_fields)), ("price adjust / warm-up bars", f"{spec.price_adjust} / {spec.min_history_bars}"), ("universe", f"{cfg.universe} (PIT={cfg.universe_is_pit})"), diff --git a/analytics/eval/summary.py b/analytics/eval/summary.py new file mode 100644 index 0000000..2ae2ec6 --- /dev/null +++ b/analytics/eval/summary.py @@ -0,0 +1,97 @@ +"""Cross-basis verdict summaries: basis/view are NON-OMITTABLE columns (R24). + +A single report states its own information set and return basis in its provenance +box. A SUMMARY does not: a table listing eleven factors' verdicts side by side is +the surface where a close-era "Watch" and an exec-era "Watch" get read as the same +claim, because nothing on the row says they are not. Design v3.2 §3.6 R24 makes +the rule structural: any multi-verdict rendering carries ``basis`` and ``view`` +columns, and a caller that omits them gets a readable error instead of a table. + +This is the summary-level form of the #76 lesson ("a rendering that does not state +its provenance"), and it is deliberately a REFUSAL rather than a default: filling +in a missing basis would be the silent degradation the project forbids, and +guessing it from the majority of the rows is worse than refusing. + +Layering: pure formatting over plain mappings — no pandas, no qt, no store. The +row dicts come from whoever assembled them (a registry read, a directory of report +JSONs); this module only decides what a table is allowed to omit. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +from analytics.eval.contract import IDENTITY_FIELDS + +#: Columns every cross-basis summary MUST carry. Same tuple the contract declares +#: as a report's identity fields — one list, not two spellings (author once). +REQUIRED_SUMMARY_COLUMNS: tuple[str, ...] = IDENTITY_FIELDS + +_MISSING = "—" + + +def require_basis_columns(columns: Sequence[str]) -> tuple[str, ...]: + """Return ``columns`` as a tuple, or raise naming the omitted identity column. + + Split out from the renderer so a caller that builds a table some other way + (HTML, a notebook) can enforce the same rule without re-deriving it. + """ + resolved = tuple(str(c) for c in columns) + missing = [c for c in REQUIRED_SUMMARY_COLUMNS if c not in resolved] + if missing: + raise ValueError( + f"a cross-basis verdict summary must carry the identity column(s) " + f"{missing}: a verdict from one view/return-basis is a DIFFERENT " + f"statistical claim from the same word under another, and a table that " + f"omits the column invites them to be read as one (design v3.2 §3.6 " + f"R24). Add the column — the summary will not infer or default it." + ) + return resolved + + +def _cell(row: Mapping[str, object], column: str) -> str: + value = row.get(column, None) + return _MISSING if value is None else str(value) + + +def render_verdict_summary( + rows: Sequence[Mapping[str, object]], + *, + columns: Sequence[str], + title: str | None = None, +) -> str: + """A deterministic Markdown table of many verdicts, basis/view guaranteed present. + + ``columns`` fixes the column order (so a diff of two summaries is a diff of the + data). Every row must SUPPLY the identity columns as non-empty values: declaring + the column and then leaving it blank would satisfy the letter of R24 and defeat + its purpose, so it is rejected with the same readable error shape. + """ + resolved = require_basis_columns(columns) + for index, row in enumerate(rows): + blank = [ + c + for c in REQUIRED_SUMMARY_COLUMNS + if not str(row.get(c, "") or "").strip() + ] + if blank: + raise ValueError( + f"cross-basis summary row {index} ({row.get('factor_id', '?')!r}) " + f"leaves the identity column(s) {blank} empty. Declaring the column " + f"and not filling it is the same omission R24 forbids." + ) + lines: list[str] = [] + if title: + lines += [f"### {title}", ""] + lines.append("| " + " | ".join(resolved) + " |") + lines.append("|" + "|".join("---" for _ in resolved) + "|") + for row in rows: + lines.append("| " + " | ".join(_cell(row, c) for c in resolved) + " |") + return "\n".join(lines) + "\n" + + +__all__ = [ + "REQUIRED_SUMMARY_COLUMNS", + "render_verdict_summary", + "require_basis_columns", +] diff --git a/docs/factors/d5_runner_difference_catalogue.md b/docs/factors/d5_runner_difference_catalogue.md index 83dcdd9..c6c91e7 100644 --- a/docs/factors/d5_runner_difference_catalogue.md +++ b/docs/factors/d5_runner_difference_catalogue.md @@ -265,3 +265,25 @@ import 的 runner loader，即便统一 runner 已上线）、**要么显式记 > 新字段自动带进 JSON，而这四个字段是 PR #86（契约 v1.0）与 PR #91（v1.1 `lookback_depth`） > 加的。`requires` 经 `clean_value` 的 `str()` 兜底渲染成 **repr 字符串列表**。 > **属预期漂移，非回归**——但若不预先登记，会在对账时被当成 D5 引入的差异去追。 + +### 七之二、C3（评估契约 v1.0）引入的 artifact 漂移 —— 同样预先登记 + +C3 升级 `analytics/eval` 契约至 v1.0（版本陈述见 `analytics/eval/contract.py` 的模块 +docstring，按 #74 先例）。它改变 artifact 的**三处且仅三处**，全部为**新增**，零删除、 +零数值变化： + +| # | 位置 | 变化 | 成因 | +|---|---|---|---| +| 1 | JSON `eval_config` 块 | **+2 键** `view` / `return_basis` | `report_to_dict` 的 `sanitize_payload(vars(report.cfg))` 自动带上 `EvalConfig` 的新字段（与 §七 的 `spec` 16→20 完全同一机制） | +| 2 | JSON 顶层 | **+1 键** `eval_contract_version` | 显式写入：一个 verdict 只有对着产生它的契约版本才可解释（#74 的教训） | +| 3 | Markdown `## 0. Header & Provenance` | **+4 行** `evaluation contract` / `requires (endpoint inputs)` / `adjustment / overnight boundary` / `lookback depth ...` | R24 的身份字段 + D1 契约 v1.0/v1.1 的三个声明维，从 `vars(spec)` 的 repr 转述升级为具名行 | + +**没有变的**（`tests/test_eval_contract_v1.py` 逐值钉住）：`VerdictThresholds` 的**每一个** +默认门（`min_abs_icir=0.30` / `min_incremental_abs_icir=0.15` / +`min_monotonicity_spearman=0.0` / 三段样本门），以及三轴判决 / 非对称门 / +unknown-never-convicts / N_eff CI / exploratory 封顶 Watch 的全部规则。设计 §十 R24 明令 +**禁止**在被评判的 run 上调门；门槛标定要单独预注册 run。 + +⚠️ 因此 C5 对账在**逐字节**层面对 22 份 md/json 一定不成立，能成立的是 **IC / ICIR / +分位价差 / verdict 的值级对账**加上「JSON 差异恰为上表三项 + §七 的 spec 四键」这个 +**结构性**断言。把「逐字节相同」当作 C5 的通过条件会让人在这里误判为回归。 diff --git a/tests/test_eval_contract_v1.py b/tests/test_eval_contract_v1.py new file mode 100644 index 0000000..43a900d --- /dev/null +++ b/tests/test_eval_contract_v1.py @@ -0,0 +1,199 @@ +"""Factor-evaluation contract v1.0 (D5 C3): identity fields + basis-column guard. + +Three groups, matching what the upgrade is allowed and forbidden to do: + +1. IDENTITY — ``EvalConfig`` carries ``view`` / ``return_basis`` and enforces the + legal pairing at construction time (design §1.4 mechanism 1, D0's single + source), and the report states them. +2. NON-OMITTABLE BASIS COLUMN (R24) — a cross-basis summary that omits ``basis`` / + ``view`` REFUSES to render. +3. THE ADJUDICATION CORE DID NOT MOVE — every ``VerdictThresholds`` default is + pinned to its frozen close-era value. "Upgrade, do not rewrite" is only a claim + until a test says a moved threshold is a failure; §十 R24 forbids re-calibrating + a bar inside the refactor, so the bar is nailed down here rather than described. +""" + +from __future__ import annotations + +import pytest + +from analytics.eval import ( + EVAL_CONTRACT_VERSION, + EvalConfig, + VerdictThresholds, + basis_identity_phrase, + render_verdict_summary, + require_basis_columns, +) +from analytics.eval.render import _requires_row +from data.availability_policy import ReturnBasis, View + + +def _cfg(**overrides) -> EvalConfig: + base = dict( + universe="000905.SH", + universe_is_pit=True, + start="2021-07-01", + end="2026-06-30", + is_exploratory=True, + post_hoc_selected=False, + ) + base.update(overrides) + return EvalConfig(**base) + + +# --------------------------------------------------------------------------- # +# 1. identity fields +# --------------------------------------------------------------------------- # +def test_default_identity_is_the_decision_exec_pairing(): + """The contract's DEFAULT is the operating pairing, not the legacy one.""" + cfg = _cfg() + assert cfg.view == View.DECISION.value + assert cfg.return_basis == ReturnBasis.EXEC_TO_EXEC.value + + +def test_legacy_close_pairing_is_still_constructible(): + """close<->close_to_close stays legal: the legacy backtest path still uses it. + + Retiring close_to_close is the RUNNER's convergence (exec-only), not a ban at + the contract level — banning it here would break the legacy event model's + reports for no correctness gain. + """ + cfg = _cfg(view="close", return_basis="close_to_close") + assert (cfg.view, cfg.return_basis) == ("close", "close_to_close") + + +@pytest.mark.parametrize( + ("view", "basis"), + [("close", "exec_to_exec"), ("decision", "close_to_close")], +) +def test_illegal_pairing_is_a_construction_error(view, basis): + """The mixed pairings raise — the failure the D0 pairing rule exists to cause.""" + with pytest.raises(ValueError, match="illegal view/basis pairing"): + _cfg(view=view, return_basis=basis) + + +def test_unknown_view_is_rejected_not_passed_through(): + with pytest.raises(ValueError): + _cfg(view="intraday_9am", return_basis="exec_to_exec") + + +def test_enum_members_normalize_to_canonical_strings(): + """An enum member in, canonical VALUE out — the exported record has one spelling.""" + cfg = _cfg(view=View.DECISION, return_basis=ReturnBasis.EXEC_TO_EXEC) + assert cfg.view == "decision" and cfg.return_basis == "exec_to_exec" + assert isinstance(cfg.view, str) and type(cfg.view) is str + + +def test_identity_phrase_is_authored_once(): + """The provenance row COMPOSES the phrase rather than spelling it again.""" + cfg = _cfg() + phrase = basis_identity_phrase(cfg.view, cfg.return_basis) + assert phrase == "view=decision x return_basis=exec_to_exec" + + +# --------------------------------------------------------------------------- # +# 2. the non-omittable basis column (R24) +# --------------------------------------------------------------------------- # +_ROWS = [ + {"factor_id": "a_20", "verdict": "Watch", "return_basis": "exec_to_exec", "view": "decision"}, + { + "factor_id": "b_20", + "verdict": "Watch", + "return_basis": "close_to_close", + "view": "close", + }, +] + + +def test_summary_renders_when_basis_and_view_are_present(): + out = render_verdict_summary( + _ROWS, columns=("factor_id", "verdict", "return_basis", "view") + ) + assert out == ( + "| factor_id | verdict | return_basis | view |\n" + "|---|---|---|---|\n" + "| a_20 | Watch | exec_to_exec | decision |\n" + "| b_20 | Watch | close_to_close | close |\n" + ) + + +@pytest.mark.parametrize( + "columns", + [ + ("factor_id", "verdict"), # both omitted + ("factor_id", "verdict", "return_basis"), # view omitted + ("factor_id", "verdict", "view"), # basis omitted + ], +) +def test_summary_refuses_to_render_without_the_identity_columns(columns): + with pytest.raises(ValueError, match="identity column"): + render_verdict_summary(_ROWS, columns=columns) + + +def test_declaring_the_column_and_leaving_it_blank_is_the_same_omission(): + rows = [dict(_ROWS[0]), {**_ROWS[1], "return_basis": ""}] + with pytest.raises(ValueError, match="leaves the identity column"): + render_verdict_summary(rows, columns=("factor_id", "return_basis", "view")) + + +def test_require_basis_columns_is_reusable_on_its_own(): + assert require_basis_columns(["view", "return_basis", "x"]) == ( + "view", + "return_basis", + "x", + ) + with pytest.raises(ValueError): + require_basis_columns(["x"]) + + +# --------------------------------------------------------------------------- # +# 3. the adjudication core did NOT move (§十 R24: no re-calibration in a refactor) +# --------------------------------------------------------------------------- # +def test_every_verdict_threshold_default_is_the_frozen_close_era_value(): + """Pinned VALUES, not "unchanged" prose. + + These are the UNVALIDATED close-era defaults (v0.7-v0.9). Carrying them over + byte-for-byte is what makes this cycle's verdicts comparable to the eleven-factor + loop's; moving one inside a refactor would make every verdict in this cycle + uninterpretable, and would do it silently. + """ + t = VerdictThresholds() + assert t.min_rebalances == 12 + assert t.min_effective_samples == 24.0 + assert t.min_span_days == 365 + assert t.min_abs_icir == 0.30 + assert t.min_incremental_abs_icir == 0.15 + assert t.min_ic_win_rate == 0.55 + assert t.min_abs_nw_t == 2.0 + assert t.min_monotonicity_spearman == 0.0 + + +def test_contract_version_is_stated(): + assert EVAL_CONTRACT_VERSION == "1.0" + + +# --------------------------------------------------------------------------- # +# provenance rendering of the factor's declarations +# --------------------------------------------------------------------------- # +class _FakeField: + def __init__(self, field: str, source: str) -> None: + self.field, self.source = field, source + + +class _FakeSpec: + def __init__(self, requires) -> None: + self.requires = requires + + +def test_requires_row_renders_source_dot_field_sorted(): + spec = _FakeSpec( + (_FakeField("close", "market_daily"), _FakeField("volume", "stk_mins_1min")) + ) + assert _requires_row(spec) == "market_daily.close, stk_mins_1min.volume" + + +def test_empty_requires_is_marked_not_blank(): + """An empty declaration is stated, never rendered as an empty row.""" + assert _requires_row(_FakeSpec(())) == "(none declared)" + assert _requires_row(_FakeSpec(None)) == "(none declared)" From 0363f0cfb9ff151b764590eae3ce127ca48c0041 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Sat, 25 Jul 2026 14:42:19 -0700 Subject: [PATCH 2/6] feat(factors): per-day gate-attrition channel through the ONE engine (D5 C4 groundwork) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FOUND WHILE SCOPING C4, and the reason this is a separate commit: four of the eleven minute factors publish a scarcity / neutralization disclosure as an ADDED report section, and three of them build it from a per-symbol DAY-LEVEL frame the factor's compute function emits through its diagnostics_out sink. The old eval runners had that frame because they ran the compute themselves. The unified runner will not: values come from the D3 store through the D4/D4b materializer, and the store persists VALUES ONLY. The day counts are not in it and cannot be recovered from it. That left exactly two fates for the disclosure — recompute it, or drop it. Dropping a disclosure because the plumbing changed is the silent degradation this project forbids, so the channel exists and its cost is stated wherever it is paid: * MinuteStreamBinding gains an OPTIONAL per_symbol_diagnostics callable, kept SEPARATE from per_symbol on purpose — per_symbol is the value path D4b verified cell for cell, and a diagnostics request must not be able to change it. Asking re-runs the per-symbol math on bars already in memory: no extra I/O, no value effect. Bound for the three factors that publish a frame; the others answer has_minute_diagnostics() == False and return an EMPTY frame, which a caller must never read as "no attrition" (hence the separate, explicit predicate). * materialize_range(..., diagnostics=sink) — default None is the D4b path byte for byte. The sink is EMIT-WINDOW scoped, so the denominator keeps the meaning it has in the frozen artifacts ("the days this run scored"), not "the days that were loaded"; for a pooled factor only the TERMINAL saturation load is observed, since the expansion steps are the search, not the computation. A daily factor plus a sink is a readable error rather than an empty sink. * service.panel(..., diagnostics=sink) materializes the WHOLE requested range even on a full store hit, and refuses a shared sink for several factors (the frames carry no factor label, so a summarizer would silently reduce the mixture). COST, stated rather than buried: three of eleven factors can never be served from a warm store when their disclosure is wanted. That is a permanent charge the D5 plan did not price, and it is the honest half of the trade above. Mutation evidence (run, not claimed; each mutation asserted to hit its target): * _sink_diagnostics returns an empty frame -> the VACUITY pre-assertion in the values-unchanged test FAILS (rc=1) instead of the equality passing for the wrong reason — the sink-stayed-empty shape is what makes such a test unfailable; * drop the emit-window slice -> the scoping test FAILS with 576 loaded days; * remove the sink-forces-refill branch -> the warm-store test FAILS on calls == 0. Gates: pytest 2210 passed / ruff clean / 31 configs / phase0 ic 0.9600 annual 0.8408 unchanged / exec baseline --verify 77/77 / frozen panel mtimes untouched. --- factors/compute/minute/binding.py | 77 ++++++- factors/materialize.py | 55 ++++- factors/service.py | 27 ++- tests/test_minute_diagnostics_channel.py | 276 +++++++++++++++++++++++ 4 files changed, 425 insertions(+), 10 deletions(-) create mode 100644 tests/test_minute_diagnostics_channel.py diff --git a/factors/compute/minute/binding.py b/factors/compute/minute/binding.py index 169b397..f133bdc 100644 --- a/factors/compute/minute/binding.py +++ b/factors/compute/minute/binding.py @@ -164,6 +164,17 @@ class MinuteStreamBinding: per_symbol: Callable[[Factor, pd.DataFrame], pd.DataFrame] combine: Callable[[Factor, pd.DataFrame], pd.Series] + #: OPTIONAL per-symbol DAY-LEVEL diagnostics (D5): the gate-attrition frame a + #: factor's scarcity disclosure reduces. Only the factors that HAVE such a + #: disclosure set it; ``None`` means "this factor publishes no per-day + #: diagnostics", which the caller must state rather than infer. + #: + #: It is a SEPARATE callable rather than an extra ``per_symbol`` output on + #: purpose: ``per_symbol`` is the value path D4b verified cell-for-cell, and a + #: diagnostics request must not be able to change it. The cost is that asking + #: for diagnostics re-runs the per-symbol math on bars ALREADY in memory (no + #: extra I/O, no effect on any value); that cost is disclosed at the call site. + per_symbol_diagnostics: Callable[[Factor, pd.DataFrame], pd.DataFrame] | None = None def _pure_stream(compute_fn) -> MinuteStreamBinding: @@ -190,6 +201,41 @@ def _combine(factor: Factor, stats: pd.DataFrame) -> pd.Series: return MinuteStreamBinding(per_symbol=_per_symbol, combine=_combine) +def _pure_stream_with_diagnostics(compute_fn) -> MinuteStreamBinding: + """:func:`_pure_stream` plus the compute function's day-level diagnostics sink.""" + base = _pure_stream(compute_fn) + return MinuteStreamBinding( + per_symbol=base.per_symbol, + combine=base.combine, + per_symbol_diagnostics=_sink_diagnostics(compute_fn), + ) + + +def _sink_diagnostics(compute_fn) -> Callable[[Factor, pd.DataFrame], pd.DataFrame]: + """Per-symbol diagnostics via the compute function's own ``diagnostics_out`` sink. + + The three ridge/peak-family factors already expose the day-level gate-attrition + frame their scarcity disclosure reduces; this only routes it. The frame is + returned CONCATENATED and the symbol column is left exactly as the compute + function wrote it — the summarizers read it, and re-labelling here would be a + second place for the schema to drift. + """ + + def _call(factor: Factor, bars: pd.DataFrame) -> pd.DataFrame: + sink: list[pd.DataFrame] = [] + compute_fn( + bars, + lookback_days=factor.lookback_days, # type: ignore[attr-defined] + name=factor.name, + diagnostics_out=sink, + ) + if not sink: + return pd.DataFrame() + return pd.concat(sink, ignore_index=False) + + return _call + + def _amp_cut_per_symbol(factor: Factor, bars: pd.DataFrame) -> pd.DataFrame: """Steps 1-3 of ``intraday_amp_cut``: the ``(v_mean, v_std)`` intermediate.""" return compute_amp_cut_stats( @@ -219,9 +265,14 @@ def _amp_cut_combine(factor: Factor, stats: pd.DataFrame) -> pd.Series: ), PeakIntervalKurtosisFactor: _pure_stream(compute_peak_interval_kurtosis), ValleyRelativeVwapFactor: _pure_stream(compute_valley_relative_vwap), - ValleyRidgeVwapRatioFactor: _pure_stream(compute_valley_ridge_vwap_ratio), - RidgeMinuteReturnFactor: _pure_stream(compute_ridge_minute_return), - PeakRidgeAmountRatioFactor: _pure_stream(compute_peak_ridge_amount_ratio), + # The three factors that publish a per-day gate-attrition disclosure. + ValleyRidgeVwapRatioFactor: _pure_stream_with_diagnostics( + compute_valley_ridge_vwap_ratio + ), + RidgeMinuteReturnFactor: _pure_stream_with_diagnostics(compute_ridge_minute_return), + PeakRidgeAmountRatioFactor: _pure_stream_with_diagnostics( + compute_peak_ridge_amount_ratio + ), } #: The factors whose cross-sectional combine is NOT the identity — i.e. the ones @@ -388,6 +439,24 @@ def minute_stats_from_bars(factor: Factor, bars: pd.DataFrame) -> pd.DataFrame: return _stream_binding(factor).per_symbol(factor, bars) +def has_minute_diagnostics(factor: Factor) -> bool: + """True iff ``factor`` publishes a per-symbol day-level diagnostics frame.""" + return _stream_binding(factor).per_symbol_diagnostics is not None + + +def minute_diagnostics_from_bars(factor: Factor, bars: pd.DataFrame) -> pd.DataFrame: + """``factor``'s per-symbol day-level diagnostics frame (empty when it has none). + + Returns an EMPTY frame — not an error — for a factor without a diagnostics + binding, so a caller can ask uniformly; whether the factor HAS one is the + separate, explicit :func:`has_minute_diagnostics` question, and a caller that + needs the disclosure must check it rather than read an empty frame as "no + attrition". + """ + fn = _stream_binding(factor).per_symbol_diagnostics + return pd.DataFrame() if fn is None else fn(factor, bars) + + def combine_minute_stats(factor: Factor, stats: pd.DataFrame) -> pd.Series: """``factor``'s daily raw Series from the ASSEMBLED per-symbol intermediates. @@ -410,6 +479,8 @@ def is_cross_sectional_minute(factor: Factor) -> bool: "BindingFn", "MinuteStreamBinding", "combine_minute_stats", + "has_minute_diagnostics", + "minute_diagnostics_from_bars", "is_cross_sectional_minute", "is_minute_bound", "is_valid_day_pooled", diff --git a/factors/materialize.py b/factors/materialize.py index 8114466..8fa4ea0 100644 --- a/factors/materialize.py +++ b/factors/materialize.py @@ -64,6 +64,7 @@ combine_minute_stats, is_minute_bound, is_valid_day_pooled, + minute_diagnostics_from_bars, minute_raw_from_bars, minute_stats_from_bars, pooled_baseline_days, @@ -250,6 +251,7 @@ def materialize_range( sources: MaterializeSources, decision_cutoff: str = DEFAULT_DECISION_TIME, warmup: int | None = None, + diagnostics: list | None = None, ) -> pd.Series: """Compute ``factor``'s RAW values for ``view`` over ``[emit_start, emit_end]``. @@ -257,6 +259,13 @@ def materialize_range( applies the view lag, trims to ``warmup`` (= ``spec.lookback_depth``) trailing trading days before ``emit_start``, computes, and returns the emit-window rows. Single-date and batch calls give bit-identical values (design §3.5 P8). + + ``diagnostics``: OPTIONAL list the per-symbol day-level gate-attrition frames + are appended to (emit-window rows only), for the factors that publish a + scarcity disclosure. Default ``None`` = the D4b-verified value path, byte for + byte: nothing extra is computed and nothing is collected. Supplying a sink + NEVER changes a returned value — it re-runs the per-symbol math on bars already + in memory (no extra I/O) purely to observe the day-level counts. """ resolved_view = View(view) symbols = _requested_universe(symbols) # one engine-level normalization @@ -270,9 +279,15 @@ def materialize_range( if is_minute_factor(factor): raw = _materialize_minute( factor, resolved_view, symbols, load_start, emit_start, emit_end, w, - sources, decision_cutoff, + sources, decision_cutoff, diagnostics, ) else: + if diagnostics is not None: + raise ValueError( + f"{factor.name} is a daily factor; per-day minute diagnostics are " + f"only published by minute factors. Refusing to return an EMPTY " + f"sink, which a caller would read as 'no attrition'." + ) raw = _materialize_daily( factor, resolved_view, symbols, load_start, emit_start, emit_end, w, sources, ) @@ -300,7 +315,8 @@ def _materialize_daily( def _materialize_minute( - factor, view, symbols, load_start, emit_start, emit_end, warmup, sources, decision_cutoff, + factor, view, symbols, load_start, emit_start, emit_end, warmup, sources, + decision_cutoff, diagnostics=None, ) -> pd.Series: """STREAMING minute materialization (D4b): one symbol's bars at a time. @@ -347,12 +363,12 @@ def _materialize_minute( if pooled: stats = _pooled_symbol_stats( factor, view, sym, emit_start, emit_end, warmup, sources, - decision_cutoff, **pooled_kw, + decision_cutoff, diagnostics=diagnostics, **pooled_kw, ) else: stats = _bounded_symbol_stats( factor, view, sym, load_start, emit_start, emit_end, warmup, sources, - decision_cutoff, + decision_cutoff, diagnostics=diagnostics, ) if stats is None or stats.empty: continue @@ -385,8 +401,30 @@ def _symbol_bars(provider, symbol: str, start, end) -> pd.DataFrame: return bars if bool(mine.all()) else bars[mine] +def _collect_diagnostics(factor, work, emit_start, emit_end, diagnostics) -> None: + """Append ``factor``'s emit-window per-day diagnostics for ONE symbol, if asked. + + ``work`` is the SAME cutoff-filtered, trimmed frame the value path used, so the + counts describe the days the values were computed from — not a differently + loaded window. Restricting to the emit window matches the disclosure's meaning + ("the days this run scored"); the warm-up days ahead of it were loaded to make + those days computable and were never part of the reported denominator. + """ + if diagnostics is None: + return + frame = minute_diagnostics_from_bars(factor, work) + if frame.empty: + return + index = pd.DatetimeIndex(frame.index) + within = (index >= emit_start) & (index <= emit_end) + kept = frame[within] + if not kept.empty: + diagnostics.append(kept) + + def _bounded_symbol_stats( - factor, view, symbol, load_start, emit_start, emit_end, warmup, sources, decision_cutoff, + factor, view, symbol, load_start, emit_start, emit_end, warmup, sources, + decision_cutoff, *, diagnostics=None, ) -> pd.DataFrame | None: """One BOUNDED minute factor's per-symbol intermediate over the fixed window. @@ -403,12 +441,13 @@ def _bounded_symbol_stats( if view is View.DECISION: bars = minute_decision_cutoff(bars, decision_time=decision_cutoff) bars = _trim_minute(bars, emit_start, warmup) + _collect_diagnostics(factor, bars, emit_start, emit_end, diagnostics) return minute_stats_from_bars(factor, bars) def _pooled_symbol_stats( factor, view, symbol, emit_start, emit_end, warmup, sources, decision_cutoff, - *, floor, baseline_days, lookback_days, + *, floor, baseline_days, lookback_days, diagnostics=None, ) -> pd.DataFrame | None: """Saturation-expanding load for ONE symbol of a valid-day-pooled factor. @@ -482,6 +521,10 @@ def _pooled_symbol_stats( baseline_days=baseline_days, lookback_days=lookback_days, symbols=[symbol], ): + # Only the TERMINAL load is observed: the intermediate expansion steps + # are the search, not the computation, and reporting their day counts + # would inflate the disclosure with days that were re-derived deeper. + _collect_diagnostics(factor, work, emit_start, emit_end, diagnostics) return stats load_start = load_start - chunk # Never silently return an UNSATURATED result (red line #9: no silent diff --git a/factors/service.py b/factors/service.py index 8d0dc1b..3ad577d 100644 --- a/factors/service.py +++ b/factors/service.py @@ -80,6 +80,7 @@ def _ensure_coverage( sources: MaterializeSources, view: View, cutoff: str, + diagnostics: list | None = None, ) -> pd.Series: """Read-through: fill any date the store lacks for ``factor``, return the Series. @@ -89,6 +90,15 @@ def _ensure_coverage( (possibly empty). The per-date fill (dates = one) and the batch fill (dates = many) leave BIT-IDENTICAL stores because the materializer's trailing trim is warmup-consistent (design §3.5 P8). + + ``diagnostics``: when a sink is supplied the WHOLE requested range is + materialized even on a full store hit. The store persists VALUES, not the + day-level gate-attrition counts a scarcity disclosure reduces, so a warm store + can serve the values and cannot serve the disclosure. The two honest options + were "recompute" and "publish the report without the section"; the second is + the silent degradation this project forbids, so the cost is paid and stated. + Values are unaffected either way (the engine is deterministic in code + data, + which is exactly what the store key asserts). """ stored = store.read_valid(key, expected_fingerprint=fingerprint) have = ( @@ -96,7 +106,7 @@ def _ensure_coverage( if stored is not None and not stored.empty else pd.DatetimeIndex([]) ) - missing = dates[~dates.isin(have)] + missing = dates if diagnostics is not None else dates[~dates.isin(have)] if len(missing): fresh = materialize_range( factor, @@ -106,6 +116,7 @@ def _ensure_coverage( emit_end=missing.max(), sources=sources, decision_cutoff=cutoff, + diagnostics=diagnostics, ) if not fresh.empty: store.upsert(key, fresh, fingerprint=fingerprint) @@ -166,17 +177,30 @@ def panel( view: object = View.DECISION, basis: object = ReturnBasis.EXEC_TO_EXEC, params_by_id: Mapping[str, Mapping[str, object]] | None = None, + diagnostics: list | None = None, ) -> pd.DataFrame: """Raw factor values for ``factor_ids`` over ``decisions`` x ``universe``. Read-through: any decision date the store lacks is materialized once (batch) and appended. The view x basis pairing is enforced up front. + + ``diagnostics``: optional per-day gate-attrition sink for the ONE factor whose + scarcity disclosure needs it — see :func:`_ensure_coverage`. Requesting it for + several factors at once is refused rather than served: the frames from two + factors would land in one list with nothing to tell them apart, and a + summarizer would silently reduce the mixture. """ resolved_view, _ = require_legal_pairing(view, basis) factor_ids = list(factor_ids) symbols = [str(s) for s in universe] if not decisions: raise ValueError("panel() needs at least one DecisionPoint.") + if diagnostics is not None and len(factor_ids) != 1: + raise ValueError( + f"panel(diagnostics=...) takes exactly ONE factor_id (got " + f"{factor_ids}): the per-day diagnostics frames carry no factor label, " + f"so several factors' frames in one sink would be reduced together." + ) cutoff = _uniform_cutoff(decisions) dates = pd.DatetimeIndex(sorted({pd.Timestamp(d.date).normalize() for d in decisions})) @@ -188,6 +212,7 @@ def panel( series_by_id[fid] = _ensure_coverage( factor, key, fp, dates=dates, symbols=symbols, store=store, sources=sources, view=resolved_view, cutoff=cutoff, + diagnostics=diagnostics, ) return _assemble(factor_ids, series_by_id, dates, symbols) diff --git a/tests/test_minute_diagnostics_channel.py b/tests/test_minute_diagnostics_channel.py new file mode 100644 index 0000000..c571a94 --- /dev/null +++ b/tests/test_minute_diagnostics_channel.py @@ -0,0 +1,276 @@ +"""D5 C4 groundwork: the per-day gate-attrition channel (network-free). + +WHY THIS EXISTS. Four of the eleven minute factors publish a scarcity / +neutralization disclosure as an ADDED report section, and three of them build it +from a per-symbol DAY-LEVEL frame that the factor's compute function emits through +a ``diagnostics_out`` sink. The old eval runners had that frame because they ran +the compute themselves. The unified runner does not: values come from the D3 value +store through the D4/D4b materializer, and the store persists VALUES ONLY — the +day counts are not in it and cannot be recovered from it. + +So the disclosure had exactly two possible fates: recompute it, or drop it. Dropping +a mandatory-by-construction disclosure because the plumbing changed is the silent +degradation this project forbids, so the channel exists and its cost is stated at +every gate it passes. + +WHAT IS PINNED HERE +* asking for diagnostics NEVER changes a value (the D4b-verified path is untouched); +* the sink is emit-window-scoped, so the denominators mean the same thing they meant + in the frozen artifacts (the days the run scored), not "the days that were loaded"; +* a factor with no diagnostics binding yields an EMPTY frame and answers False to + ``has_minute_diagnostics`` — an empty frame must never be readable as "no attrition"; +* a daily factor + a sink is a readable error, not an empty sink; +* the service refuses a shared sink for several factors (the frames carry no factor + label) and, on a WARM store, still materializes when a sink is asked for. + +Every invariance claim below carries recorded mutation evidence, and each mutation +was first asserted to change its target — a sink that silently stayed empty would +make "values unchanged" pass for the wrong reason, which is the exact failure shape +this repo keeps catching in itself. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from data.availability_policy import View +from data.clean.intraday_schema import normalize_intraday_bars +from factors import registry as factor_registry +from factors.compute.minute.binding import ( + _MINUTE_STREAM_BINDINGS, + has_minute_diagnostics, + minute_diagnostics_from_bars, +) +from factors.materialize import MaterializeSources, materialize_range +from factors.service import DecisionPoint, panel +from factors.store.values import FactorValueStore + +#: Factors whose disclosure needs the day-level frame. Asserted as a SET against +#: the binding table below, so adding a fourth without a decision goes red. +DIAGNOSTIC_FACTOR_IDS = ( + "valley_ridge_vwap_ratio_20", + "ridge_minute_return_20", + "peak_ridge_amount_ratio_20", +) + +N_SYMBOLS = 12 +SYMBOLS = [f"6000{i:02d}.SH" for i in range(N_SYMBOLS)] +DATES = pd.bdate_range("2021-01-04", periods=48) +EMIT_START, EMIT_END = DATES[44], DATES[47] +BARS_PER_DAY = 238 + + +def _bars() -> pd.DataFrame: + rng = np.random.RandomState(31) + rows: list[tuple] = [] + for s in SYMBOLS: + for d in DATES: + base = pd.Timestamp(d) + pd.Timedelta("09:31:00") + price = 100.0 + rng.normal(0, 2) + for i in range(BARS_PER_DAY): + t = base + pd.Timedelta(minutes=i) + price += rng.normal(0, 0.05) + slot = 1e4 * (1.0 + 0.3 * np.sin(i / 12.0)) + erupt = 6.0 if (rng.rand() < 0.06) else 1.0 + vol = slot * erupt * (1.0 + 0.1 * rng.rand()) + w = 0.15 * price * (1.0 + (2.0 if erupt > 1 else 0.0)) * (0.5 + rng.rand()) + hi, lo = price + abs(w) * rng.rand(), price - abs(w) * rng.rand() + cl = lo + (hi - lo) * rng.rand() + rows.append((t, s, price, hi, lo, cl, vol, cl * vol)) + cols = ["time", "symbol", "open", "high", "low", "close", "volume", "amount"] + return normalize_intraday_bars(pd.DataFrame(rows, columns=cols), freq="1min") + + +BARS = _bars() + + +class Prov: + """Cache-free minute provider that honours ``symbols`` and counts its calls.""" + + def __init__(self) -> None: + self.calls = 0 + + def minute_bars(self, symbols, start, end): + self.calls += 1 + names = [str(s) for s in symbols] + if not names: + return BARS.iloc[0:0] + t = BARS.index.get_level_values("time") + sym = pd.Index(BARS.index.get_level_values("symbol")) + return BARS[ + (t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end)) & sym.isin(names) + ] + + def earliest_available(self, symbols): + return DATES[0] + + +def _materialize(factor_id, *, diagnostics=None, provider=None): + return materialize_range( + factor_registry.build(factor_id), + view=View.CLOSE, + symbols=list(SYMBOLS), + emit_start=EMIT_START, + emit_end=EMIT_END, + sources=MaterializeSources(minute=provider or Prov()), + diagnostics=diagnostics, + ) + + +# --------------------------------------------------------------------------- # +# which factors publish a frame at all +# --------------------------------------------------------------------------- # +def test_the_diagnostic_set_is_exactly_the_declared_three(): + """Derived from the binding table, not from this list — a fourth goes red.""" + declared = { + cls().name + for cls, binding in _MINUTE_STREAM_BINDINGS.items() + if binding.per_symbol_diagnostics is not None + } + assert declared == set(DIAGNOSTIC_FACTOR_IDS) + + +def test_a_factor_without_a_binding_answers_false_and_yields_an_empty_frame(): + """An empty frame is NOT "no attrition" — the caller must ask the other question.""" + factor = factor_registry.build("volume_peak_count_20") + assert has_minute_diagnostics(factor) is False + assert minute_diagnostics_from_bars(factor, BARS).empty + + +@pytest.mark.parametrize("factor_id", DIAGNOSTIC_FACTOR_IDS) +def test_a_declared_factor_answers_true(factor_id): + assert has_minute_diagnostics(factor_registry.build(factor_id)) is True + + +# --------------------------------------------------------------------------- # +# the values do not move +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("factor_id", DIAGNOSTIC_FACTOR_IDS) +def test_asking_for_diagnostics_does_not_move_a_single_value(factor_id): + """MUTATION EVIDENCE, and the pre-assertion that makes it non-vacuous. + + "Values unchanged with a sink" is trivially true if the sink is never filled — + that is the shape of an unfailable test. So the sink is asserted NON-EMPTY + first: the comparison only means something once the diagnostics path has + actually run. Recorded mutation: returning ``pd.DataFrame()`` from + ``_sink_diagnostics`` makes the pre-assertion FAIL (rc=1) rather than letting + the equality pass for the wrong reason. + """ + without = _materialize(factor_id) + sink: list[pd.DataFrame] = [] + with_sink = _materialize(factor_id, diagnostics=sink) + + assert sink, "the diagnostics sink stayed empty — the comparison below is vacuous" + assert sum(len(f) for f in sink) > 0 + pd.testing.assert_series_equal(without, with_sink, check_exact=True) + + +@pytest.mark.parametrize("factor_id", DIAGNOSTIC_FACTOR_IDS) +def test_the_sink_is_scoped_to_the_emit_window(factor_id): + """The denominator must mean "the days this run scored", not "the days loaded". + + The engine loads warm-up / saturation history far behind ``emit_start``; letting + those days into the frame would inflate ``symbol_days`` against the frozen + artifacts' figure for no reason a reader could see. + """ + sink: list[pd.DataFrame] = [] + _materialize(factor_id, diagnostics=sink) + index = pd.DatetimeIndex(pd.concat(sink).index) + assert index.min() >= EMIT_START and index.max() <= EMIT_END + + +@pytest.mark.parametrize("factor_id", DIAGNOSTIC_FACTOR_IDS) +def test_the_frames_carry_the_symbol_label_the_summarizers_read(factor_id): + sink: list[pd.DataFrame] = [] + _materialize(factor_id, diagnostics=sink) + frame = pd.concat(sink) + assert "symbol" in frame.columns + assert set(frame["symbol"]).issubset(set(SYMBOLS)) + assert "classifiable_bars" in frame.columns + + +def test_a_factor_without_diagnostics_leaves_the_sink_empty_rather_than_erroring(): + """Uniform call shape; whether a factor HAS a disclosure stays a separate question.""" + sink: list[pd.DataFrame] = [] + _materialize("volume_peak_count_20", diagnostics=sink) + assert sink == [] + + +def test_a_daily_factor_with_a_sink_is_a_readable_error(): + """Refusing beats returning an empty sink a caller would read as "no attrition".""" + + class DailyProv: + def daily_panel(self, symbols, start, end): + idx = pd.MultiIndex.from_product( + [pd.bdate_range("2021-01-04", periods=60), SYMBOLS[:2]], + names=["date", "symbol"], + ) + return pd.DataFrame({"close": np.linspace(10.0, 20.0, len(idx))}, index=idx) + + with pytest.raises(ValueError, match="daily factor"): + materialize_range( + factor_registry.build("volatility_20"), + view=View.CLOSE, + symbols=SYMBOLS[:2], + emit_start=EMIT_START, + emit_end=EMIT_END, + sources=MaterializeSources(daily=DailyProv()), + diagnostics=[], + ) + + +# --------------------------------------------------------------------------- # +# the service-level contract +# --------------------------------------------------------------------------- # +def _decisions(): + return [DecisionPoint(date=d) for d in [EMIT_START, EMIT_END]] + + +def test_a_shared_sink_for_several_factors_is_refused(tmp_path): + store = FactorValueStore(tmp_path / "store") + with pytest.raises(ValueError, match="exactly ONE factor_id"): + panel( + ["ridge_minute_return_20", "valley_ridge_vwap_ratio_20"], + SYMBOLS, + _decisions(), + store=store, + sources=MaterializeSources(minute=Prov()), + diagnostics=[], + ) + + +def test_a_warm_store_serves_values_but_a_sink_forces_the_recompute(tmp_path): + """The disclosed cost, pinned as behaviour rather than left in a docstring. + + Warm read with NO sink: zero provider calls (the store answers). Warm read WITH + a sink: the engine runs again, because the day counts are not in the store. + """ + store = FactorValueStore(tmp_path / "store") + src_cold = MaterializeSources(minute=Prov()) + cold = panel( + ["ridge_minute_return_20"], SYMBOLS, _decisions(), + store=store, sources=src_cold, + ) + assert src_cold.minute.calls > 0 + + warm_prov = Prov() + warm = panel( + ["ridge_minute_return_20"], SYMBOLS, _decisions(), + store=store, sources=MaterializeSources(minute=warm_prov), + ) + assert warm_prov.calls == 0, "a warm store must not re-read minute bars" + pd.testing.assert_frame_equal(cold, warm, check_exact=True) + + diag_prov = Prov() + sink: list[pd.DataFrame] = [] + with_diag = panel( + ["ridge_minute_return_20"], SYMBOLS, _decisions(), + store=store, sources=MaterializeSources(minute=diag_prov), + diagnostics=sink, + ) + assert diag_prov.calls > 0, "asking for diagnostics must re-run the engine" + assert sink, "the sink must be filled by that re-run" + # and the recompute reproduces the stored values exactly + pd.testing.assert_frame_equal(cold, with_diag, check_exact=True) From cc06d324a2b6de36c4dc32b544844acd559bfdfe Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Sat, 25 Jul 2026 14:58:03 -0700 Subject: [PATCH 3/6] fix(analytics/eval): the v1.0 identity default must describe the callers that omit it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RECORDING MY OWN DEFECT, one commit old. 6046bbe defaulted EvalConfig.view / return_basis to decision / exec_to_exec — the pairing the design is converging on, and the wrong default for the code that exists today. Each of the eleven live runners builds ONE EvalConfig (_build_eval_config) and hands the SAME object to its close_to_close reports AND, through qt.exec_basis_eval, to its exec ones. With the flattering default, every close artifact would have declared a basis it was not scored on — the describe-the-check drift of #76/#78/#82, introduced by the very field added to prevent it, in the same commit that added it. Nothing had re-run yet, so no artifact carries the false label; the fix is before the fact. A default's only job is to be TRUE for the callers that omit the field. Those callers are on the close basis, so the default is close / close_to_close, and the exec path states its identity EXPLICITLY — mirroring what qt.exec_basis_eval already does one line earlier for the SPEC (intraday_spec_variant). When the close runners retire in C6, flipping the default is a one-line change with no false statement left behind. Pinned: a test asserts the default IS the legacy pairing and says why, and a structural test asserts qt.exec_basis_eval restates the identity rather than inheriting the caller's. The identity phrase test now covers both pairings, so the author-once helper cannot be right for one and wrong for the other. --- analytics/eval/config.py | 16 ++++++++-- qt/exec_basis_eval.py | 11 ++++++- tests/test_eval_contract_v1.py | 53 +++++++++++++++++++++++++--------- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/analytics/eval/config.py b/analytics/eval/config.py index 38e5e31..5797351 100644 --- a/analytics/eval/config.py +++ b/analytics/eval/config.py @@ -109,8 +109,20 @@ class EvalConfig: n_factors_screened: int | None = None data_snapshot_id: str | None = None # -- contract v1.0 identity (see analytics/eval/contract.py) -------------- - view: str = View.DECISION.value - return_basis: str = ReturnBasis.EXEC_TO_EXEC.value + # + # THE DEFAULT IS THE LEGACY PAIRING, ON PURPOSE. A default's only job is to be + # TRUE for the callers that do not pass the field, and today those are the + # eleven close-basis runners: each builds ONE EvalConfig and hands it to BOTH + # its close_to_close reports and (through qt.exec_basis_eval) its exec ones. + # Defaulting to decision/exec_to_exec would make every close artifact state a + # basis it was not scored on — the describe-the-check drift of #76/#78/#82, + # introduced by the very field added to prevent it. The exec path sets the + # identity EXPLICITLY, exactly as it already derives the spec's exec twin + # (``intraday_spec_variant``), and the unified exec-only runner declares it too. + # When the close runners retire (D5 C6) this default becomes a one-line change + # with no false statement left behind. + view: str = View.CLOSE.value + return_basis: str = ReturnBasis.CLOSE_TO_CLOSE.value success_criteria: VerdictThresholds | None = None def __post_init__(self) -> None: diff --git a/qt/exec_basis_eval.py b/qt/exec_basis_eval.py index e829aa9..8091089 100644 --- a/qt/exec_basis_eval.py +++ b/qt/exec_basis_eval.py @@ -30,7 +30,7 @@ from __future__ import annotations import time -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path import pandas as pd @@ -41,6 +41,7 @@ FactorEvalReport, StandardFactorEvaluator, ) +from data.availability_policy import ReturnBasis, View from analytics.eval.figures import render_factor_dashboard from analytics.factor import forward_returns as _close_forward_returns from data.clean.schema import DATE_LEVEL @@ -210,6 +211,14 @@ def run_exec_basis_evaluation( started = time.monotonic() params = ExecBasisParams.from_config(cfg) exec_spec = intraday_spec_variant(spec, params) + # Contract v1.0: the EvalConfig's identity travels with the basis, exactly as + # the spec's does one line above. The caller hands ONE config to its close + # reports and to this function, so stating the exec identity HERE is what keeps + # both artifacts truthful about which information set they describe. The + # pairing is re-validated by EvalConfig's own construction check. + eval_cfg = replace( + eval_cfg, view=View.DECISION.value, return_basis=ReturnBasis.EXEC_TO_EXEC.value + ) horizon = spec.forward_return_horizon prices = build_exec_price_panel( diff --git a/tests/test_eval_contract_v1.py b/tests/test_eval_contract_v1.py index 43a900d..dd946f9 100644 --- a/tests/test_eval_contract_v1.py +++ b/tests/test_eval_contract_v1.py @@ -45,22 +45,40 @@ def _cfg(**overrides) -> EvalConfig: # --------------------------------------------------------------------------- # # 1. identity fields # --------------------------------------------------------------------------- # -def test_default_identity_is_the_decision_exec_pairing(): - """The contract's DEFAULT is the operating pairing, not the legacy one.""" +def test_the_default_describes_the_callers_that_do_not_pass_it(): + """The default is the LEGACY pairing, and that is the point. + + A default's only job is to be TRUE for the callers that omit the field. Today + those are the eleven close-basis runners, each of which builds ONE EvalConfig + and hands it to both its close_to_close reports AND (via qt.exec_basis_eval) its + exec ones. Defaulting to decision/exec_to_exec would have made every close + artifact state a basis it was not scored on — the exact describe-the-check drift + the field was added to prevent, introduced by the field itself. The exec path + sets the identity explicitly instead (see the test below). + """ cfg = _cfg() - assert cfg.view == View.DECISION.value - assert cfg.return_basis == ReturnBasis.EXEC_TO_EXEC.value + assert cfg.view == View.CLOSE.value + assert cfg.return_basis == ReturnBasis.CLOSE_TO_CLOSE.value + +def test_the_exec_pairing_is_constructible_and_is_what_the_exec_path_declares(): + cfg = _cfg(view="decision", return_basis="exec_to_exec") + assert (cfg.view, cfg.return_basis) == ("decision", "exec_to_exec") -def test_legacy_close_pairing_is_still_constructible(): - """close<->close_to_close stays legal: the legacy backtest path still uses it. - Retiring close_to_close is the RUNNER's convergence (exec-only), not a ban at - the contract level — banning it here would break the legacy event model's - reports for no correctness gain. +def test_the_exec_basis_module_restates_the_identity_it_scores_on(): + """qt.exec_basis_eval must not inherit the caller's close-basis identity. + + Structural rather than descriptive: the module derives the exec twin of the + SPEC one line earlier, and the config's identity has to travel the same way or + the exec artifact would carry the close label. """ - cfg = _cfg(view="close", return_basis="close_to_close") - assert (cfg.view, cfg.return_basis) == ("close", "close_to_close") + import inspect + + from qt import exec_basis_eval + + source = inspect.getsource(exec_basis_eval.run_exec_basis_evaluation) + assert "replace(" in source and "EXEC_TO_EXEC" in source and "DECISION" in source @pytest.mark.parametrize( @@ -87,9 +105,16 @@ def test_enum_members_normalize_to_canonical_strings(): def test_identity_phrase_is_authored_once(): """The provenance row COMPOSES the phrase rather than spelling it again.""" - cfg = _cfg() - phrase = basis_identity_phrase(cfg.view, cfg.return_basis) - assert phrase == "view=decision x return_basis=exec_to_exec" + cfg = _cfg(view="decision", return_basis="exec_to_exec") + assert ( + basis_identity_phrase(cfg.view, cfg.return_basis) + == "view=decision x return_basis=exec_to_exec" + ) + legacy = _cfg() + assert ( + basis_identity_phrase(legacy.view, legacy.return_basis) + == "view=close x return_basis=close_to_close" + ) # --------------------------------------------------------------------------- # From c6c03af77aa19e73a1de91bdcae1c101f79e9f4b Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Sat, 25 Jul 2026 14:59:11 -0700 Subject: [PATCH 4/6] test(factors): pin the 14:50 visibility property; record jump_amount_corr as the one exception MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FOUND BY MEASURING BEFORE COMMITTING TO THE MULTI-HOUR C5 RUN. qt/panel_leg_probe.py compares the D4/D4b materializer's values on the real evaluation plane against the D1 frozen panel — the fourth leg's comparison, taken as a probe first (the D5a discipline that stopped D5 being started blind). The expectation was a small, front-loaded warm-up difference. jump_amount_corr_20 came back with 1,155,500 of 1,158,382 both-finite cells differing, 931,394 of them AFTER year one where warm-up cannot be the explanation, max relative difference 1.99 on a correlation bounded in [-1, 1]. CHASED RATHER THAN EXPLAINED AWAY, and the cause is not the engine: * reproducing the OLD runner's exact read + compute for one symbol matches the frozen panel EXACTLY (1210 cells, max |diff| 0.000e+00) — the baseline is reproducible and the comparison target is right; * the same symbol under the new CLOSE view differs in 19 of 1210 cells (the warm-up difference that WAS expected); under the DECISION view, all 1210 differ; * so the whole difference is the 14:50 truncation the decision view applies — and compute_jump_amount_corr is the ONLY one of the eleven with no decision-time truncation of its own (zero decision_time / prepare_visible_minute_bars references in its module; the other ten have 4-10 each), so the old runners handed it 09:30-15:00 of day d. THE DECISIVE TEST, now committed. Perturb ONLY the bars strictly after the cutoff, assert every pre-cutoff row byte-identical, ask whether the day-d value moves. On the real cache (12 CSI500 names, 2021-07..2021-12, 361,500 bars, 4.6% post-cutoff): nine factors moved 0 cells at max |diff| 0.000e+00; jump_amount_corr_20 moved 1,477 of 1,477 at max |diff| 1.276. The committed test reproduces the same split on synthetic bars with a realistic 09:31-11:30 / 13:01-15:00 session (a session ending before 14:50 would make the perturbation a no-op — refused by a pre-assertion). WHAT IT MEANS, stated and not fixed. Under exec_to_exec — the only basis the D5/D7 contract keeps — the entry anchor for date d is d's 14:51 execution bar, so a value built from d's 14:50-15:00 bars uses information from after its own entry. PR #79's merge-time self-check asserted the opposite in general ("the factor is known at 14:50"); for this one factor that is false, and #79's review never returned. Its exec report also DECLARES decision_cutoff=14:50 (via intraday_spec_variant), which its values do not honour. On close_to_close, entry is close(d) and using day d is the conventional, already-declared-optimistic boundary, not a new defect. Flagged, not repaired, per the D5a precedent for found defects: truncating it would change a published factor's values and could move its verdict — a research decision with a pre-registered hypothesis attached, not a refactor's business (§〇). The exception is asserted POSITIVELY, so a later "fix" turns the test red and forces the call to be made deliberately rather than noticed afterwards. CONSEQUENCE FOR C5 the plan did not anticipate: for this factor the frozen D1 panel and the new decision-view engine differ wholesale, and the correct engine is the NEW one. The fourth leg's "relative <= 1e-12" cannot be the pass condition here; the attribution is "the baseline had no 14:50 truncation for this factor", and making the new engine reproduce the baseline would be reproducing the leak. --- qt/panel_leg_probe.py | 196 +++++++++++++++++++++++ tests/test_decision_cutoff_visibility.py | 164 +++++++++++++++++++ 2 files changed, 360 insertions(+) create mode 100644 qt/panel_leg_probe.py create mode 100644 tests/test_decision_cutoff_visibility.py diff --git a/qt/panel_leg_probe.py b/qt/panel_leg_probe.py new file mode 100644 index 0000000..b6d8cc9 --- /dev/null +++ b/qt/panel_leg_probe.py @@ -0,0 +1,196 @@ +"""D5 C5 fourth-leg PROBE: the new engine's panel vs the frozen D1 baseline. + +Not the acceptance run — a MEASUREMENT taken before committing to one, in the same +spirit as ``qt.saturation_probe`` (which is why the multi-hour D5 run was not +started blind). It answers ONE question the C5 plan needs an answer to first: + + how big, and of what shape, is the difference between the values the D4/D4b + materializer produces on the real evaluation plane and the D1 frozen panel the + fourth leg compares against? + +The question matters because the two are NOT expected to agree cell for cell, and +the reason is structural rather than numerical. The frozen panel was produced by the +old runners' loaders, which read exactly ``[data.start, data.end]`` and did NO +warm-up extension (``docs/factors/d5_runner_difference_catalogue.md`` §四). The +materializer loads ``lookback_depth`` trailing trading days of warm-up for a bounded +factor and expands to saturation for a valid-day-pooled one. So the early part of +the window is structurally under-warmed in the BASELINE and correctly warmed in the +new engine, and the fourth leg's "relative <= 1e-12" phrasing cannot apply to it. + +What this probe reports, per factor: + +* how many cells differ, and the NaN-set changes split by direction (NaN -> finite is + the warm-up being filled in; finite -> NaN would be the alarming direction); +* the difference profile BY MONTH — the warm-up explanation predicts a front-loaded + profile decaying to zero, and a difference that does NOT decay is a different + animal that needs its own attribution; +* the residual on the cells both sides call finite OUTSIDE the early region, which is + where the "float re-association only" claim can actually be tested. + +Cache-only (``stk_mins_live_calls`` is structurally 0: the store read has no fetch +closure). Resumable: results are appended per factor to ``--out`` as JSON lines and a +re-run skips factors already present unless ``--force``. + +Run (needs the gitignored ``artifacts/`` tree — see +``docs/factors/d5_exec_baseline_freeze.md`` for the symlink note): +``python -m qt.panel_leg_probe --factors jump_amount_corr_20`` +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +from data.availability_policy import View +from factors import registry as factor_registry +from factors.materialize import MaterializeSources, materialize_range +from qt.factor_hotpath_smoke import CacheMinuteProvider + +DEFAULT_CACHE_ROOT = "artifacts/cache/tushare/v1" +DEFAULT_PANEL_DIR = "artifacts/refactor_baseline/panels" +EVAL_START = pd.Timestamp("2021-07-01") +EVAL_END = pd.Timestamp("2026-06-30") + + +def frozen_panel(panel_dir: str, factor_id: str) -> pd.Series: + """The frozen D1 baseline as a ``MultiIndex(date, symbol)`` Series.""" + frame = pd.read_parquet(Path(panel_dir) / f"{factor_id}.parquet") + return ( + frame.set_index(["date", "symbol"])[factor_id] + .sort_index(kind="mergesort") + ) + + +def compare(factor_id: str, symbols: list[str], cache_root: str, panel_dir: str) -> dict: + """Materialize ``factor_id`` on the real plane and profile the difference.""" + base = frozen_panel(panel_dir, factor_id) + provider = CacheMinuteProvider(cache_root) + started = time.monotonic() + fresh = materialize_range( + factor_registry.build(factor_id), + view=View.DECISION, + symbols=symbols, + emit_start=EVAL_START, + emit_end=EVAL_END, + sources=MaterializeSources(minute=provider), + ).sort_index(kind="mergesort") + wall = time.monotonic() - started + + joined = pd.DataFrame({"base": base}).join( + pd.DataFrame({"fresh": fresh}), how="outer" + ) + b, f = joined["base"].to_numpy(), joined["fresh"].to_numpy() + b_nan, f_nan = np.isnan(b), np.isnan(f) + both = ~b_nan & ~f_nan + filled = b_nan & ~f_nan # warm-up filled in (the expected direction) + lost = ~b_nan & f_nan # the alarming direction + # Relative difference on the cells both sides call finite. + denom = np.maximum(np.abs(b), np.abs(f)) + rel = np.zeros(len(b)) + with np.errstate(invalid="ignore", divide="ignore"): + rel[both] = np.where( + denom[both] > 0, np.abs(b[both] - f[both]) / denom[both], 0.0 + ) + differing = both & (rel > 1e-12) + + months = pd.PeriodIndex( + joined.index.get_level_values("date"), freq="M" + ).astype(str) + by_month: dict[str, dict[str, int]] = {} + for label in sorted(set(months)): + m = months == label + entry = { + "filled": int((filled & m).sum()), + "lost": int((lost & m).sum()), + "differing_finite": int((differing & m).sum()), + } + if any(entry.values()): + by_month[label] = entry + + # The residual OUTSIDE the first year, where warm-up cannot be the explanation. + late = joined.index.get_level_values("date") >= EVAL_START + pd.DateOffset(years=1) + late_both = both & late + return { + "factor": factor_id, + "wall_seconds": round(wall, 1), + "live_calls": provider.live_calls, + "rows_base": int(len(base)), + "rows_fresh": int(len(fresh)), + "rows_union": int(len(joined)), + "both_finite": int(both.sum()), + "nan_filled": int(filled.sum()), + "nan_lost": int(lost.sum()), + "differing_finite": int(differing.sum()), + "max_rel_diff": float(rel[both].max()) if both.any() else None, + "max_rel_diff_after_year1": ( + float(rel[late_both].max()) if late_both.any() else None + ), + "differing_finite_after_year1": int((differing & late).sum()), + "nan_filled_after_year1": int((filled & late).sum()), + "by_month": by_month, + } + + +def _done(out_path: Path) -> set[str]: + if not out_path.exists(): + return set() + return { + json.loads(line)["factor"] + for line in out_path.read_text().splitlines() + if line.strip() + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--cache-root", default=DEFAULT_CACHE_ROOT) + parser.add_argument("--panel-dir", default=DEFAULT_PANEL_DIR) + parser.add_argument( + "--symbols-file", + required=True, + help="one symbol per line — the REQUESTED universe, which must be the " + "runner's (996 for the CSI500 plane), NOT the frozen panel's symbol list " + "(995: 300114.SZ has no minute bars and never entered the panel). The " + "distinction is load-bearing for the cross-sectional factor.", + ) + parser.add_argument("--factors", nargs="+", required=True) + parser.add_argument("--out", default="panel_leg_probe.jsonl") + parser.add_argument("--force", action="store_true") + args = parser.parse_args(argv) + + symbols = [ + s.strip() for s in Path(args.symbols_file).read_text().splitlines() if s.strip() + ] + out_path = Path(args.out) + out_path.parent.mkdir(parents=True, exist_ok=True) + done = set() if args.force else _done(out_path) + print(f"universe: {len(symbols)} symbols | emit {EVAL_START.date()}..{EVAL_END.date()}") + for factor_id in args.factors: + if factor_id in done: + print(f"{factor_id}: SKIP (already in {out_path})") + continue + print(f"{factor_id}: running...", flush=True) + record = compare(factor_id, symbols, args.cache_root, args.panel_dir) + with out_path.open("a") as fh: + fh.write(json.dumps(record) + "\n") + print( + f"{factor_id}: both_finite={record['both_finite']:,} " + f"NaN->finite={record['nan_filled']:,} finite->NaN={record['nan_lost']:,} " + f"differing_finite={record['differing_finite']:,} " + f"max_rel={record['max_rel_diff']} " + f"| after year 1: differing={record['differing_finite_after_year1']:,} " + f"filled={record['nan_filled_after_year1']:,} " + f"max_rel={record['max_rel_diff_after_year1']} " + f"| wall={record['wall_seconds']}s live_calls={record['live_calls']}", + flush=True, + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_decision_cutoff_visibility.py b/tests/test_decision_cutoff_visibility.py new file mode 100644 index 0000000..06f943d --- /dev/null +++ b/tests/test_decision_cutoff_visibility.py @@ -0,0 +1,164 @@ +"""Does a minute factor's day-d value depend on bars AFTER the 14:50 cutoff? + +THE PROPERTY. Under the ``exec_to_exec`` basis the entry anchor for date ``d`` is +``d``'s 14:51 execution bar (``qt.exec_forward_returns``). A factor value at ``d`` +may therefore only use information visible at 14:50; anything later is information +from after its own entry. The D0 policy table states the same rule for the minute +endpoint (``available_time <= d 14:50``), and the D4 materializer enforces it in the +DECISION view — but that enforcement arrived with the materializer, and the eleven +factors' published values were produced by the old runners, which hand the compute +functions FULL-DAY bars and rely on each compute truncating for itself. + +THE TEST. Perturb ONLY the bars strictly after the cutoff, assert every pre-cutoff +row is byte-identical (so a failure cannot be blamed on the fixture), and ask +whether the value moves. This is the "perturb the future -> value unchanged" shape +the project already uses, aimed at the WITHIN-DAY boundary rather than the +across-day one. + +RESULT, ENCODED RATHER THAN DESCRIBED. Nine of the ten bars-bound minute factors are +clean. ``jump_amount_corr_20`` is not: its compute is the ONLY one of the eleven that +applies no decision-time truncation of its own (grep: zero ``decision_time`` / +``prepare_visible_minute_bars`` references in its module), so under the old runners +it saw 09:30-15:00 of day d. + +That is recorded here as a KNOWN, NAMED exception rather than fixed. Truncating it +would change a published factor's values and could move its verdict — a definition +-affecting research decision, not a refactor's business (design v3.2 §〇). The +exception is asserted POSITIVELY, so the day someone changes it, this test goes red +and the change has to be made deliberately instead of noticed later. + +WHY THE EXISTING SUITE DID NOT CATCH IT — and why that is not a contradiction. +``tests/test_jump_amount_corr_factor.py::test_pit_perturbing_future_bars_does_not_ +change_factor_at_d`` is correct and passes: it perturbs day ``d+1`` and checks day +``d``, i.e. the ACROSS-DAY boundary. Its fixture sessions are six bars long, so a +14:50 boundary does not exist in them at all. The WITHIN-DAY boundary is a different +property, it only became load-bearing when PR #79 moved the entry anchor from +``close(d)`` to ``14:51(d)``, and nothing tested it. Two boundaries, two tests. + +Measured on the real cache the same way (12 CSI500 names, 2021-07..2021-12, 361,500 +bars, 4.6% of them post-cutoff): nine factors moved 0 cells; ``jump_amount_corr_20`` +moved 1,477 of 1,477 with max |diff| = 1.276 on a correlation bounded in [-1, 1]. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from data.clean.intraday_schema import normalize_intraday_bars +from factors import registry as factor_registry +from factors.compute.minute.binding import _MINUTE_STREAM_BINDINGS, minute_raw_from_bars + +CUTOFF = "14:50:00" + +#: The one factor whose value is NOT decision-cutoff-safe on its own. Named, not +#: inferred — see the module docstring for why it is recorded instead of fixed. +KNOWN_POST_CUTOFF_DEPENDENT = "jump_amount_corr_20" + +SYMBOLS = [f"6000{i:02d}.SH" for i in range(12)] +DATES = pd.bdate_range("2021-01-04", periods=40) + + +def _session_times(day: pd.Timestamp) -> list[pd.Timestamp]: + """A realistic A-share session: 09:31-11:30 and 13:01-15:00 (240 one-minute bars). + + The full afternoon matters: a session that stopped before 14:50 would have no + post-cutoff bars at all and the perturbation below would be a no-op — the + unfailable-test shape this repo keeps catching. The pre-assertion in + :func:`_poison` refuses that fixture outright. + """ + morning = pd.date_range(day + pd.Timedelta("09:31:00"), periods=120, freq="1min") + afternoon = pd.date_range(day + pd.Timedelta("13:01:00"), periods=120, freq="1min") + return list(morning) + list(afternoon) + + +def _bars() -> pd.DataFrame: + rng = np.random.RandomState(5) + rows: list[tuple] = [] + for s in SYMBOLS: + for d in DATES: + price = 100.0 + rng.normal(0, 2) + for i, t in enumerate(_session_times(d)): + price += rng.normal(0, 0.05) + slot = 1e4 * (1.0 + 0.3 * np.sin(i / 12.0)) + erupt = 6.0 if (rng.rand() < 0.06) else 1.0 + vol = slot * erupt * (1.0 + 0.1 * rng.rand()) + w = 0.15 * price * (1.0 + (2.0 if erupt > 1 else 0.0)) * (0.5 + rng.rand()) + hi, lo = price + abs(w) * rng.rand(), price - abs(w) * rng.rand() + cl = lo + (hi - lo) * rng.rand() + rows.append((t, s, price, hi, lo, cl, vol, cl * vol)) + cols = ["time", "symbol", "open", "high", "low", "close", "volume", "amount"] + return normalize_intraday_bars(pd.DataFrame(rows, columns=cols), freq="1min") + + +BARS = _bars() + + +def _poison(bars: pd.DataFrame) -> pd.DataFrame: + """Scale prices/volumes of the POST-cutoff bars only; leave the rest untouched. + + Returns the perturbed frame after asserting BOTH halves of what makes the test + meaningful: the post-cutoff rows really changed, and the pre-cutoff rows did not. + """ + times = pd.DatetimeIndex(bars.index.get_level_values("time")) + after = bars["available_time"].to_numpy() > ( + times.normalize() + pd.Timedelta(CUTOFF) + ).to_numpy() + assert after.any(), "fixture has no post-cutoff bars — the perturbation is a no-op" + rng = np.random.RandomState(11) + out = bars.copy() + for col in ("open", "high", "low", "close"): + out.loc[after, col] = out.loc[after, col] * (1.0 + 0.5 * rng.rand(int(after.sum()))) + for col in ("volume", "amount"): + out.loc[after, col] = out.loc[after, col] * 7.0 + assert not out.loc[after].equals(bars.loc[after]), "perturbation changed nothing" + pd.testing.assert_frame_equal(out.loc[~after], bars.loc[~after], check_exact=True) + return out + + +def _moved_cells(factor_id: str) -> tuple[int, int, float]: + factor = factor_registry.build(factor_id) + before = minute_raw_from_bars(factor, BARS) + after = minute_raw_from_bars(factor, _poison(BARS)) + joined = pd.DataFrame({"a": before, "b": after}).dropna() + if joined.empty: + return 0, 0, 0.0 + diff = (joined["a"] - joined["b"]).abs() + return len(joined), int((diff > 1e-12).sum()), float(diff.max()) + + +BOUND_FACTOR_IDS = tuple(sorted(cls().name for cls in _MINUTE_STREAM_BINDINGS)) +CLEAN_FACTOR_IDS = tuple(f for f in BOUND_FACTOR_IDS if f != KNOWN_POST_CUTOFF_DEPENDENT) + + +@pytest.mark.parametrize("factor_id", CLEAN_FACTOR_IDS) +def test_value_does_not_depend_on_bars_after_the_decision_cutoff(factor_id): + """The nine that truncate for themselves: perturbing the future moves nothing.""" + cells, moved, worst = _moved_cells(factor_id) + assert cells > 0, f"{factor_id}: no comparable cells — the test would be vacuous" + assert moved == 0, ( + f"{factor_id} moved {moved}/{cells} values (max |diff| {worst:.3e}) when ONLY " + f"post-{CUTOFF} bars changed: its day-d value uses information from after its " + f"own 14:51 entry anchor." + ) + + +def test_the_known_exception_is_still_exactly_one_factor_and_still_leaks(): + """jump_amount_corr_20: recorded as a defect, asserted positively. + + Asserted in the POSITIVE direction on purpose. A test that merely tolerated the + exception would stay green whether it was fixed, worsened, or spread to a second + factor. This one goes red on any of those, which is what forces the decision to + be taken deliberately. + """ + assert KNOWN_POST_CUTOFF_DEPENDENT in BOUND_FACTOR_IDS + cells, moved, worst = _moved_cells(KNOWN_POST_CUTOFF_DEPENDENT) + assert cells > 0 + assert moved > 0, ( + f"{KNOWN_POST_CUTOFF_DEPENDENT} no longer depends on post-{CUTOFF} bars. If " + f"that was deliberate, this factor's PUBLISHED values changed: update the " + f"exception list here, and re-state its verdict rather than letting the " + f"artifacts drift silently." + ) + assert worst > 0.0 From 082e8f0ebde96a23b9adb9c1333cbe7642807fe2 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Sat, 25 Jul 2026 15:09:41 -0700 Subject: [PATCH 5/6] style(factors): keep the binding __all__ in its existing sorted order --- factors/compute/minute/binding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/factors/compute/minute/binding.py b/factors/compute/minute/binding.py index f133bdc..4493b16 100644 --- a/factors/compute/minute/binding.py +++ b/factors/compute/minute/binding.py @@ -480,10 +480,10 @@ def is_cross_sectional_minute(factor: Factor) -> bool: "MinuteStreamBinding", "combine_minute_stats", "has_minute_diagnostics", - "minute_diagnostics_from_bars", "is_cross_sectional_minute", "is_minute_bound", "is_valid_day_pooled", + "minute_diagnostics_from_bars", "minute_raw_from_bars", "minute_stats_from_bars", "pooled_baseline_days", From babcc9bfd2c92a80e2b38e6812e8d208d8f9b24d Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Sat, 25 Jul 2026 16:05:02 -0700 Subject: [PATCH 6/6] fix(analytics/eval): derive the exec identity from measured facts; two runs, two claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review APPROVE-WITH-NITS follow-up: MEDIUM-1, MEDIUM-2, LOW-1. MEDIUM-1 — the test that claimed to pin cc06d32 could not. It matched SUBSTRINGS in the function's source (`"replace(" in source and "EXEC_TO_EXEC" in source and "DECISION" in source`). Review broke it by hand: gut the assignment to a bare `replace(eval_cfg)`, leave the two tokens in a COMMENT — the exec path then inherits the caller's CLOSE identity and every exec artifact declares `view=close/close_to_close` while being scored on exec returns. 18 tests in the file passed; 52 across the three files that import `exec_basis_eval` passed. #78's rule again (a lexical assertion cannot hold a semantic claim), failing in the worst direction: it waved through exactly the defect the commit under test had just fixed. It also had a false-positive direction — refactoring that line into a helper deletes the tokens and reddens a correct build. The identity restatement is now the extracted `exec_identity()` and the claim is BEHAVIOURAL: close-default config in, `view=decision` / `return_basis=exec_to_exec` out, and every other field bit-identical. MEDIUM-2 — the exec path declared `view=decision` unconditionally, and on two classes of run that sentence was false. `view` is defined as "the information set the FACTOR VALUES were taken under", but a with-book run has TWO information sets: the subject factor's and the book's (value_ep / value_bp / volatility_20 take close(d) and the evening daily_basic(d) — design §1.1's live defect, open until D7). One field silently reported the clean one. That is the "report declares a check it did not do" failure this field exists to prevent, one level up. Free to fix now because no artifact carries it yet; after C5/C6 it would be baked into 22 files. Chose the coordinator's (1)+(2): derive what can be derived, register what cannot. * `EvalConfig.book_view` (None = no book) — the book's view, declared by the ONLY party that knows it, the caller. `run_exec_basis_evaluation(book_view="close")` defaults to what every current caller actually does (the J4 rule: a default must be true for the callers that omit it). The no-book and with-book runs now get SEPARATE configs, so neither has to speak for the other. Deliberately NOT pairing-checked against the basis: a close-view book under an exec basis IS the disclosure, and refusing it would not un-mix the information sets, only stop the artifact from saying they are mixed. * `subject_view()` derives the SUBJECT's view from a measured fact — `factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE`, a deny list of factors whose compute applies no 14:50 truncation. It lives with the factors (it is a fact about values, not about the eval path) and tests/test_decision_cutoff_visibility.py MEASURES it; the test's local copy of the name is gone, because two copies are two things to keep in step and the failure mode of drifting apart is an artifact declaring an information set its values do not have. * Consequence, intended and loud: `jump_amount_corr_20` yields (close, exec_to_exec), which `EvalConfig` refuses. Correct — until that factor is truncated there IS no honest exec artifact of it, and a blocked run beats a false one (red line #9). The generic pairing message is re-raised with the specific reason so nobody hunts for a config typo. The separate correctness-fix PR clears it by dropping one entry from the deny list. LOW-1 — `test_identity_phrase_is_authored_once` asserted the phrase's FORMAT, not that no second place spells it. The property does hold (one producer, three call sites) but a name must not claim more than its assertions: renamed to `test_identity_phrase_covers_both_pairings_and_states_the_book`. A GAP I FOUND IN MY OWN NEW TESTS, and closed. Unit-testing `exec_identity` says nothing about how the caller WIRES it: passing `book_view=None` for the with-book run — a book was supplied, the report says there was none — passed all 28 tests. The end-to-end assertion now lives in tests/test_exec_basis_eval.py, reads both written JSONs, and checks the rendered Markdown says it too. Same shape as MEDIUM-1, one level down; recorded rather than quietly patched. Mutation evidence (every mutation first asserted to change its target): A gut `exec_identity` to `replace(eval_cfg)` (the review's own escape) -> 3 FAIL (was: 52 pass) rc=1 -> restored pass B with-book run gets `book_view=None` -> pre-check: 28 passed (the gap, proven before fixing); after adding the wiring test -> 1 FAIL `assert None == 'close'` rc=1 -> pass C empty `NOT_DECISION_CUTOFF_SAFE` -> 3 FAIL, incl. the MEASUREMENT test finding jump dirty rc=1 -> restored pass D `subject_view` back to a hard-coded `View.DECISION` -> 1 FAIL rc=1 -> restored pass Catalogue §七之二 updated BEFORE any artifact regenerates: `eval_config` is +3 keys (not +2), the no_book/with_book blocks now legitimately differ in `book_view`, and the jump_amount_corr exec block is registered as an intended loud refusal. Gates: pytest 2228 passed / ruff clean / 31 configs / phase0 ic 0.9600 annual 0.8408 unchanged / provisional anchors green / exec baseline --verify 77/77 at 45c14aa / frozen panel mtimes untouched. --- analytics/eval/config.py | 26 +++++ analytics/eval/contract.py | 20 +++- analytics/eval/render.py | 3 +- .../factors/d5_runner_difference_catalogue.md | 6 +- factors/compute/minute/binding.py | 26 +++++ qt/exec_basis_eval.py | 86 ++++++++++++-- tests/test_decision_cutoff_visibility.py | 66 ++++++++--- tests/test_eval_contract_v1.py | 105 +++++++++++++++--- tests/test_exec_basis_eval.py | 48 +++++++- 9 files changed, 339 insertions(+), 47 deletions(-) diff --git a/analytics/eval/config.py b/analytics/eval/config.py index 5797351..4b63519 100644 --- a/analytics/eval/config.py +++ b/analytics/eval/config.py @@ -76,6 +76,20 @@ class EvalConfig: pairings are decision<->exec_to_exec and close<->close_to_close, and anything else raises here rather than being a doc convention (design §1.4 mechanism 1, enforced through ``data.availability_policy``). + book_view : (contract v1.0) the view the KNOWN-FACTOR BOOK was taken under, + or None when no book was supplied. A with-book run has TWO information + sets — the subject factor's and the book's — and ``view`` can only + describe one of them, so it describes the subject and this describes the + book. Design §1.1 records the current book (value_ep / value_bp / + volatility_20) as taking close(d) values while exec holding windows open + at 14:51(d): a KNOWN live defect, closed at D7. Until then the honest + artifact says ``view=decision, book_view=close`` — one field claiming + both would be exactly the "report declares a check it did not do" failure + this contract exists to prevent, one level up. + + DELIBERATELY NOT pairing-checked against ``return_basis``. The book being + on a different view from the subject IS the disclosure; refusing it would + not fix the mixture, it would only stop the artifact from saying so. success_criteria : the PRE-REGISTERED verdict bar (design §6, v0.6). A frozen :class:`~analytics.eval.verdict.VerdictThresholds` declared HERE, before ``evaluate`` runs, IS pre-registered by construction: you cannot tune the @@ -123,6 +137,7 @@ class EvalConfig: # with no false statement left behind. view: str = View.CLOSE.value return_basis: str = ReturnBasis.CLOSE_TO_CLOSE.value + book_view: str | None = None success_criteria: VerdictThresholds | None = None def __post_init__(self) -> None: @@ -288,6 +303,17 @@ def _check_view_basis(self) -> None: ) object.__setattr__(self, "view", resolved_view.value) object.__setattr__(self, "return_basis", resolved_basis.value) + if self.book_view is not None: + try: + book = View(self.book_view) + except ValueError: + allowed = ", ".join(repr(v.value) for v in View) + raise ValueError( + f"EvalConfig.book_view must be None (no book supplied) or one " + f"of {allowed}; got {self.book_view!r}. A book whose view is " + f"unstated is exactly the mixture this field exists to expose." + ) from None + object.__setattr__(self, "book_view", book.value) def _check_success_criteria(self) -> None: """The pre-registered bar, if supplied, must be a real VerdictThresholds. diff --git a/analytics/eval/contract.py b/analytics/eval/contract.py index 27e1408..050e5ac 100644 --- a/analytics/eval/contract.py +++ b/analytics/eval/contract.py @@ -56,16 +56,30 @@ #: guard all refer to ONE list instead of three spellings of it. IDENTITY_FIELDS: tuple[str, ...] = ("view", "return_basis") +#: How an absent book is rendered. A with-book run whose book view is unknown and a +#: no-book run must not look alike, so the absence is WORDED, never left blank. +NO_BOOK = "none (no book supplied)" -def basis_identity_phrase(view: str, return_basis: str) -> str: - """The ONE sentence that states a report's information set + return basis. + +def basis_identity_phrase( + view: str, return_basis: str, book_view: str | None +) -> str: + """The ONE sentence that states a report's information sets + return basis. Author-once (#76/#78/#82): the provenance row, the cross-basis summary header and any prose that needs to say which basis a number belongs to COMPOSE this string rather than each writing their own. A regex cannot assert that no other sentence says this; "there is no other sentence" can. + + ``book_view`` is part of the sentence rather than an optional addendum: a + with-book evaluation carries TWO information sets, and a phrase that mentions + only the subject's would be the same one-field-for-two-facts problem the field + was added to fix (see ``EvalConfig.book_view``). """ - return f"view={view} x return_basis={return_basis}" + return ( + f"view={view} x return_basis={return_basis}, " + f"book_view={book_view or NO_BOOK}" + ) __all__ = ["EVAL_CONTRACT_VERSION", "IDENTITY_FIELDS", "basis_identity_phrase"] diff --git a/analytics/eval/render.py b/analytics/eval/render.py index e461f40..e1ed9a2 100644 --- a/analytics/eval/render.py +++ b/analytics/eval/render.py @@ -250,7 +250,8 @@ def _provenance_rows(report: FactorEvalReport) -> list[tuple[str, str]]: # -- contract v1.0 identity + factor declarations (D1 contract v1.0/v1.1) -- ( "evaluation contract", - f"v{EVAL_CONTRACT_VERSION}, {basis_identity_phrase(cfg.view, cfg.return_basis)}", + f"v{EVAL_CONTRACT_VERSION}, " + + basis_identity_phrase(cfg.view, cfg.return_basis, cfg.book_view), ), ("requires (endpoint inputs)", _requires_row(spec)), ( diff --git a/docs/factors/d5_runner_difference_catalogue.md b/docs/factors/d5_runner_difference_catalogue.md index c6c91e7..35bf1ed 100644 --- a/docs/factors/d5_runner_difference_catalogue.md +++ b/docs/factors/d5_runner_difference_catalogue.md @@ -274,10 +274,14 @@ docstring，按 #74 先例）。它改变 artifact 的**三处且仅三处**， | # | 位置 | 变化 | 成因 | |---|---|---|---| -| 1 | JSON `eval_config` 块 | **+2 键** `view` / `return_basis` | `report_to_dict` 的 `sanitize_payload(vars(report.cfg))` 自动带上 `EvalConfig` 的新字段（与 §七 的 `spec` 16→20 完全同一机制） | +| 1 | JSON `eval_config` 块 | **+3 键** `view` / `return_basis` / `book_view` | `report_to_dict` 的 `sanitize_payload(vars(report.cfg))` 自动带上 `EvalConfig` 的新字段（与 §七 的 `spec` 16→20 完全同一机制） | | 2 | JSON 顶层 | **+1 键** `eval_contract_version` | 显式写入：一个 verdict 只有对着产生它的契约版本才可解释（#74 的教训） | | 3 | Markdown `## 0. Header & Provenance` | **+4 行** `evaluation contract` / `requires (endpoint inputs)` / `adjustment / overnight boundary` / `lookback depth ...` | R24 的身份字段 + D1 契约 v1.0/v1.1 的三个声明维，从 `vars(spec)` 的 repr 转述升级为具名行 | +⚠️ **`book_view` 使 exec 侧 no_book 与 with_book 两份 artifact 的 `eval_config` 块首次不同**（`null` vs `"close"`）。这是**有意**的：一次 with-book 评估携带**两个**信息集（候选因子的与因子簿的），一个 `view` 字段表达不了；设计 §1.1 记录的因子簿 close-view 活缺陷要到 D7 才关，在那之前诚实的 artifact 就该写 `view=decision, book_view=close`。对账时**不要**把这一处不同当作 no_book/with_book 之间的回归。 + +⚠️ **`jump_amount_corr_20` 的 exec 评估现在会 loud raise**（`qt.exec_basis_eval.exec_identity`）：它的值是 close 视图（compute 无 14:50 截断，实测），(close, exec_to_exec) 非法配对。这是**故意的阻断**——在该因子被截断修正之前，它没有诚实的 exec artifact；而写一份声明 `view=decision` 的假 artifact 正是本字段要防的事。修正由**独立的 correctness-fix PR** 处理；它落地时从 `factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE` 移除该条即可解除阻断。 + **没有变的**（`tests/test_eval_contract_v1.py` 逐值钉住）：`VerdictThresholds` 的**每一个** 默认门（`min_abs_icir=0.30` / `min_incremental_abs_icir=0.15` / `min_monotonicity_spearman=0.0` / 三段样本门），以及三轴判决 / 非对称门 / diff --git a/factors/compute/minute/binding.py b/factors/compute/minute/binding.py index 4493b16..13f02b0 100644 --- a/factors/compute/minute/binding.py +++ b/factors/compute/minute/binding.py @@ -388,6 +388,30 @@ def is_valid_day_pooled(factor: Factor) -> bool: ) +#: MEASURED exceptions: minute factors whose DAY-d VALUE depends on bars AFTER the +#: 14:50 decision cutoff, i.e. whose compute applies no cutoff of its own and must +#: therefore be handed pre-truncated bars to be decision-view. Under the +#: ``exec_to_exec`` basis a value like that uses information from after its own +#: 14:51 entry anchor, so an evaluation of it CANNOT honestly declare +#: ``view=decision``. +#: +#: This is a DENY LIST OF MEASURED FACTS, never a proof of safety: a factor absent +#: from it has either been measured clean or never measured. +#: ``tests/test_decision_cutoff_visibility.py`` is what does the measuring (perturb +#: only the post-cutoff bars, assert the pre-cutoff rows byte-identical, ask whether +#: the value moves) and it pins this set to exactly its own contents, so a second +#: offender cannot join silently. +#: +#: Declared HERE, next to the other minute-factor partitions, because it is a fact +#: about a factor's values — not about the evaluation path that has to consult it. +NOT_DECISION_CUTOFF_SAFE: frozenset[type[Factor]] = frozenset({JumpAmountCorrFactor}) + + +def is_decision_cutoff_safe(factor: Factor) -> bool: + """True iff ``factor``'s value at d uses no bar after d's 14:50 cutoff.""" + return type(factor) not in NOT_DECISION_CUTOFF_SAFE + + def is_minute_bound(factor: Factor) -> bool: """True iff ``factor`` has a bars-only raw-compute binding here.""" return type(factor) in _MINUTE_BINDINGS @@ -474,6 +498,7 @@ def is_cross_sectional_minute(factor: Factor) -> bool: __all__ = [ "CROSS_SECTIONAL_MINUTE_FACTORS", + "NOT_DECISION_CUTOFF_SAFE", "STATS_VALUE_COL", "VALID_DAY_POOLED_FACTORS", "BindingFn", @@ -481,6 +506,7 @@ def is_cross_sectional_minute(factor: Factor) -> bool: "combine_minute_stats", "has_minute_diagnostics", "is_cross_sectional_minute", + "is_decision_cutoff_safe", "is_minute_bound", "is_valid_day_pooled", "minute_diagnostics_from_bars", diff --git a/qt/exec_basis_eval.py b/qt/exec_basis_eval.py index 8091089..ef9e65c 100644 --- a/qt/exec_basis_eval.py +++ b/qt/exec_basis_eval.py @@ -42,6 +42,8 @@ StandardFactorEvaluator, ) from data.availability_policy import ReturnBasis, View +from factors import registry as factor_registry +from factors.compute.minute.binding import is_decision_cutoff_safe from analytics.eval.figures import render_factor_dashboard from analytics.factor import forward_returns as _close_forward_returns from data.clean.schema import DATE_LEVEL @@ -152,6 +154,62 @@ def build_disclosure( return disclosure +def subject_view(factor_id: str) -> str: + """The information-set view the factor's OWN values were taken under. + + DERIVED from a measured fact, not written down: a minute factor whose compute + applies no 14:50 truncation of its own was handed full-day bars and is therefore + on the CLOSE view, whatever the run intends. The deny list lives with the + factors (``factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE``) and is + pinned by the perturb-the-post-cutoff-bars test; an id the registry does not + know (a synthetic spec) is not on that list and is treated as decision-view. + """ + try: + factor = factor_registry.build(factor_id) + except Exception: # noqa: BLE001 - an unregistered id has no measured exception + return View.DECISION.value + return View.DECISION.value if is_decision_cutoff_safe(factor) else View.CLOSE.value + + +def exec_identity( + eval_cfg: EvalConfig, *, factor_id: str, book_view: str | None +) -> EvalConfig: + """The exec-basis twin of ``eval_cfg`` — the identity DERIVED, not asserted. + + The caller hands ONE config to its close reports and to this module, so the + exec identity has to be restated here or the exec artifact inherits the close + label. ``return_basis`` is the one constant (it is this module's entire + purpose); ``view`` comes from :func:`subject_view` and ``book_view`` from the + caller, who is the only one who knows how the book was built. + + A factor that is not decision-cutoff-safe yields the pairing + (close, exec_to_exec), which ``EvalConfig`` refuses — and refusing is correct: + such a factor's values use information from after their own 14:51 entry anchor, + so no honest exec-basis artifact of it exists until that is fixed. The refusal + is re-raised with the specific reason rather than left as the generic pairing + message, which would send the reader looking for a config typo. + """ + view = subject_view(factor_id) + try: + return replace( + eval_cfg, + view=view, + return_basis=ReturnBasis.EXEC_TO_EXEC.value, + book_view=book_view, + ) + except ValueError as exc: + raise ValueError( + f"cannot build an exec_to_exec evaluation of {factor_id!r}: its values " + f"are on the {view!r} view because its compute applies no 14:50 " + f"truncation of its own (measured — see " + f"factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE and " + f"tests/test_decision_cutoff_visibility.py), so the day-d value uses " + f"bars from after its own 14:51 entry anchor. Declaring view=decision " + f"anyway would be the report stating a check it did not do; the " + f"correctness fix is tracked separately." + ) from exc + + def _write_report(report: FactorEvalReport, report_dir: Path, stem: str) -> tuple[Path, Path]: md_path = report_dir / f"{stem}.md" json_path = report_dir / f"{stem}.json" @@ -201,23 +259,31 @@ def run_exec_basis_evaluation( report_dir: Path, stem: str, force_rebuild: bool = False, + book_view: str = View.CLOSE.value, ) -> ExecBasisEvaluation: """Build the exec-to-exec returns, sanity-check them, evaluate twice, report. ``panel`` is the front-adjusted daily panel (it carries the raw ``adj_factor`` the adjustment identity needs and defines the shared price grid); ``spec`` is the factor's DAILY spec, from which the intraday variant is derived. + + ``book_view`` declares which information set ``book`` was built from. It + defaults to ``close`` because that is what every current caller does — the + confirmed book takes close(d) values and the evening daily_basic(d), the live + defect design §1.1 records and D7 closes. The decision-view book path passes + ``decision`` explicitly. Only the caller knows this, so it is a parameter + rather than something guessed here. """ started = time.monotonic() params = ExecBasisParams.from_config(cfg) exec_spec = intraday_spec_variant(spec, params) - # Contract v1.0: the EvalConfig's identity travels with the basis, exactly as - # the spec's does one line above. The caller hands ONE config to its close - # reports and to this function, so stating the exec identity HERE is what keeps - # both artifacts truthful about which information set they describe. The - # pairing is re-validated by EvalConfig's own construction check. - eval_cfg = replace( - eval_cfg, view=View.DECISION.value, return_basis=ReturnBasis.EXEC_TO_EXEC.value + # Contract v1.0: the identity travels with the basis, exactly as the spec's does + # one line above — and the two runs below carry DIFFERENT identities, because + # the no-book run has no book and the with-book run has a close-view one. One + # shared config would have to pick one of those and be wrong about the other. + cfg_no_book = exec_identity(eval_cfg, factor_id=spec.factor_id, book_view=None) + cfg_with_book = exec_identity( + eval_cfg, factor_id=spec.factor_id, book_view=book_view ) horizon = spec.forward_return_horizon @@ -314,7 +380,7 @@ def run_exec_basis_evaluation( execution_capacity=disclosure, ) report_no_book, ir_no_book = evaluator.evaluate_with_ir( - factor_panel, exec_spec, eval_cfg, ctx_no_book + factor_panel, exec_spec, cfg_no_book, ctx_no_book ) ctx_with_book = EvalContext( forward_returns=exec_returns, @@ -324,7 +390,7 @@ def run_exec_basis_evaluation( execution_capacity=disclosure, ) report_with_book, ir_with_book = evaluator.evaluate_with_ir( - factor_panel, exec_spec, eval_cfg, ctx_with_book + factor_panel, exec_spec, cfg_with_book, ctx_with_book ) nb_md, nb_json = _write_report(report_no_book, report_dir, f"{stem}_exec_no_book") @@ -402,6 +468,8 @@ def format_exec_basis_line(result: ExecBasisEvaluation) -> str: __all__ = [ "ExecBasisEvaluation", "build_disclosure", + "exec_identity", "format_exec_basis_line", "run_exec_basis_evaluation", + "subject_view", ] diff --git a/tests/test_decision_cutoff_visibility.py b/tests/test_decision_cutoff_visibility.py index 06f943d..ba0f980 100644 --- a/tests/test_decision_cutoff_visibility.py +++ b/tests/test_decision_cutoff_visibility.py @@ -21,12 +21,19 @@ ``prepare_visible_minute_bars`` references in its module), so under the old runners it saw 09:30-15:00 of day d. -That is recorded here as a KNOWN, NAMED exception rather than fixed. Truncating it -would change a published factor's values and could move its verdict — a definition +That is recorded as a KNOWN, NAMED exception rather than fixed. Truncating it would +change a published factor's values and could move its verdict — a definition -affecting research decision, not a refactor's business (design v3.2 §〇). The exception is asserted POSITIVELY, so the day someone changes it, this test goes red and the change has to be made deliberately instead of noticed later. +THE LIST IS PRODUCTION STATE, NOT TEST STATE. It lives in +``factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE`` because the exec path +consults it to decide whether an evaluation may declare ``view=decision`` +(``qt.exec_basis_eval.subject_view``); this file is what MEASURES it. A copy here +would be a second thing to keep in step, and the failure mode of drifting apart is +an artifact declaring an information set its values do not have. + WHY THE EXISTING SUITE DID NOT CATCH IT — and why that is not a contradiction. ``tests/test_jump_amount_corr_factor.py::test_pit_perturbing_future_bars_does_not_ change_factor_at_d`` is correct and passes: it perturbs day ``d+1`` and checks day @@ -48,13 +55,19 @@ from data.clean.intraday_schema import normalize_intraday_bars from factors import registry as factor_registry -from factors.compute.minute.binding import _MINUTE_STREAM_BINDINGS, minute_raw_from_bars +from factors.compute.minute.binding import ( + _MINUTE_STREAM_BINDINGS, + NOT_DECISION_CUTOFF_SAFE, + minute_raw_from_bars, +) CUTOFF = "14:50:00" -#: The one factor whose value is NOT decision-cutoff-safe on its own. Named, not -#: inferred — see the module docstring for why it is recorded instead of fixed. -KNOWN_POST_CUTOFF_DEPENDENT = "jump_amount_corr_20" +#: DERIVED from the production deny list, not spelled again here. The exec path has +#: to consult the same fact to decide whether it may declare ``view=decision`` +#: (``qt.exec_basis_eval.subject_view``), so the list lives with the factors and +#: this file MEASURES it. Two copies would be two things to keep in step. +KNOWN_POST_CUTOFF_DEPENDENT_IDS = frozenset(c().name for c in NOT_DECISION_CUTOFF_SAFE) SYMBOLS = [f"6000{i:02d}.SH" for i in range(12)] DATES = pd.bdate_range("2021-01-04", periods=40) @@ -129,7 +142,9 @@ def _moved_cells(factor_id: str) -> tuple[int, int, float]: BOUND_FACTOR_IDS = tuple(sorted(cls().name for cls in _MINUTE_STREAM_BINDINGS)) -CLEAN_FACTOR_IDS = tuple(f for f in BOUND_FACTOR_IDS if f != KNOWN_POST_CUTOFF_DEPENDENT) +CLEAN_FACTOR_IDS = tuple( + f for f in BOUND_FACTOR_IDS if f not in KNOWN_POST_CUTOFF_DEPENDENT_IDS +) @pytest.mark.parametrize("factor_id", CLEAN_FACTOR_IDS) @@ -150,15 +165,32 @@ def test_the_known_exception_is_still_exactly_one_factor_and_still_leaks(): Asserted in the POSITIVE direction on purpose. A test that merely tolerated the exception would stay green whether it was fixed, worsened, or spread to a second factor. This one goes red on any of those, which is what forces the decision to - be taken deliberately. + be taken deliberately — and because the exec path DERIVES its right to declare + ``view=decision`` from the same list, a stale entry here is not cosmetic. + """ + assert KNOWN_POST_CUTOFF_DEPENDENT_IDS == {"jump_amount_corr_20"} + for factor_id in sorted(KNOWN_POST_CUTOFF_DEPENDENT_IDS): + assert factor_id in BOUND_FACTOR_IDS + cells, moved, worst = _moved_cells(factor_id) + assert cells > 0 + assert moved > 0, ( + f"{factor_id} no longer depends on post-{CUTOFF} bars. If that was " + f"deliberate, this factor's PUBLISHED values changed: drop it from " + f"factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE, and re-state " + f"its verdict rather than letting the artifacts drift silently." + ) + assert worst > 0.0 + + +def test_every_clean_factor_is_absent_from_the_production_deny_list(): + """The measurement and the list the exec path trusts cannot disagree. + + Without this, a factor could be measured clean here while still sitting on the + deny list (its exec evaluation blocked for no reason) or, worse, be measured + dirty and be missing from the list (its exec artifact declaring view=decision). """ - assert KNOWN_POST_CUTOFF_DEPENDENT in BOUND_FACTOR_IDS - cells, moved, worst = _moved_cells(KNOWN_POST_CUTOFF_DEPENDENT) - assert cells > 0 - assert moved > 0, ( - f"{KNOWN_POST_CUTOFF_DEPENDENT} no longer depends on post-{CUTOFF} bars. If " - f"that was deliberate, this factor's PUBLISHED values changed: update the " - f"exception list here, and re-state its verdict rather than letting the " - f"artifacts drift silently." + for factor_id in CLEAN_FACTOR_IDS: + assert factor_id not in KNOWN_POST_CUTOFF_DEPENDENT_IDS + assert set(CLEAN_FACTOR_IDS) | KNOWN_POST_CUTOFF_DEPENDENT_IDS == set( + BOUND_FACTOR_IDS ) - assert worst > 0.0 diff --git a/tests/test_eval_contract_v1.py b/tests/test_eval_contract_v1.py index dd946f9..c899dcc 100644 --- a/tests/test_eval_contract_v1.py +++ b/tests/test_eval_contract_v1.py @@ -67,18 +67,78 @@ def test_the_exec_pairing_is_constructible_and_is_what_the_exec_path_declares(): def test_the_exec_basis_module_restates_the_identity_it_scores_on(): - """qt.exec_basis_eval must not inherit the caller's close-basis identity. + """BEHAVIOURAL: a close-default config in, the exec identity out. + + The first version of this test matched SUBSTRINGS in the function's source + (``"replace(" in source and "EXEC_TO_EXEC" in source``). Review broke it by + hand: gut the assignment, leave the two matched names in a COMMENT, and the exec + path silently inherits the caller's close identity while 52 tests stay green. + That is #78's rule again — a lexical assertion cannot hold a semantic claim — and + it failed in the worst direction, waving through the very defect the commit under + test had just fixed. It also had a false-positive direction: refactoring the line + into a helper would have deleted those names and reddened a correct build. + + So the claim is now made about BEHAVIOUR, on the extracted helper. + """ + from qt.exec_basis_eval import exec_identity + + legacy = _cfg() + assert (legacy.view, legacy.return_basis) == ("close", "close_to_close") + + out = exec_identity(legacy, factor_id="volume_peak_count_20", book_view="close") + assert out.view == "decision" + assert out.return_basis == "exec_to_exec" + assert out.book_view == "close" + # ... and nothing else moved: the identity is all that travels with the basis. + ignore = {"view", "return_basis", "book_view"} + assert {k: v for k, v in vars(out).items() if k not in ignore} == { + k: v for k, v in vars(legacy).items() if k not in ignore + } + - Structural rather than descriptive: the module derives the exec twin of the - SPEC one line earlier, and the config's identity has to travel the same way or - the exec artifact would carry the close label. +def test_a_factor_that_is_not_cutoff_safe_cannot_get_an_exec_identity(): + """The derivation refuses rather than declaring a view the values do not have. + + ``jump_amount_corr_20``'s compute applies no 14:50 truncation, so its values are + close-view; (close, exec_to_exec) is not a legal pairing, and that refusal is + the correct outcome — there is no honest exec artifact of it until it is fixed. """ - import inspect + from qt.exec_basis_eval import exec_identity, subject_view + + assert subject_view("jump_amount_corr_20") == "close" + assert subject_view("volume_peak_count_20") == "decision" + with pytest.raises(ValueError, match="no 14:50 truncation of its own"): + exec_identity(_cfg(), factor_id="jump_amount_corr_20", book_view="close") + + +def test_the_no_book_and_with_book_runs_do_not_share_one_book_view(): + """Two runs, two information sets, two configs — never one shared claim.""" + from qt.exec_basis_eval import exec_identity + + no_book = exec_identity(_cfg(), factor_id="volume_peak_count_20", book_view=None) + with_book = exec_identity(_cfg(), factor_id="volume_peak_count_20", book_view="close") + assert no_book.book_view is None + assert with_book.book_view == "close" + assert no_book.view == with_book.view == "decision" - from qt import exec_basis_eval - source = inspect.getsource(exec_basis_eval.run_exec_basis_evaluation) - assert "replace(" in source and "EXEC_TO_EXEC" in source and "DECISION" in source +def test_book_view_is_validated_and_an_unknown_value_is_refused(): + assert _cfg(book_view=None).book_view is None + assert _cfg(book_view=View.CLOSE).book_view == "close" + with pytest.raises(ValueError, match="book_view must be None"): + _cfg(book_view="whatever") + + +def test_book_view_is_not_pairing_checked_against_the_basis(): + """A close-view book under an exec basis is the DISCLOSURE, not a config error. + + Refusing it would not un-mix the information sets; it would only stop the + artifact from saying they are mixed (design §1.1's live defect, open until D7). + """ + cfg = _cfg(view="decision", return_basis="exec_to_exec", book_view="close") + assert (cfg.view, cfg.return_basis, cfg.book_view) == ( + "decision", "exec_to_exec", "close", + ) @pytest.mark.parametrize( @@ -103,20 +163,35 @@ def test_enum_members_normalize_to_canonical_strings(): assert isinstance(cfg.view, str) and type(cfg.view) is str -def test_identity_phrase_is_authored_once(): - """The provenance row COMPOSES the phrase rather than spelling it again.""" - cfg = _cfg(view="decision", return_basis="exec_to_exec") +def test_identity_phrase_covers_both_pairings_and_states_the_book(): + """Renamed (review LOW-1): this pins the phrase's FORMAT, not author-once. + + The old name claimed "authored once", which it never tested — it cannot see + whether some other module spells the same sentence itself. That property does + hold (one producer in ``analytics/eval/contract.py``, composed at its call + sites), but a name must not claim more than its assertions. + """ + cfg = _cfg(view="decision", return_basis="exec_to_exec", book_view="close") assert ( - basis_identity_phrase(cfg.view, cfg.return_basis) - == "view=decision x return_basis=exec_to_exec" + basis_identity_phrase(cfg.view, cfg.return_basis, cfg.book_view) + == "view=decision x return_basis=exec_to_exec, book_view=close" ) legacy = _cfg() assert ( - basis_identity_phrase(legacy.view, legacy.return_basis) - == "view=close x return_basis=close_to_close" + basis_identity_phrase(legacy.view, legacy.return_basis, legacy.book_view) + == "view=close x return_basis=close_to_close, book_view=none (no book supplied)" ) +def test_an_absent_book_is_worded_never_blank(): + """"No book" is a stated fact, so it can never be read as a missing field.""" + absent = basis_identity_phrase("close", "close_to_close", None) + stated = basis_identity_phrase("close", "close_to_close", "close") + assert absent.endswith("book_view=none (no book supplied)") + assert absent != stated + assert "book_view=\n" not in absent and not absent.endswith("book_view=") + + # --------------------------------------------------------------------------- # # 2. the non-omittable basis column (R24) # --------------------------------------------------------------------------- # diff --git a/tests/test_exec_basis_eval.py b/tests/test_exec_basis_eval.py index ec31def..c9e9aa8 100644 --- a/tests/test_exec_basis_eval.py +++ b/tests/test_exec_basis_eval.py @@ -8,7 +8,15 @@ * the mandatory disclosure (execution parameters, coverage loss by cause, measured live-call count) actually reaches the rendered report; * adding that disclosure does NOT move a verdict axis — the Tradable axis stays - NOT_ASSESSED, because a return basis is not a fill-feasibility measurement. + NOT_ASSESSED, because a return basis is not a fill-feasibility measurement; + * the two exec artifacts state their OWN information sets (contract v1.0): the + no-book report says it had no book, the with-book report names the book's view. + +That last one is here rather than beside the ``exec_identity`` unit tests on +purpose. Unit-testing the helper says nothing about how the caller WIRES it, and +that gap is real: passing ``book_view=None`` for the with-book run — a book was +supplied, the report says there was none — passed 28 tests before this file +asserted it. """ from __future__ import annotations @@ -168,6 +176,44 @@ def test_exec_basis_eval_writes_its_own_reports_and_leaves_the_control_alone(tmp assert out.minute_live_calls == 0 +def test_the_two_exec_artifacts_state_their_own_information_sets(tmp_path): + """no_book says "no book"; with_book NAMES the book's view (contract v1.0). + + One shared config would have to pick one of those and be wrong about the other + — and being wrong here is the §1.1 live defect (a close-view book scored against + an exec holding window) going unmentioned in the artifact that contains it. + """ + cfg, panel, factor, book, spec, eval_cfg, report_dir = _fixture(tmp_path) + out = run_exec_basis_evaluation( + factor, spec, eval_cfg, book, + cfg=cfg, panel=panel, symbols=list(SYMBOLS), logger=LOGGER, + report_dir=report_dir, stem="demo", + ) + + no_book = json.loads(out.no_book_json.read_text(encoding="utf-8"))["eval_config"] + with_book = json.loads(out.with_book_json.read_text(encoding="utf-8"))["eval_config"] + + # Both are exec-basis evaluations of a decision-view subject factor ... + for block in (no_book, with_book): + assert block["view"] == "decision" + assert block["return_basis"] == "exec_to_exec" + # ... but only ONE of them had a book, and they must not claim otherwise. + assert no_book["book_view"] is None + assert with_book["book_view"] == "close" + + # The caller's own config is untouched: the identity travels with the basis, + # it does not leak back into the close-basis reports the caller also writes. + assert (eval_cfg.view, eval_cfg.return_basis) == ("close", "close_to_close") + assert eval_cfg.book_view is None + + # And the rendered Markdown says it too, through the author-once phrase. + md = out.with_book_md.read_text(encoding="utf-8") + assert "book_view=close" in md + assert "book_view=none (no book supplied)" in out.no_book_md.read_text( + encoding="utf-8" + ) + + def test_exec_basis_eval_discloses_parameters_and_coverage_in_every_report(tmp_path): cfg, panel, factor, book, spec, eval_cfg, report_dir = _fixture(tmp_path) out = run_exec_basis_evaluation(