diff --git a/analytics/eval/__init__.py b/analytics/eval/__init__.py index 65e3b3c..9ca2a3b 100644 --- a/analytics/eval/__init__.py +++ b/analytics/eval/__init__.py @@ -19,6 +19,11 @@ """ from analytics.eval.config import EvalConfig +from analytics.eval.contract import ( + EVAL_CONTRACT_VERSION, + IDENTITY_FIELDS, + basis_identity_phrase, +) from analytics.eval.evaluator import EvalIR, FactorEvaluator from analytics.eval.ir import EvalContext, StandardEvalIR, build_eval_ir from analytics.eval.report import ( @@ -27,6 +32,11 @@ extract_verdict_inputs, ) from analytics.eval.standard import StandardFactorEvaluator +from analytics.eval.summary import ( + REQUIRED_SUMMARY_COLUMNS, + render_verdict_summary, + require_basis_columns, +) from analytics.eval.sections import ( MANDATORY_SECTIONS, VERDICT_KEYS, @@ -56,6 +66,12 @@ __all__ = [ "ADOPT", "AXIS_FAIL", + "EVAL_CONTRACT_VERSION", + "IDENTITY_FIELDS", + "REQUIRED_SUMMARY_COLUMNS", + "basis_identity_phrase", + "render_verdict_summary", + "require_basis_columns", "AXIS_INSUFFICIENT_DATA", "AXIS_NAMES", "AXIS_NOT_ASSESSED", diff --git a/analytics/eval/config.py b/analytics/eval/config.py index c9171c4..4b63519 100644 --- a/analytics/eval/config.py +++ b/analytics/eval/config.py @@ -18,6 +18,7 @@ from dataclasses import dataclass from analytics.eval.verdict import VerdictThresholds +from data.availability_policy import ReturnBasis, View, require_legal_pairing # Tolerance for spotting the base (multiplier 1.0) cost scenario among floats. _BASE_COST = 1.0 @@ -65,6 +66,30 @@ class EvalConfig: n_factors_screened : multiple-testing background (how many factors were looked at to find this one). data_snapshot_id : data/cache version, for reproducibility. + view : (contract v1.0) the INFORMATION-SET view the factor values were taken + under — ``"decision"`` (14:50) or ``"close"``. Not decoration: a verdict + computed on one view is a different statistical claim from the same + verdict on the other, and before v1.0 a report could not say which it was. + return_basis : (contract v1.0) the forward-return basis the factor was scored + on — ``"exec_to_exec"`` (the 14:51 execution anchor) or + ``"close_to_close"``. VALIDATED AS A PAIR with ``view``: the two legal + pairings are decision<->exec_to_exec and close<->close_to_close, and + anything else raises here rather than being a doc convention (design + §1.4 mechanism 1, enforced through ``data.availability_policy``). + book_view : (contract v1.0) the view the KNOWN-FACTOR BOOK was taken under, + or None when no book was supplied. A with-book run has TWO information + sets — the subject factor's and the book's — and ``view`` can only + describe one of them, so it describes the subject and this describes the + book. Design §1.1 records the current book (value_ep / value_bp / + volatility_20) as taking close(d) values while exec holding windows open + at 14:51(d): a KNOWN live defect, closed at D7. Until then the honest + artifact says ``view=decision, book_view=close`` — one field claiming + both would be exactly the "report declares a check it did not do" failure + this contract exists to prevent, one level up. + + DELIBERATELY NOT pairing-checked against ``return_basis``. The book being + on a different view from the subject IS the disclosure; refusing it would + not fix the mixture, it would only stop the artifact from saying so. success_criteria : the PRE-REGISTERED verdict bar (design §6, v0.6). A frozen :class:`~analytics.eval.verdict.VerdictThresholds` declared HERE, before ``evaluate`` runs, IS pre-registered by construction: you cannot tune the @@ -97,6 +122,22 @@ class EvalConfig: tuned: bool = False n_factors_screened: int | None = None data_snapshot_id: str | None = None + # -- contract v1.0 identity (see analytics/eval/contract.py) -------------- + # + # THE DEFAULT IS THE LEGACY PAIRING, ON PURPOSE. A default's only job is to be + # TRUE for the callers that do not pass the field, and today those are the + # eleven close-basis runners: each builds ONE EvalConfig and hands it to BOTH + # its close_to_close reports and (through qt.exec_basis_eval) its exec ones. + # Defaulting to decision/exec_to_exec would make every close artifact state a + # basis it was not scored on — the describe-the-check drift of #76/#78/#82, + # introduced by the very field added to prevent it. The exec path sets the + # identity EXPLICITLY, exactly as it already derives the spec's exec twin + # (``intraday_spec_variant``), and the unified exec-only runner declares it too. + # When the close runners retire (D5 C6) this default becomes a one-line change + # with no false statement left behind. + view: str = View.CLOSE.value + return_basis: str = ReturnBasis.CLOSE_TO_CLOSE.value + book_view: str | None = None success_criteria: VerdictThresholds | None = None def __post_init__(self) -> None: @@ -106,6 +147,7 @@ def __post_init__(self) -> None: self._check_costs() self._check_capacity() self._check_declared_sequences() + self._check_view_basis() self._check_success_criteria() # -- validators (enforcement layer #1) -------------------------------- @@ -247,6 +289,32 @@ def _check_capacity(self) -> None: f"{self.limit_feasibility!r}." ) + def _check_view_basis(self) -> None: + """Contract v1.0: the view x return-basis pairing, enforced at construction. + + Delegated to :func:`data.availability_policy.require_legal_pairing` — the + SINGLE source of the legal pairs (D0). Re-stating the rule here would be + the describe-the-check drift the project keeps paying for. The normalized + enum VALUES are written back so the exported record carries the canonical + spelling whatever the caller passed (enum member or string). + """ + resolved_view, resolved_basis = require_legal_pairing( + self.view, self.return_basis + ) + object.__setattr__(self, "view", resolved_view.value) + object.__setattr__(self, "return_basis", resolved_basis.value) + if self.book_view is not None: + try: + book = View(self.book_view) + except ValueError: + allowed = ", ".join(repr(v.value) for v in View) + raise ValueError( + f"EvalConfig.book_view must be None (no book supplied) or one " + f"of {allowed}; got {self.book_view!r}. A book whose view is " + f"unstated is exactly the mixture this field exists to expose." + ) from None + object.__setattr__(self, "book_view", book.value) + def _check_success_criteria(self) -> None: """The pre-registered bar, if supplied, must be a real VerdictThresholds. diff --git a/analytics/eval/contract.py b/analytics/eval/contract.py new file mode 100644 index 0000000..050e5ac --- /dev/null +++ b/analytics/eval/contract.py @@ -0,0 +1,85 @@ +"""The factor-evaluation contract's VERSION and its declared identity fields. + +The statistical adjudication core (three axes / asymmetric gate / "unknown never +convicts" / N_eff CI / exploratory capped at Watch) is NOT rewritten by the +factor-layer refactor — it is UPGRADED, and this module is the explicit version +statement that upgrade owes (the PR #74 precedent: a change to a frozen contract +is stated, not slipped in). + +CONTRACT v1.0 — WHAT CHANGED AND WHY +------------------------------------ +v0.9 (the eleven-factor loop's frozen contract) described a factor and one +evaluation of it, but it could not say WHICH INFORMATION SET the factor values +were taken under. That was survivable while every run used one basis; it stopped +being survivable when the same factor started being evaluated on two +(``close_to_close`` and ``exec_to_exec``, PR #79), because two reports then +carried the same verdict word for two different statistical claims. + +v1.0 adds exactly two identity fields to ``EvalConfig`` and renders four more off +the ``FactorSpec`` that D1 already made mandatory: + +* ``EvalConfig.view`` / ``EvalConfig.return_basis`` — the information-set view and + the forward-return basis, validated as a LEGAL PAIRING at construction time via + :func:`data.availability_policy.require_legal_pairing`. A close-view factor + scored on ``exec_to_exec`` returns is now a readable error, not a convention. +* the provenance box renders ``FactorSpec.requires`` / ``adjustment`` / + ``overnight_boundary`` / ``lookback_depth`` (contract v1.0/v1.1 of the FACTOR + spec, PR #86 / #91) as named rows instead of leaving them to the JSON's + ``vars(spec)`` dump. + +WHAT DID NOT CHANGE (and must not, in this step) +----------------------------------------------- +Every decision rule and every threshold. ``VerdictThresholds`` defaults +(``min_abs_icir=0.30``, ``min_incremental_abs_icir=0.15``, +``min_monotonicity_spearman=0.0``, the three-part sample gate) are carried over +BYTE-FOR-BYTE as the UNVALIDATED close-era defaults they have always been +(design v3.2 §十 R24). Re-calibrating a bar is a separate pre-registered run; a +refactor that quietly moved one would make every verdict in this cycle +uninterpretable. + +IMPACT SURFACE +-------------- +Two new keys in the exported ``eval_config`` block and one new top-level key +(``eval_contract_version``) in every report JSON, plus four new provenance rows in +the Markdown. No metric, no axis input and no threshold moves. The D5 C5 +reconciliation registers this as a named, expected artifact drift. +""" + +from __future__ import annotations + +#: The evaluation contract version this package implements. Bumped ONLY with a +#: written statement of what changed (this module's docstring is that statement). +EVAL_CONTRACT_VERSION = "1.0" + +#: The identity fields a v1.0 report must be able to state about itself. Named +#: here once so the config validator, the renderer and the cross-basis summary +#: guard all refer to ONE list instead of three spellings of it. +IDENTITY_FIELDS: tuple[str, ...] = ("view", "return_basis") + +#: How an absent book is rendered. A with-book run whose book view is unknown and a +#: no-book run must not look alike, so the absence is WORDED, never left blank. +NO_BOOK = "none (no book supplied)" + + +def basis_identity_phrase( + view: str, return_basis: str, book_view: str | None +) -> str: + """The ONE sentence that states a report's information sets + return basis. + + Author-once (#76/#78/#82): the provenance row, the cross-basis summary header + and any prose that needs to say which basis a number belongs to COMPOSE this + string rather than each writing their own. A regex cannot assert that no other + sentence says this; "there is no other sentence" can. + + ``book_view`` is part of the sentence rather than an optional addendum: a + with-book evaluation carries TWO information sets, and a phrase that mentions + only the subject's would be the same one-field-for-two-facts problem the field + was added to fix (see ``EvalConfig.book_view``). + """ + return ( + f"view={view} x return_basis={return_basis}, " + f"book_view={book_view or NO_BOOK}" + ) + + +__all__ = ["EVAL_CONTRACT_VERSION", "IDENTITY_FIELDS", "basis_identity_phrase"] diff --git a/analytics/eval/render.py b/analytics/eval/render.py index a0d53a1..e1ed9a2 100644 --- a/analytics/eval/render.py +++ b/analytics/eval/render.py @@ -21,6 +21,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING +from analytics.eval.contract import EVAL_CONTRACT_VERSION, basis_identity_phrase from analytics.eval.sections import MANDATORY_SECTIONS, Skipped from data.quality.report import clean_value, sanitize_text @@ -150,6 +151,10 @@ def report_to_dict(report: FactorEvalReport) -> dict: sections.append(dict(sorted(entry.items()))) return { "criteria_source": report.criteria_source, + # Contract v1.0 (analytics/eval/contract.py): stated in the machine-readable + # record so a stored verdict can be matched to the contract that produced it + # — the #74 lesson that a verdict is only interpretable against its contract. + "eval_contract_version": EVAL_CONTRACT_VERSION, "eval_config": sanitize_payload(vars(report.cfg)), "schema_version": report.SCHEMA_VERSION, "sections": sections, @@ -219,6 +224,21 @@ def render_report(report: FactorEvalReport, mandatory: tuple[str, ...]) -> str: return "\n".join(lines).rstrip() + "\n" +def _requires_row(spec) -> str: + """``spec.requires`` as ``source.field`` pairs, sorted, or an explicit marker. + + The JSON's ``vars(spec)`` dump renders these through ``clean_value``'s ``str()`` + fallback, i.e. as dataclass REPRs — readable by accident, not by design. The + provenance box states them as data (contract v1.0), and an EMPTY declaration is + marked rather than rendered as a blank row: "this factor declares no endpoint + input" is a fact a reader must not have to infer from whitespace. + """ + fields = tuple(spec.requires or ()) + if not fields: + return "(none declared)" + return ", ".join(sorted(f"{f.source}.{f.field}" for f in fields)) + + def _provenance_rows(report: FactorEvalReport) -> list[tuple[str, str]]: spec, cfg = report.spec, report.cfg rows: list[tuple[str, object]] = [ @@ -227,6 +247,18 @@ def _provenance_rows(report: FactorEvalReport) -> list[tuple[str, str]]: ("family", spec.family), ("hypothesis (expected IC sign, fixed pre-run)", f"{spec.expected_ic_sign:+d}"), ("horizon / return basis", f"h={spec.forward_return_horizon} / {spec.return_basis}"), + # -- contract v1.0 identity + factor declarations (D1 contract v1.0/v1.1) -- + ( + "evaluation contract", + f"v{EVAL_CONTRACT_VERSION}, " + + basis_identity_phrase(cfg.view, cfg.return_basis, cfg.book_view), + ), + ("requires (endpoint inputs)", _requires_row(spec)), + ( + "adjustment / overnight boundary", + f"{spec.adjustment} / {spec.overnight_boundary}", + ), + ("lookback depth (trailing trading days)", spec.lookback_depth), ("input fields", ", ".join(spec.input_fields)), ("price adjust / warm-up bars", f"{spec.price_adjust} / {spec.min_history_bars}"), ("universe", f"{cfg.universe} (PIT={cfg.universe_is_pit})"), diff --git a/analytics/eval/summary.py b/analytics/eval/summary.py new file mode 100644 index 0000000..2ae2ec6 --- /dev/null +++ b/analytics/eval/summary.py @@ -0,0 +1,97 @@ +"""Cross-basis verdict summaries: basis/view are NON-OMITTABLE columns (R24). + +A single report states its own information set and return basis in its provenance +box. A SUMMARY does not: a table listing eleven factors' verdicts side by side is +the surface where a close-era "Watch" and an exec-era "Watch" get read as the same +claim, because nothing on the row says they are not. Design v3.2 §3.6 R24 makes +the rule structural: any multi-verdict rendering carries ``basis`` and ``view`` +columns, and a caller that omits them gets a readable error instead of a table. + +This is the summary-level form of the #76 lesson ("a rendering that does not state +its provenance"), and it is deliberately a REFUSAL rather than a default: filling +in a missing basis would be the silent degradation the project forbids, and +guessing it from the majority of the rows is worse than refusing. + +Layering: pure formatting over plain mappings — no pandas, no qt, no store. The +row dicts come from whoever assembled them (a registry read, a directory of report +JSONs); this module only decides what a table is allowed to omit. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +from analytics.eval.contract import IDENTITY_FIELDS + +#: Columns every cross-basis summary MUST carry. Same tuple the contract declares +#: as a report's identity fields — one list, not two spellings (author once). +REQUIRED_SUMMARY_COLUMNS: tuple[str, ...] = IDENTITY_FIELDS + +_MISSING = "—" + + +def require_basis_columns(columns: Sequence[str]) -> tuple[str, ...]: + """Return ``columns`` as a tuple, or raise naming the omitted identity column. + + Split out from the renderer so a caller that builds a table some other way + (HTML, a notebook) can enforce the same rule without re-deriving it. + """ + resolved = tuple(str(c) for c in columns) + missing = [c for c in REQUIRED_SUMMARY_COLUMNS if c not in resolved] + if missing: + raise ValueError( + f"a cross-basis verdict summary must carry the identity column(s) " + f"{missing}: a verdict from one view/return-basis is a DIFFERENT " + f"statistical claim from the same word under another, and a table that " + f"omits the column invites them to be read as one (design v3.2 §3.6 " + f"R24). Add the column — the summary will not infer or default it." + ) + return resolved + + +def _cell(row: Mapping[str, object], column: str) -> str: + value = row.get(column, None) + return _MISSING if value is None else str(value) + + +def render_verdict_summary( + rows: Sequence[Mapping[str, object]], + *, + columns: Sequence[str], + title: str | None = None, +) -> str: + """A deterministic Markdown table of many verdicts, basis/view guaranteed present. + + ``columns`` fixes the column order (so a diff of two summaries is a diff of the + data). Every row must SUPPLY the identity columns as non-empty values: declaring + the column and then leaving it blank would satisfy the letter of R24 and defeat + its purpose, so it is rejected with the same readable error shape. + """ + resolved = require_basis_columns(columns) + for index, row in enumerate(rows): + blank = [ + c + for c in REQUIRED_SUMMARY_COLUMNS + if not str(row.get(c, "") or "").strip() + ] + if blank: + raise ValueError( + f"cross-basis summary row {index} ({row.get('factor_id', '?')!r}) " + f"leaves the identity column(s) {blank} empty. Declaring the column " + f"and not filling it is the same omission R24 forbids." + ) + lines: list[str] = [] + if title: + lines += [f"### {title}", ""] + lines.append("| " + " | ".join(resolved) + " |") + lines.append("|" + "|".join("---" for _ in resolved) + "|") + for row in rows: + lines.append("| " + " | ".join(_cell(row, c) for c in resolved) + " |") + return "\n".join(lines) + "\n" + + +__all__ = [ + "REQUIRED_SUMMARY_COLUMNS", + "render_verdict_summary", + "require_basis_columns", +] diff --git a/docs/factors/d5_runner_difference_catalogue.md b/docs/factors/d5_runner_difference_catalogue.md index 83dcdd9..35bf1ed 100644 --- a/docs/factors/d5_runner_difference_catalogue.md +++ b/docs/factors/d5_runner_difference_catalogue.md @@ -265,3 +265,29 @@ import 的 runner loader，即便统一 runner 已上线）、**要么显式记 > 新字段自动带进 JSON，而这四个字段是 PR #86（契约 v1.0）与 PR #91（v1.1 `lookback_depth`） > 加的。`requires` 经 `clean_value` 的 `str()` 兜底渲染成 **repr 字符串列表**。 > **属预期漂移，非回归**——但若不预先登记，会在对账时被当成 D5 引入的差异去追。 + +### 七之二、C3（评估契约 v1.0）引入的 artifact 漂移 —— 同样预先登记 + +C3 升级 `analytics/eval` 契约至 v1.0（版本陈述见 `analytics/eval/contract.py` 的模块 +docstring，按 #74 先例）。它改变 artifact 的**三处且仅三处**，全部为**新增**，零删除、 +零数值变化： + +| # | 位置 | 变化 | 成因 | +|---|---|---|---| +| 1 | JSON `eval_config` 块 | **+3 键** `view` / `return_basis` / `book_view` | `report_to_dict` 的 `sanitize_payload(vars(report.cfg))` 自动带上 `EvalConfig` 的新字段（与 §七 的 `spec` 16→20 完全同一机制） | +| 2 | JSON 顶层 | **+1 键** `eval_contract_version` | 显式写入：一个 verdict 只有对着产生它的契约版本才可解释（#74 的教训） | +| 3 | Markdown `## 0. Header & Provenance` | **+4 行** `evaluation contract` / `requires (endpoint inputs)` / `adjustment / overnight boundary` / `lookback depth ...` | R24 的身份字段 + D1 契约 v1.0/v1.1 的三个声明维，从 `vars(spec)` 的 repr 转述升级为具名行 | + +⚠️ **`book_view` 使 exec 侧 no_book 与 with_book 两份 artifact 的 `eval_config` 块首次不同**（`null` vs `"close"`）。这是**有意**的：一次 with-book 评估携带**两个**信息集（候选因子的与因子簿的），一个 `view` 字段表达不了；设计 §1.1 记录的因子簿 close-view 活缺陷要到 D7 才关，在那之前诚实的 artifact 就该写 `view=decision, book_view=close`。对账时**不要**把这一处不同当作 no_book/with_book 之间的回归。 + +⚠️ **`jump_amount_corr_20` 的 exec 评估现在会 loud raise**（`qt.exec_basis_eval.exec_identity`）：它的值是 close 视图（compute 无 14:50 截断，实测），(close, exec_to_exec) 非法配对。这是**故意的阻断**——在该因子被截断修正之前，它没有诚实的 exec artifact；而写一份声明 `view=decision` 的假 artifact 正是本字段要防的事。修正由**独立的 correctness-fix PR** 处理；它落地时从 `factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE` 移除该条即可解除阻断。 + +**没有变的**（`tests/test_eval_contract_v1.py` 逐值钉住）：`VerdictThresholds` 的**每一个** +默认门（`min_abs_icir=0.30` / `min_incremental_abs_icir=0.15` / +`min_monotonicity_spearman=0.0` / 三段样本门），以及三轴判决 / 非对称门 / +unknown-never-convicts / N_eff CI / exploratory 封顶 Watch 的全部规则。设计 §十 R24 明令 +**禁止**在被评判的 run 上调门；门槛标定要单独预注册 run。 + +⚠️ 因此 C5 对账在**逐字节**层面对 22 份 md/json 一定不成立，能成立的是 **IC / ICIR / +分位价差 / verdict 的值级对账**加上「JSON 差异恰为上表三项 + §七 的 spec 四键」这个 +**结构性**断言。把「逐字节相同」当作 C5 的通过条件会让人在这里误判为回归。 diff --git a/factors/compute/minute/binding.py b/factors/compute/minute/binding.py index 169b397..13f02b0 100644 --- a/factors/compute/minute/binding.py +++ b/factors/compute/minute/binding.py @@ -164,6 +164,17 @@ class MinuteStreamBinding: per_symbol: Callable[[Factor, pd.DataFrame], pd.DataFrame] combine: Callable[[Factor, pd.DataFrame], pd.Series] + #: OPTIONAL per-symbol DAY-LEVEL diagnostics (D5): the gate-attrition frame a + #: factor's scarcity disclosure reduces. Only the factors that HAVE such a + #: disclosure set it; ``None`` means "this factor publishes no per-day + #: diagnostics", which the caller must state rather than infer. + #: + #: It is a SEPARATE callable rather than an extra ``per_symbol`` output on + #: purpose: ``per_symbol`` is the value path D4b verified cell-for-cell, and a + #: diagnostics request must not be able to change it. The cost is that asking + #: for diagnostics re-runs the per-symbol math on bars ALREADY in memory (no + #: extra I/O, no effect on any value); that cost is disclosed at the call site. + per_symbol_diagnostics: Callable[[Factor, pd.DataFrame], pd.DataFrame] | None = None def _pure_stream(compute_fn) -> MinuteStreamBinding: @@ -190,6 +201,41 @@ def _combine(factor: Factor, stats: pd.DataFrame) -> pd.Series: return MinuteStreamBinding(per_symbol=_per_symbol, combine=_combine) +def _pure_stream_with_diagnostics(compute_fn) -> MinuteStreamBinding: + """:func:`_pure_stream` plus the compute function's day-level diagnostics sink.""" + base = _pure_stream(compute_fn) + return MinuteStreamBinding( + per_symbol=base.per_symbol, + combine=base.combine, + per_symbol_diagnostics=_sink_diagnostics(compute_fn), + ) + + +def _sink_diagnostics(compute_fn) -> Callable[[Factor, pd.DataFrame], pd.DataFrame]: + """Per-symbol diagnostics via the compute function's own ``diagnostics_out`` sink. + + The three ridge/peak-family factors already expose the day-level gate-attrition + frame their scarcity disclosure reduces; this only routes it. The frame is + returned CONCATENATED and the symbol column is left exactly as the compute + function wrote it — the summarizers read it, and re-labelling here would be a + second place for the schema to drift. + """ + + def _call(factor: Factor, bars: pd.DataFrame) -> pd.DataFrame: + sink: list[pd.DataFrame] = [] + compute_fn( + bars, + lookback_days=factor.lookback_days, # type: ignore[attr-defined] + name=factor.name, + diagnostics_out=sink, + ) + if not sink: + return pd.DataFrame() + return pd.concat(sink, ignore_index=False) + + return _call + + def _amp_cut_per_symbol(factor: Factor, bars: pd.DataFrame) -> pd.DataFrame: """Steps 1-3 of ``intraday_amp_cut``: the ``(v_mean, v_std)`` intermediate.""" return compute_amp_cut_stats( @@ -219,9 +265,14 @@ def _amp_cut_combine(factor: Factor, stats: pd.DataFrame) -> pd.Series: ), PeakIntervalKurtosisFactor: _pure_stream(compute_peak_interval_kurtosis), ValleyRelativeVwapFactor: _pure_stream(compute_valley_relative_vwap), - ValleyRidgeVwapRatioFactor: _pure_stream(compute_valley_ridge_vwap_ratio), - RidgeMinuteReturnFactor: _pure_stream(compute_ridge_minute_return), - PeakRidgeAmountRatioFactor: _pure_stream(compute_peak_ridge_amount_ratio), + # The three factors that publish a per-day gate-attrition disclosure. + ValleyRidgeVwapRatioFactor: _pure_stream_with_diagnostics( + compute_valley_ridge_vwap_ratio + ), + RidgeMinuteReturnFactor: _pure_stream_with_diagnostics(compute_ridge_minute_return), + PeakRidgeAmountRatioFactor: _pure_stream_with_diagnostics( + compute_peak_ridge_amount_ratio + ), } #: The factors whose cross-sectional combine is NOT the identity — i.e. the ones @@ -337,6 +388,30 @@ def is_valid_day_pooled(factor: Factor) -> bool: ) +#: MEASURED exceptions: minute factors whose DAY-d VALUE depends on bars AFTER the +#: 14:50 decision cutoff, i.e. whose compute applies no cutoff of its own and must +#: therefore be handed pre-truncated bars to be decision-view. Under the +#: ``exec_to_exec`` basis a value like that uses information from after its own +#: 14:51 entry anchor, so an evaluation of it CANNOT honestly declare +#: ``view=decision``. +#: +#: This is a DENY LIST OF MEASURED FACTS, never a proof of safety: a factor absent +#: from it has either been measured clean or never measured. +#: ``tests/test_decision_cutoff_visibility.py`` is what does the measuring (perturb +#: only the post-cutoff bars, assert the pre-cutoff rows byte-identical, ask whether +#: the value moves) and it pins this set to exactly its own contents, so a second +#: offender cannot join silently. +#: +#: Declared HERE, next to the other minute-factor partitions, because it is a fact +#: about a factor's values — not about the evaluation path that has to consult it. +NOT_DECISION_CUTOFF_SAFE: frozenset[type[Factor]] = frozenset({JumpAmountCorrFactor}) + + +def is_decision_cutoff_safe(factor: Factor) -> bool: + """True iff ``factor``'s value at d uses no bar after d's 14:50 cutoff.""" + return type(factor) not in NOT_DECISION_CUTOFF_SAFE + + def is_minute_bound(factor: Factor) -> bool: """True iff ``factor`` has a bars-only raw-compute binding here.""" return type(factor) in _MINUTE_BINDINGS @@ -388,6 +463,24 @@ def minute_stats_from_bars(factor: Factor, bars: pd.DataFrame) -> pd.DataFrame: return _stream_binding(factor).per_symbol(factor, bars) +def has_minute_diagnostics(factor: Factor) -> bool: + """True iff ``factor`` publishes a per-symbol day-level diagnostics frame.""" + return _stream_binding(factor).per_symbol_diagnostics is not None + + +def minute_diagnostics_from_bars(factor: Factor, bars: pd.DataFrame) -> pd.DataFrame: + """``factor``'s per-symbol day-level diagnostics frame (empty when it has none). + + Returns an EMPTY frame — not an error — for a factor without a diagnostics + binding, so a caller can ask uniformly; whether the factor HAS one is the + separate, explicit :func:`has_minute_diagnostics` question, and a caller that + needs the disclosure must check it rather than read an empty frame as "no + attrition". + """ + fn = _stream_binding(factor).per_symbol_diagnostics + return pd.DataFrame() if fn is None else fn(factor, bars) + + def combine_minute_stats(factor: Factor, stats: pd.DataFrame) -> pd.Series: """``factor``'s daily raw Series from the ASSEMBLED per-symbol intermediates. @@ -405,14 +498,18 @@ def is_cross_sectional_minute(factor: Factor) -> bool: __all__ = [ "CROSS_SECTIONAL_MINUTE_FACTORS", + "NOT_DECISION_CUTOFF_SAFE", "STATS_VALUE_COL", "VALID_DAY_POOLED_FACTORS", "BindingFn", "MinuteStreamBinding", "combine_minute_stats", + "has_minute_diagnostics", "is_cross_sectional_minute", + "is_decision_cutoff_safe", "is_minute_bound", "is_valid_day_pooled", + "minute_diagnostics_from_bars", "minute_raw_from_bars", "minute_stats_from_bars", "pooled_baseline_days", diff --git a/factors/materialize.py b/factors/materialize.py index 8114466..8fa4ea0 100644 --- a/factors/materialize.py +++ b/factors/materialize.py @@ -64,6 +64,7 @@ combine_minute_stats, is_minute_bound, is_valid_day_pooled, + minute_diagnostics_from_bars, minute_raw_from_bars, minute_stats_from_bars, pooled_baseline_days, @@ -250,6 +251,7 @@ def materialize_range( sources: MaterializeSources, decision_cutoff: str = DEFAULT_DECISION_TIME, warmup: int | None = None, + diagnostics: list | None = None, ) -> pd.Series: """Compute ``factor``'s RAW values for ``view`` over ``[emit_start, emit_end]``. @@ -257,6 +259,13 @@ def materialize_range( applies the view lag, trims to ``warmup`` (= ``spec.lookback_depth``) trailing trading days before ``emit_start``, computes, and returns the emit-window rows. Single-date and batch calls give bit-identical values (design §3.5 P8). + + ``diagnostics``: OPTIONAL list the per-symbol day-level gate-attrition frames + are appended to (emit-window rows only), for the factors that publish a + scarcity disclosure. Default ``None`` = the D4b-verified value path, byte for + byte: nothing extra is computed and nothing is collected. Supplying a sink + NEVER changes a returned value — it re-runs the per-symbol math on bars already + in memory (no extra I/O) purely to observe the day-level counts. """ resolved_view = View(view) symbols = _requested_universe(symbols) # one engine-level normalization @@ -270,9 +279,15 @@ def materialize_range( if is_minute_factor(factor): raw = _materialize_minute( factor, resolved_view, symbols, load_start, emit_start, emit_end, w, - sources, decision_cutoff, + sources, decision_cutoff, diagnostics, ) else: + if diagnostics is not None: + raise ValueError( + f"{factor.name} is a daily factor; per-day minute diagnostics are " + f"only published by minute factors. Refusing to return an EMPTY " + f"sink, which a caller would read as 'no attrition'." + ) raw = _materialize_daily( factor, resolved_view, symbols, load_start, emit_start, emit_end, w, sources, ) @@ -300,7 +315,8 @@ def _materialize_daily( def _materialize_minute( - factor, view, symbols, load_start, emit_start, emit_end, warmup, sources, decision_cutoff, + factor, view, symbols, load_start, emit_start, emit_end, warmup, sources, + decision_cutoff, diagnostics=None, ) -> pd.Series: """STREAMING minute materialization (D4b): one symbol's bars at a time. @@ -347,12 +363,12 @@ def _materialize_minute( if pooled: stats = _pooled_symbol_stats( factor, view, sym, emit_start, emit_end, warmup, sources, - decision_cutoff, **pooled_kw, + decision_cutoff, diagnostics=diagnostics, **pooled_kw, ) else: stats = _bounded_symbol_stats( factor, view, sym, load_start, emit_start, emit_end, warmup, sources, - decision_cutoff, + decision_cutoff, diagnostics=diagnostics, ) if stats is None or stats.empty: continue @@ -385,8 +401,30 @@ def _symbol_bars(provider, symbol: str, start, end) -> pd.DataFrame: return bars if bool(mine.all()) else bars[mine] +def _collect_diagnostics(factor, work, emit_start, emit_end, diagnostics) -> None: + """Append ``factor``'s emit-window per-day diagnostics for ONE symbol, if asked. + + ``work`` is the SAME cutoff-filtered, trimmed frame the value path used, so the + counts describe the days the values were computed from — not a differently + loaded window. Restricting to the emit window matches the disclosure's meaning + ("the days this run scored"); the warm-up days ahead of it were loaded to make + those days computable and were never part of the reported denominator. + """ + if diagnostics is None: + return + frame = minute_diagnostics_from_bars(factor, work) + if frame.empty: + return + index = pd.DatetimeIndex(frame.index) + within = (index >= emit_start) & (index <= emit_end) + kept = frame[within] + if not kept.empty: + diagnostics.append(kept) + + def _bounded_symbol_stats( - factor, view, symbol, load_start, emit_start, emit_end, warmup, sources, decision_cutoff, + factor, view, symbol, load_start, emit_start, emit_end, warmup, sources, + decision_cutoff, *, diagnostics=None, ) -> pd.DataFrame | None: """One BOUNDED minute factor's per-symbol intermediate over the fixed window. @@ -403,12 +441,13 @@ def _bounded_symbol_stats( if view is View.DECISION: bars = minute_decision_cutoff(bars, decision_time=decision_cutoff) bars = _trim_minute(bars, emit_start, warmup) + _collect_diagnostics(factor, bars, emit_start, emit_end, diagnostics) return minute_stats_from_bars(factor, bars) def _pooled_symbol_stats( factor, view, symbol, emit_start, emit_end, warmup, sources, decision_cutoff, - *, floor, baseline_days, lookback_days, + *, floor, baseline_days, lookback_days, diagnostics=None, ) -> pd.DataFrame | None: """Saturation-expanding load for ONE symbol of a valid-day-pooled factor. @@ -482,6 +521,10 @@ def _pooled_symbol_stats( baseline_days=baseline_days, lookback_days=lookback_days, symbols=[symbol], ): + # Only the TERMINAL load is observed: the intermediate expansion steps + # are the search, not the computation, and reporting their day counts + # would inflate the disclosure with days that were re-derived deeper. + _collect_diagnostics(factor, work, emit_start, emit_end, diagnostics) return stats load_start = load_start - chunk # Never silently return an UNSATURATED result (red line #9: no silent diff --git a/factors/service.py b/factors/service.py index 8d0dc1b..3ad577d 100644 --- a/factors/service.py +++ b/factors/service.py @@ -80,6 +80,7 @@ def _ensure_coverage( sources: MaterializeSources, view: View, cutoff: str, + diagnostics: list | None = None, ) -> pd.Series: """Read-through: fill any date the store lacks for ``factor``, return the Series. @@ -89,6 +90,15 @@ def _ensure_coverage( (possibly empty). The per-date fill (dates = one) and the batch fill (dates = many) leave BIT-IDENTICAL stores because the materializer's trailing trim is warmup-consistent (design §3.5 P8). + + ``diagnostics``: when a sink is supplied the WHOLE requested range is + materialized even on a full store hit. The store persists VALUES, not the + day-level gate-attrition counts a scarcity disclosure reduces, so a warm store + can serve the values and cannot serve the disclosure. The two honest options + were "recompute" and "publish the report without the section"; the second is + the silent degradation this project forbids, so the cost is paid and stated. + Values are unaffected either way (the engine is deterministic in code + data, + which is exactly what the store key asserts). """ stored = store.read_valid(key, expected_fingerprint=fingerprint) have = ( @@ -96,7 +106,7 @@ def _ensure_coverage( if stored is not None and not stored.empty else pd.DatetimeIndex([]) ) - missing = dates[~dates.isin(have)] + missing = dates if diagnostics is not None else dates[~dates.isin(have)] if len(missing): fresh = materialize_range( factor, @@ -106,6 +116,7 @@ def _ensure_coverage( emit_end=missing.max(), sources=sources, decision_cutoff=cutoff, + diagnostics=diagnostics, ) if not fresh.empty: store.upsert(key, fresh, fingerprint=fingerprint) @@ -166,17 +177,30 @@ def panel( view: object = View.DECISION, basis: object = ReturnBasis.EXEC_TO_EXEC, params_by_id: Mapping[str, Mapping[str, object]] | None = None, + diagnostics: list | None = None, ) -> pd.DataFrame: """Raw factor values for ``factor_ids`` over ``decisions`` x ``universe``. Read-through: any decision date the store lacks is materialized once (batch) and appended. The view x basis pairing is enforced up front. + + ``diagnostics``: optional per-day gate-attrition sink for the ONE factor whose + scarcity disclosure needs it — see :func:`_ensure_coverage`. Requesting it for + several factors at once is refused rather than served: the frames from two + factors would land in one list with nothing to tell them apart, and a + summarizer would silently reduce the mixture. """ resolved_view, _ = require_legal_pairing(view, basis) factor_ids = list(factor_ids) symbols = [str(s) for s in universe] if not decisions: raise ValueError("panel() needs at least one DecisionPoint.") + if diagnostics is not None and len(factor_ids) != 1: + raise ValueError( + f"panel(diagnostics=...) takes exactly ONE factor_id (got " + f"{factor_ids}): the per-day diagnostics frames carry no factor label, " + f"so several factors' frames in one sink would be reduced together." + ) cutoff = _uniform_cutoff(decisions) dates = pd.DatetimeIndex(sorted({pd.Timestamp(d.date).normalize() for d in decisions})) @@ -188,6 +212,7 @@ def panel( series_by_id[fid] = _ensure_coverage( factor, key, fp, dates=dates, symbols=symbols, store=store, sources=sources, view=resolved_view, cutoff=cutoff, + diagnostics=diagnostics, ) return _assemble(factor_ids, series_by_id, dates, symbols) diff --git a/qt/exec_basis_eval.py b/qt/exec_basis_eval.py index e829aa9..ef9e65c 100644 --- a/qt/exec_basis_eval.py +++ b/qt/exec_basis_eval.py @@ -30,7 +30,7 @@ from __future__ import annotations import time -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path import pandas as pd @@ -41,6 +41,9 @@ FactorEvalReport, StandardFactorEvaluator, ) +from data.availability_policy import ReturnBasis, View +from factors import registry as factor_registry +from factors.compute.minute.binding import is_decision_cutoff_safe from analytics.eval.figures import render_factor_dashboard from analytics.factor import forward_returns as _close_forward_returns from data.clean.schema import DATE_LEVEL @@ -151,6 +154,62 @@ def build_disclosure( return disclosure +def subject_view(factor_id: str) -> str: + """The information-set view the factor's OWN values were taken under. + + DERIVED from a measured fact, not written down: a minute factor whose compute + applies no 14:50 truncation of its own was handed full-day bars and is therefore + on the CLOSE view, whatever the run intends. The deny list lives with the + factors (``factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE``) and is + pinned by the perturb-the-post-cutoff-bars test; an id the registry does not + know (a synthetic spec) is not on that list and is treated as decision-view. + """ + try: + factor = factor_registry.build(factor_id) + except Exception: # noqa: BLE001 - an unregistered id has no measured exception + return View.DECISION.value + return View.DECISION.value if is_decision_cutoff_safe(factor) else View.CLOSE.value + + +def exec_identity( + eval_cfg: EvalConfig, *, factor_id: str, book_view: str | None +) -> EvalConfig: + """The exec-basis twin of ``eval_cfg`` — the identity DERIVED, not asserted. + + The caller hands ONE config to its close reports and to this module, so the + exec identity has to be restated here or the exec artifact inherits the close + label. ``return_basis`` is the one constant (it is this module's entire + purpose); ``view`` comes from :func:`subject_view` and ``book_view`` from the + caller, who is the only one who knows how the book was built. + + A factor that is not decision-cutoff-safe yields the pairing + (close, exec_to_exec), which ``EvalConfig`` refuses — and refusing is correct: + such a factor's values use information from after their own 14:51 entry anchor, + so no honest exec-basis artifact of it exists until that is fixed. The refusal + is re-raised with the specific reason rather than left as the generic pairing + message, which would send the reader looking for a config typo. + """ + view = subject_view(factor_id) + try: + return replace( + eval_cfg, + view=view, + return_basis=ReturnBasis.EXEC_TO_EXEC.value, + book_view=book_view, + ) + except ValueError as exc: + raise ValueError( + f"cannot build an exec_to_exec evaluation of {factor_id!r}: its values " + f"are on the {view!r} view because its compute applies no 14:50 " + f"truncation of its own (measured — see " + f"factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE and " + f"tests/test_decision_cutoff_visibility.py), so the day-d value uses " + f"bars from after its own 14:51 entry anchor. Declaring view=decision " + f"anyway would be the report stating a check it did not do; the " + f"correctness fix is tracked separately." + ) from exc + + def _write_report(report: FactorEvalReport, report_dir: Path, stem: str) -> tuple[Path, Path]: md_path = report_dir / f"{stem}.md" json_path = report_dir / f"{stem}.json" @@ -200,16 +259,32 @@ def run_exec_basis_evaluation( report_dir: Path, stem: str, force_rebuild: bool = False, + book_view: str = View.CLOSE.value, ) -> ExecBasisEvaluation: """Build the exec-to-exec returns, sanity-check them, evaluate twice, report. ``panel`` is the front-adjusted daily panel (it carries the raw ``adj_factor`` the adjustment identity needs and defines the shared price grid); ``spec`` is the factor's DAILY spec, from which the intraday variant is derived. + + ``book_view`` declares which information set ``book`` was built from. It + defaults to ``close`` because that is what every current caller does — the + confirmed book takes close(d) values and the evening daily_basic(d), the live + defect design §1.1 records and D7 closes. The decision-view book path passes + ``decision`` explicitly. Only the caller knows this, so it is a parameter + rather than something guessed here. """ started = time.monotonic() params = ExecBasisParams.from_config(cfg) exec_spec = intraday_spec_variant(spec, params) + # Contract v1.0: the identity travels with the basis, exactly as the spec's does + # one line above — and the two runs below carry DIFFERENT identities, because + # the no-book run has no book and the with-book run has a close-view one. One + # shared config would have to pick one of those and be wrong about the other. + cfg_no_book = exec_identity(eval_cfg, factor_id=spec.factor_id, book_view=None) + cfg_with_book = exec_identity( + eval_cfg, factor_id=spec.factor_id, book_view=book_view + ) horizon = spec.forward_return_horizon prices = build_exec_price_panel( @@ -305,7 +380,7 @@ def run_exec_basis_evaluation( execution_capacity=disclosure, ) report_no_book, ir_no_book = evaluator.evaluate_with_ir( - factor_panel, exec_spec, eval_cfg, ctx_no_book + factor_panel, exec_spec, cfg_no_book, ctx_no_book ) ctx_with_book = EvalContext( forward_returns=exec_returns, @@ -315,7 +390,7 @@ def run_exec_basis_evaluation( execution_capacity=disclosure, ) report_with_book, ir_with_book = evaluator.evaluate_with_ir( - factor_panel, exec_spec, eval_cfg, ctx_with_book + factor_panel, exec_spec, cfg_with_book, ctx_with_book ) nb_md, nb_json = _write_report(report_no_book, report_dir, f"{stem}_exec_no_book") @@ -393,6 +468,8 @@ def format_exec_basis_line(result: ExecBasisEvaluation) -> str: __all__ = [ "ExecBasisEvaluation", "build_disclosure", + "exec_identity", "format_exec_basis_line", "run_exec_basis_evaluation", + "subject_view", ] diff --git a/qt/panel_leg_probe.py b/qt/panel_leg_probe.py new file mode 100644 index 0000000..b6d8cc9 --- /dev/null +++ b/qt/panel_leg_probe.py @@ -0,0 +1,196 @@ +"""D5 C5 fourth-leg PROBE: the new engine's panel vs the frozen D1 baseline. + +Not the acceptance run — a MEASUREMENT taken before committing to one, in the same +spirit as ``qt.saturation_probe`` (which is why the multi-hour D5 run was not +started blind). It answers ONE question the C5 plan needs an answer to first: + + how big, and of what shape, is the difference between the values the D4/D4b + materializer produces on the real evaluation plane and the D1 frozen panel the + fourth leg compares against? + +The question matters because the two are NOT expected to agree cell for cell, and +the reason is structural rather than numerical. The frozen panel was produced by the +old runners' loaders, which read exactly ``[data.start, data.end]`` and did NO +warm-up extension (``docs/factors/d5_runner_difference_catalogue.md`` §四). The +materializer loads ``lookback_depth`` trailing trading days of warm-up for a bounded +factor and expands to saturation for a valid-day-pooled one. So the early part of +the window is structurally under-warmed in the BASELINE and correctly warmed in the +new engine, and the fourth leg's "relative <= 1e-12" phrasing cannot apply to it. + +What this probe reports, per factor: + +* how many cells differ, and the NaN-set changes split by direction (NaN -> finite is + the warm-up being filled in; finite -> NaN would be the alarming direction); +* the difference profile BY MONTH — the warm-up explanation predicts a front-loaded + profile decaying to zero, and a difference that does NOT decay is a different + animal that needs its own attribution; +* the residual on the cells both sides call finite OUTSIDE the early region, which is + where the "float re-association only" claim can actually be tested. + +Cache-only (``stk_mins_live_calls`` is structurally 0: the store read has no fetch +closure). Resumable: results are appended per factor to ``--out`` as JSON lines and a +re-run skips factors already present unless ``--force``. + +Run (needs the gitignored ``artifacts/`` tree — see +``docs/factors/d5_exec_baseline_freeze.md`` for the symlink note): +``python -m qt.panel_leg_probe --factors jump_amount_corr_20`` +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +from data.availability_policy import View +from factors import registry as factor_registry +from factors.materialize import MaterializeSources, materialize_range +from qt.factor_hotpath_smoke import CacheMinuteProvider + +DEFAULT_CACHE_ROOT = "artifacts/cache/tushare/v1" +DEFAULT_PANEL_DIR = "artifacts/refactor_baseline/panels" +EVAL_START = pd.Timestamp("2021-07-01") +EVAL_END = pd.Timestamp("2026-06-30") + + +def frozen_panel(panel_dir: str, factor_id: str) -> pd.Series: + """The frozen D1 baseline as a ``MultiIndex(date, symbol)`` Series.""" + frame = pd.read_parquet(Path(panel_dir) / f"{factor_id}.parquet") + return ( + frame.set_index(["date", "symbol"])[factor_id] + .sort_index(kind="mergesort") + ) + + +def compare(factor_id: str, symbols: list[str], cache_root: str, panel_dir: str) -> dict: + """Materialize ``factor_id`` on the real plane and profile the difference.""" + base = frozen_panel(panel_dir, factor_id) + provider = CacheMinuteProvider(cache_root) + started = time.monotonic() + fresh = materialize_range( + factor_registry.build(factor_id), + view=View.DECISION, + symbols=symbols, + emit_start=EVAL_START, + emit_end=EVAL_END, + sources=MaterializeSources(minute=provider), + ).sort_index(kind="mergesort") + wall = time.monotonic() - started + + joined = pd.DataFrame({"base": base}).join( + pd.DataFrame({"fresh": fresh}), how="outer" + ) + b, f = joined["base"].to_numpy(), joined["fresh"].to_numpy() + b_nan, f_nan = np.isnan(b), np.isnan(f) + both = ~b_nan & ~f_nan + filled = b_nan & ~f_nan # warm-up filled in (the expected direction) + lost = ~b_nan & f_nan # the alarming direction + # Relative difference on the cells both sides call finite. + denom = np.maximum(np.abs(b), np.abs(f)) + rel = np.zeros(len(b)) + with np.errstate(invalid="ignore", divide="ignore"): + rel[both] = np.where( + denom[both] > 0, np.abs(b[both] - f[both]) / denom[both], 0.0 + ) + differing = both & (rel > 1e-12) + + months = pd.PeriodIndex( + joined.index.get_level_values("date"), freq="M" + ).astype(str) + by_month: dict[str, dict[str, int]] = {} + for label in sorted(set(months)): + m = months == label + entry = { + "filled": int((filled & m).sum()), + "lost": int((lost & m).sum()), + "differing_finite": int((differing & m).sum()), + } + if any(entry.values()): + by_month[label] = entry + + # The residual OUTSIDE the first year, where warm-up cannot be the explanation. + late = joined.index.get_level_values("date") >= EVAL_START + pd.DateOffset(years=1) + late_both = both & late + return { + "factor": factor_id, + "wall_seconds": round(wall, 1), + "live_calls": provider.live_calls, + "rows_base": int(len(base)), + "rows_fresh": int(len(fresh)), + "rows_union": int(len(joined)), + "both_finite": int(both.sum()), + "nan_filled": int(filled.sum()), + "nan_lost": int(lost.sum()), + "differing_finite": int(differing.sum()), + "max_rel_diff": float(rel[both].max()) if both.any() else None, + "max_rel_diff_after_year1": ( + float(rel[late_both].max()) if late_both.any() else None + ), + "differing_finite_after_year1": int((differing & late).sum()), + "nan_filled_after_year1": int((filled & late).sum()), + "by_month": by_month, + } + + +def _done(out_path: Path) -> set[str]: + if not out_path.exists(): + return set() + return { + json.loads(line)["factor"] + for line in out_path.read_text().splitlines() + if line.strip() + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--cache-root", default=DEFAULT_CACHE_ROOT) + parser.add_argument("--panel-dir", default=DEFAULT_PANEL_DIR) + parser.add_argument( + "--symbols-file", + required=True, + help="one symbol per line — the REQUESTED universe, which must be the " + "runner's (996 for the CSI500 plane), NOT the frozen panel's symbol list " + "(995: 300114.SZ has no minute bars and never entered the panel). The " + "distinction is load-bearing for the cross-sectional factor.", + ) + parser.add_argument("--factors", nargs="+", required=True) + parser.add_argument("--out", default="panel_leg_probe.jsonl") + parser.add_argument("--force", action="store_true") + args = parser.parse_args(argv) + + symbols = [ + s.strip() for s in Path(args.symbols_file).read_text().splitlines() if s.strip() + ] + out_path = Path(args.out) + out_path.parent.mkdir(parents=True, exist_ok=True) + done = set() if args.force else _done(out_path) + print(f"universe: {len(symbols)} symbols | emit {EVAL_START.date()}..{EVAL_END.date()}") + for factor_id in args.factors: + if factor_id in done: + print(f"{factor_id}: SKIP (already in {out_path})") + continue + print(f"{factor_id}: running...", flush=True) + record = compare(factor_id, symbols, args.cache_root, args.panel_dir) + with out_path.open("a") as fh: + fh.write(json.dumps(record) + "\n") + print( + f"{factor_id}: both_finite={record['both_finite']:,} " + f"NaN->finite={record['nan_filled']:,} finite->NaN={record['nan_lost']:,} " + f"differing_finite={record['differing_finite']:,} " + f"max_rel={record['max_rel_diff']} " + f"| after year 1: differing={record['differing_finite_after_year1']:,} " + f"filled={record['nan_filled_after_year1']:,} " + f"max_rel={record['max_rel_diff_after_year1']} " + f"| wall={record['wall_seconds']}s live_calls={record['live_calls']}", + flush=True, + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_decision_cutoff_visibility.py b/tests/test_decision_cutoff_visibility.py new file mode 100644 index 0000000..ba0f980 --- /dev/null +++ b/tests/test_decision_cutoff_visibility.py @@ -0,0 +1,196 @@ +"""Does a minute factor's day-d value depend on bars AFTER the 14:50 cutoff? + +THE PROPERTY. Under the ``exec_to_exec`` basis the entry anchor for date ``d`` is +``d``'s 14:51 execution bar (``qt.exec_forward_returns``). A factor value at ``d`` +may therefore only use information visible at 14:50; anything later is information +from after its own entry. The D0 policy table states the same rule for the minute +endpoint (``available_time <= d 14:50``), and the D4 materializer enforces it in the +DECISION view — but that enforcement arrived with the materializer, and the eleven +factors' published values were produced by the old runners, which hand the compute +functions FULL-DAY bars and rely on each compute truncating for itself. + +THE TEST. Perturb ONLY the bars strictly after the cutoff, assert every pre-cutoff +row is byte-identical (so a failure cannot be blamed on the fixture), and ask +whether the value moves. This is the "perturb the future -> value unchanged" shape +the project already uses, aimed at the WITHIN-DAY boundary rather than the +across-day one. + +RESULT, ENCODED RATHER THAN DESCRIBED. Nine of the ten bars-bound minute factors are +clean. ``jump_amount_corr_20`` is not: its compute is the ONLY one of the eleven that +applies no decision-time truncation of its own (grep: zero ``decision_time`` / +``prepare_visible_minute_bars`` references in its module), so under the old runners +it saw 09:30-15:00 of day d. + +That is recorded as a KNOWN, NAMED exception rather than fixed. Truncating it would +change a published factor's values and could move its verdict — a definition +-affecting research decision, not a refactor's business (design v3.2 §〇). The +exception is asserted POSITIVELY, so the day someone changes it, this test goes red +and the change has to be made deliberately instead of noticed later. + +THE LIST IS PRODUCTION STATE, NOT TEST STATE. It lives in +``factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE`` because the exec path +consults it to decide whether an evaluation may declare ``view=decision`` +(``qt.exec_basis_eval.subject_view``); this file is what MEASURES it. A copy here +would be a second thing to keep in step, and the failure mode of drifting apart is +an artifact declaring an information set its values do not have. + +WHY THE EXISTING SUITE DID NOT CATCH IT — and why that is not a contradiction. +``tests/test_jump_amount_corr_factor.py::test_pit_perturbing_future_bars_does_not_ +change_factor_at_d`` is correct and passes: it perturbs day ``d+1`` and checks day +``d``, i.e. the ACROSS-DAY boundary. Its fixture sessions are six bars long, so a +14:50 boundary does not exist in them at all. The WITHIN-DAY boundary is a different +property, it only became load-bearing when PR #79 moved the entry anchor from +``close(d)`` to ``14:51(d)``, and nothing tested it. Two boundaries, two tests. + +Measured on the real cache the same way (12 CSI500 names, 2021-07..2021-12, 361,500 +bars, 4.6% of them post-cutoff): nine factors moved 0 cells; ``jump_amount_corr_20`` +moved 1,477 of 1,477 with max |diff| = 1.276 on a correlation bounded in [-1, 1]. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from data.clean.intraday_schema import normalize_intraday_bars +from factors import registry as factor_registry +from factors.compute.minute.binding import ( + _MINUTE_STREAM_BINDINGS, + NOT_DECISION_CUTOFF_SAFE, + minute_raw_from_bars, +) + +CUTOFF = "14:50:00" + +#: DERIVED from the production deny list, not spelled again here. The exec path has +#: to consult the same fact to decide whether it may declare ``view=decision`` +#: (``qt.exec_basis_eval.subject_view``), so the list lives with the factors and +#: this file MEASURES it. Two copies would be two things to keep in step. +KNOWN_POST_CUTOFF_DEPENDENT_IDS = frozenset(c().name for c in NOT_DECISION_CUTOFF_SAFE) + +SYMBOLS = [f"6000{i:02d}.SH" for i in range(12)] +DATES = pd.bdate_range("2021-01-04", periods=40) + + +def _session_times(day: pd.Timestamp) -> list[pd.Timestamp]: + """A realistic A-share session: 09:31-11:30 and 13:01-15:00 (240 one-minute bars). + + The full afternoon matters: a session that stopped before 14:50 would have no + post-cutoff bars at all and the perturbation below would be a no-op — the + unfailable-test shape this repo keeps catching. The pre-assertion in + :func:`_poison` refuses that fixture outright. + """ + morning = pd.date_range(day + pd.Timedelta("09:31:00"), periods=120, freq="1min") + afternoon = pd.date_range(day + pd.Timedelta("13:01:00"), periods=120, freq="1min") + return list(morning) + list(afternoon) + + +def _bars() -> pd.DataFrame: + rng = np.random.RandomState(5) + rows: list[tuple] = [] + for s in SYMBOLS: + for d in DATES: + price = 100.0 + rng.normal(0, 2) + for i, t in enumerate(_session_times(d)): + price += rng.normal(0, 0.05) + slot = 1e4 * (1.0 + 0.3 * np.sin(i / 12.0)) + erupt = 6.0 if (rng.rand() < 0.06) else 1.0 + vol = slot * erupt * (1.0 + 0.1 * rng.rand()) + w = 0.15 * price * (1.0 + (2.0 if erupt > 1 else 0.0)) * (0.5 + rng.rand()) + hi, lo = price + abs(w) * rng.rand(), price - abs(w) * rng.rand() + cl = lo + (hi - lo) * rng.rand() + rows.append((t, s, price, hi, lo, cl, vol, cl * vol)) + cols = ["time", "symbol", "open", "high", "low", "close", "volume", "amount"] + return normalize_intraday_bars(pd.DataFrame(rows, columns=cols), freq="1min") + + +BARS = _bars() + + +def _poison(bars: pd.DataFrame) -> pd.DataFrame: + """Scale prices/volumes of the POST-cutoff bars only; leave the rest untouched. + + Returns the perturbed frame after asserting BOTH halves of what makes the test + meaningful: the post-cutoff rows really changed, and the pre-cutoff rows did not. + """ + times = pd.DatetimeIndex(bars.index.get_level_values("time")) + after = bars["available_time"].to_numpy() > ( + times.normalize() + pd.Timedelta(CUTOFF) + ).to_numpy() + assert after.any(), "fixture has no post-cutoff bars — the perturbation is a no-op" + rng = np.random.RandomState(11) + out = bars.copy() + for col in ("open", "high", "low", "close"): + out.loc[after, col] = out.loc[after, col] * (1.0 + 0.5 * rng.rand(int(after.sum()))) + for col in ("volume", "amount"): + out.loc[after, col] = out.loc[after, col] * 7.0 + assert not out.loc[after].equals(bars.loc[after]), "perturbation changed nothing" + pd.testing.assert_frame_equal(out.loc[~after], bars.loc[~after], check_exact=True) + return out + + +def _moved_cells(factor_id: str) -> tuple[int, int, float]: + factor = factor_registry.build(factor_id) + before = minute_raw_from_bars(factor, BARS) + after = minute_raw_from_bars(factor, _poison(BARS)) + joined = pd.DataFrame({"a": before, "b": after}).dropna() + if joined.empty: + return 0, 0, 0.0 + diff = (joined["a"] - joined["b"]).abs() + return len(joined), int((diff > 1e-12).sum()), float(diff.max()) + + +BOUND_FACTOR_IDS = tuple(sorted(cls().name for cls in _MINUTE_STREAM_BINDINGS)) +CLEAN_FACTOR_IDS = tuple( + f for f in BOUND_FACTOR_IDS if f not in KNOWN_POST_CUTOFF_DEPENDENT_IDS +) + + +@pytest.mark.parametrize("factor_id", CLEAN_FACTOR_IDS) +def test_value_does_not_depend_on_bars_after_the_decision_cutoff(factor_id): + """The nine that truncate for themselves: perturbing the future moves nothing.""" + cells, moved, worst = _moved_cells(factor_id) + assert cells > 0, f"{factor_id}: no comparable cells — the test would be vacuous" + assert moved == 0, ( + f"{factor_id} moved {moved}/{cells} values (max |diff| {worst:.3e}) when ONLY " + f"post-{CUTOFF} bars changed: its day-d value uses information from after its " + f"own 14:51 entry anchor." + ) + + +def test_the_known_exception_is_still_exactly_one_factor_and_still_leaks(): + """jump_amount_corr_20: recorded as a defect, asserted positively. + + Asserted in the POSITIVE direction on purpose. A test that merely tolerated the + exception would stay green whether it was fixed, worsened, or spread to a second + factor. This one goes red on any of those, which is what forces the decision to + be taken deliberately — and because the exec path DERIVES its right to declare + ``view=decision`` from the same list, a stale entry here is not cosmetic. + """ + assert KNOWN_POST_CUTOFF_DEPENDENT_IDS == {"jump_amount_corr_20"} + for factor_id in sorted(KNOWN_POST_CUTOFF_DEPENDENT_IDS): + assert factor_id in BOUND_FACTOR_IDS + cells, moved, worst = _moved_cells(factor_id) + assert cells > 0 + assert moved > 0, ( + f"{factor_id} no longer depends on post-{CUTOFF} bars. If that was " + f"deliberate, this factor's PUBLISHED values changed: drop it from " + f"factors.compute.minute.binding.NOT_DECISION_CUTOFF_SAFE, and re-state " + f"its verdict rather than letting the artifacts drift silently." + ) + assert worst > 0.0 + + +def test_every_clean_factor_is_absent_from_the_production_deny_list(): + """The measurement and the list the exec path trusts cannot disagree. + + Without this, a factor could be measured clean here while still sitting on the + deny list (its exec evaluation blocked for no reason) or, worse, be measured + dirty and be missing from the list (its exec artifact declaring view=decision). + """ + for factor_id in CLEAN_FACTOR_IDS: + assert factor_id not in KNOWN_POST_CUTOFF_DEPENDENT_IDS + assert set(CLEAN_FACTOR_IDS) | KNOWN_POST_CUTOFF_DEPENDENT_IDS == set( + BOUND_FACTOR_IDS + ) diff --git a/tests/test_eval_contract_v1.py b/tests/test_eval_contract_v1.py new file mode 100644 index 0000000..c899dcc --- /dev/null +++ b/tests/test_eval_contract_v1.py @@ -0,0 +1,299 @@ +"""Factor-evaluation contract v1.0 (D5 C3): identity fields + basis-column guard. + +Three groups, matching what the upgrade is allowed and forbidden to do: + +1. IDENTITY — ``EvalConfig`` carries ``view`` / ``return_basis`` and enforces the + legal pairing at construction time (design §1.4 mechanism 1, D0's single + source), and the report states them. +2. NON-OMITTABLE BASIS COLUMN (R24) — a cross-basis summary that omits ``basis`` / + ``view`` REFUSES to render. +3. THE ADJUDICATION CORE DID NOT MOVE — every ``VerdictThresholds`` default is + pinned to its frozen close-era value. "Upgrade, do not rewrite" is only a claim + until a test says a moved threshold is a failure; §十 R24 forbids re-calibrating + a bar inside the refactor, so the bar is nailed down here rather than described. +""" + +from __future__ import annotations + +import pytest + +from analytics.eval import ( + EVAL_CONTRACT_VERSION, + EvalConfig, + VerdictThresholds, + basis_identity_phrase, + render_verdict_summary, + require_basis_columns, +) +from analytics.eval.render import _requires_row +from data.availability_policy import ReturnBasis, View + + +def _cfg(**overrides) -> EvalConfig: + base = dict( + universe="000905.SH", + universe_is_pit=True, + start="2021-07-01", + end="2026-06-30", + is_exploratory=True, + post_hoc_selected=False, + ) + base.update(overrides) + return EvalConfig(**base) + + +# --------------------------------------------------------------------------- # +# 1. identity fields +# --------------------------------------------------------------------------- # +def test_the_default_describes_the_callers_that_do_not_pass_it(): + """The default is the LEGACY pairing, and that is the point. + + A default's only job is to be TRUE for the callers that omit the field. Today + those are the eleven close-basis runners, each of which builds ONE EvalConfig + and hands it to both its close_to_close reports AND (via qt.exec_basis_eval) its + exec ones. Defaulting to decision/exec_to_exec would have made every close + artifact state a basis it was not scored on — the exact describe-the-check drift + the field was added to prevent, introduced by the field itself. The exec path + sets the identity explicitly instead (see the test below). + """ + cfg = _cfg() + assert cfg.view == View.CLOSE.value + assert cfg.return_basis == ReturnBasis.CLOSE_TO_CLOSE.value + + +def test_the_exec_pairing_is_constructible_and_is_what_the_exec_path_declares(): + cfg = _cfg(view="decision", return_basis="exec_to_exec") + assert (cfg.view, cfg.return_basis) == ("decision", "exec_to_exec") + + +def test_the_exec_basis_module_restates_the_identity_it_scores_on(): + """BEHAVIOURAL: a close-default config in, the exec identity out. + + The first version of this test matched SUBSTRINGS in the function's source + (``"replace(" in source and "EXEC_TO_EXEC" in source``). Review broke it by + hand: gut the assignment, leave the two matched names in a COMMENT, and the exec + path silently inherits the caller's close identity while 52 tests stay green. + That is #78's rule again — a lexical assertion cannot hold a semantic claim — and + it failed in the worst direction, waving through the very defect the commit under + test had just fixed. It also had a false-positive direction: refactoring the line + into a helper would have deleted those names and reddened a correct build. + + So the claim is now made about BEHAVIOUR, on the extracted helper. + """ + from qt.exec_basis_eval import exec_identity + + legacy = _cfg() + assert (legacy.view, legacy.return_basis) == ("close", "close_to_close") + + out = exec_identity(legacy, factor_id="volume_peak_count_20", book_view="close") + assert out.view == "decision" + assert out.return_basis == "exec_to_exec" + assert out.book_view == "close" + # ... and nothing else moved: the identity is all that travels with the basis. + ignore = {"view", "return_basis", "book_view"} + assert {k: v for k, v in vars(out).items() if k not in ignore} == { + k: v for k, v in vars(legacy).items() if k not in ignore + } + + +def test_a_factor_that_is_not_cutoff_safe_cannot_get_an_exec_identity(): + """The derivation refuses rather than declaring a view the values do not have. + + ``jump_amount_corr_20``'s compute applies no 14:50 truncation, so its values are + close-view; (close, exec_to_exec) is not a legal pairing, and that refusal is + the correct outcome — there is no honest exec artifact of it until it is fixed. + """ + from qt.exec_basis_eval import exec_identity, subject_view + + assert subject_view("jump_amount_corr_20") == "close" + assert subject_view("volume_peak_count_20") == "decision" + with pytest.raises(ValueError, match="no 14:50 truncation of its own"): + exec_identity(_cfg(), factor_id="jump_amount_corr_20", book_view="close") + + +def test_the_no_book_and_with_book_runs_do_not_share_one_book_view(): + """Two runs, two information sets, two configs — never one shared claim.""" + from qt.exec_basis_eval import exec_identity + + no_book = exec_identity(_cfg(), factor_id="volume_peak_count_20", book_view=None) + with_book = exec_identity(_cfg(), factor_id="volume_peak_count_20", book_view="close") + assert no_book.book_view is None + assert with_book.book_view == "close" + assert no_book.view == with_book.view == "decision" + + +def test_book_view_is_validated_and_an_unknown_value_is_refused(): + assert _cfg(book_view=None).book_view is None + assert _cfg(book_view=View.CLOSE).book_view == "close" + with pytest.raises(ValueError, match="book_view must be None"): + _cfg(book_view="whatever") + + +def test_book_view_is_not_pairing_checked_against_the_basis(): + """A close-view book under an exec basis is the DISCLOSURE, not a config error. + + Refusing it would not un-mix the information sets; it would only stop the + artifact from saying they are mixed (design §1.1's live defect, open until D7). + """ + cfg = _cfg(view="decision", return_basis="exec_to_exec", book_view="close") + assert (cfg.view, cfg.return_basis, cfg.book_view) == ( + "decision", "exec_to_exec", "close", + ) + + +@pytest.mark.parametrize( + ("view", "basis"), + [("close", "exec_to_exec"), ("decision", "close_to_close")], +) +def test_illegal_pairing_is_a_construction_error(view, basis): + """The mixed pairings raise — the failure the D0 pairing rule exists to cause.""" + with pytest.raises(ValueError, match="illegal view/basis pairing"): + _cfg(view=view, return_basis=basis) + + +def test_unknown_view_is_rejected_not_passed_through(): + with pytest.raises(ValueError): + _cfg(view="intraday_9am", return_basis="exec_to_exec") + + +def test_enum_members_normalize_to_canonical_strings(): + """An enum member in, canonical VALUE out — the exported record has one spelling.""" + cfg = _cfg(view=View.DECISION, return_basis=ReturnBasis.EXEC_TO_EXEC) + assert cfg.view == "decision" and cfg.return_basis == "exec_to_exec" + assert isinstance(cfg.view, str) and type(cfg.view) is str + + +def test_identity_phrase_covers_both_pairings_and_states_the_book(): + """Renamed (review LOW-1): this pins the phrase's FORMAT, not author-once. + + The old name claimed "authored once", which it never tested — it cannot see + whether some other module spells the same sentence itself. That property does + hold (one producer in ``analytics/eval/contract.py``, composed at its call + sites), but a name must not claim more than its assertions. + """ + cfg = _cfg(view="decision", return_basis="exec_to_exec", book_view="close") + assert ( + basis_identity_phrase(cfg.view, cfg.return_basis, cfg.book_view) + == "view=decision x return_basis=exec_to_exec, book_view=close" + ) + legacy = _cfg() + assert ( + basis_identity_phrase(legacy.view, legacy.return_basis, legacy.book_view) + == "view=close x return_basis=close_to_close, book_view=none (no book supplied)" + ) + + +def test_an_absent_book_is_worded_never_blank(): + """"No book" is a stated fact, so it can never be read as a missing field.""" + absent = basis_identity_phrase("close", "close_to_close", None) + stated = basis_identity_phrase("close", "close_to_close", "close") + assert absent.endswith("book_view=none (no book supplied)") + assert absent != stated + assert "book_view=\n" not in absent and not absent.endswith("book_view=") + + +# --------------------------------------------------------------------------- # +# 2. the non-omittable basis column (R24) +# --------------------------------------------------------------------------- # +_ROWS = [ + {"factor_id": "a_20", "verdict": "Watch", "return_basis": "exec_to_exec", "view": "decision"}, + { + "factor_id": "b_20", + "verdict": "Watch", + "return_basis": "close_to_close", + "view": "close", + }, +] + + +def test_summary_renders_when_basis_and_view_are_present(): + out = render_verdict_summary( + _ROWS, columns=("factor_id", "verdict", "return_basis", "view") + ) + assert out == ( + "| factor_id | verdict | return_basis | view |\n" + "|---|---|---|---|\n" + "| a_20 | Watch | exec_to_exec | decision |\n" + "| b_20 | Watch | close_to_close | close |\n" + ) + + +@pytest.mark.parametrize( + "columns", + [ + ("factor_id", "verdict"), # both omitted + ("factor_id", "verdict", "return_basis"), # view omitted + ("factor_id", "verdict", "view"), # basis omitted + ], +) +def test_summary_refuses_to_render_without_the_identity_columns(columns): + with pytest.raises(ValueError, match="identity column"): + render_verdict_summary(_ROWS, columns=columns) + + +def test_declaring_the_column_and_leaving_it_blank_is_the_same_omission(): + rows = [dict(_ROWS[0]), {**_ROWS[1], "return_basis": ""}] + with pytest.raises(ValueError, match="leaves the identity column"): + render_verdict_summary(rows, columns=("factor_id", "return_basis", "view")) + + +def test_require_basis_columns_is_reusable_on_its_own(): + assert require_basis_columns(["view", "return_basis", "x"]) == ( + "view", + "return_basis", + "x", + ) + with pytest.raises(ValueError): + require_basis_columns(["x"]) + + +# --------------------------------------------------------------------------- # +# 3. the adjudication core did NOT move (§十 R24: no re-calibration in a refactor) +# --------------------------------------------------------------------------- # +def test_every_verdict_threshold_default_is_the_frozen_close_era_value(): + """Pinned VALUES, not "unchanged" prose. + + These are the UNVALIDATED close-era defaults (v0.7-v0.9). Carrying them over + byte-for-byte is what makes this cycle's verdicts comparable to the eleven-factor + loop's; moving one inside a refactor would make every verdict in this cycle + uninterpretable, and would do it silently. + """ + t = VerdictThresholds() + assert t.min_rebalances == 12 + assert t.min_effective_samples == 24.0 + assert t.min_span_days == 365 + assert t.min_abs_icir == 0.30 + assert t.min_incremental_abs_icir == 0.15 + assert t.min_ic_win_rate == 0.55 + assert t.min_abs_nw_t == 2.0 + assert t.min_monotonicity_spearman == 0.0 + + +def test_contract_version_is_stated(): + assert EVAL_CONTRACT_VERSION == "1.0" + + +# --------------------------------------------------------------------------- # +# provenance rendering of the factor's declarations +# --------------------------------------------------------------------------- # +class _FakeField: + def __init__(self, field: str, source: str) -> None: + self.field, self.source = field, source + + +class _FakeSpec: + def __init__(self, requires) -> None: + self.requires = requires + + +def test_requires_row_renders_source_dot_field_sorted(): + spec = _FakeSpec( + (_FakeField("close", "market_daily"), _FakeField("volume", "stk_mins_1min")) + ) + assert _requires_row(spec) == "market_daily.close, stk_mins_1min.volume" + + +def test_empty_requires_is_marked_not_blank(): + """An empty declaration is stated, never rendered as an empty row.""" + assert _requires_row(_FakeSpec(())) == "(none declared)" + assert _requires_row(_FakeSpec(None)) == "(none declared)" diff --git a/tests/test_exec_basis_eval.py b/tests/test_exec_basis_eval.py index ec31def..c9e9aa8 100644 --- a/tests/test_exec_basis_eval.py +++ b/tests/test_exec_basis_eval.py @@ -8,7 +8,15 @@ * the mandatory disclosure (execution parameters, coverage loss by cause, measured live-call count) actually reaches the rendered report; * adding that disclosure does NOT move a verdict axis — the Tradable axis stays - NOT_ASSESSED, because a return basis is not a fill-feasibility measurement. + NOT_ASSESSED, because a return basis is not a fill-feasibility measurement; + * the two exec artifacts state their OWN information sets (contract v1.0): the + no-book report says it had no book, the with-book report names the book's view. + +That last one is here rather than beside the ``exec_identity`` unit tests on +purpose. Unit-testing the helper says nothing about how the caller WIRES it, and +that gap is real: passing ``book_view=None`` for the with-book run — a book was +supplied, the report says there was none — passed 28 tests before this file +asserted it. """ from __future__ import annotations @@ -168,6 +176,44 @@ def test_exec_basis_eval_writes_its_own_reports_and_leaves_the_control_alone(tmp assert out.minute_live_calls == 0 +def test_the_two_exec_artifacts_state_their_own_information_sets(tmp_path): + """no_book says "no book"; with_book NAMES the book's view (contract v1.0). + + One shared config would have to pick one of those and be wrong about the other + — and being wrong here is the §1.1 live defect (a close-view book scored against + an exec holding window) going unmentioned in the artifact that contains it. + """ + cfg, panel, factor, book, spec, eval_cfg, report_dir = _fixture(tmp_path) + out = run_exec_basis_evaluation( + factor, spec, eval_cfg, book, + cfg=cfg, panel=panel, symbols=list(SYMBOLS), logger=LOGGER, + report_dir=report_dir, stem="demo", + ) + + no_book = json.loads(out.no_book_json.read_text(encoding="utf-8"))["eval_config"] + with_book = json.loads(out.with_book_json.read_text(encoding="utf-8"))["eval_config"] + + # Both are exec-basis evaluations of a decision-view subject factor ... + for block in (no_book, with_book): + assert block["view"] == "decision" + assert block["return_basis"] == "exec_to_exec" + # ... but only ONE of them had a book, and they must not claim otherwise. + assert no_book["book_view"] is None + assert with_book["book_view"] == "close" + + # The caller's own config is untouched: the identity travels with the basis, + # it does not leak back into the close-basis reports the caller also writes. + assert (eval_cfg.view, eval_cfg.return_basis) == ("close", "close_to_close") + assert eval_cfg.book_view is None + + # And the rendered Markdown says it too, through the author-once phrase. + md = out.with_book_md.read_text(encoding="utf-8") + assert "book_view=close" in md + assert "book_view=none (no book supplied)" in out.no_book_md.read_text( + encoding="utf-8" + ) + + def test_exec_basis_eval_discloses_parameters_and_coverage_in_every_report(tmp_path): cfg, panel, factor, book, spec, eval_cfg, report_dir = _fixture(tmp_path) out = run_exec_basis_evaluation( diff --git a/tests/test_minute_diagnostics_channel.py b/tests/test_minute_diagnostics_channel.py new file mode 100644 index 0000000..c571a94 --- /dev/null +++ b/tests/test_minute_diagnostics_channel.py @@ -0,0 +1,276 @@ +"""D5 C4 groundwork: the per-day gate-attrition channel (network-free). + +WHY THIS EXISTS. Four of the eleven minute factors publish a scarcity / +neutralization disclosure as an ADDED report section, and three of them build it +from a per-symbol DAY-LEVEL frame that the factor's compute function emits through +a ``diagnostics_out`` sink. The old eval runners had that frame because they ran +the compute themselves. The unified runner does not: values come from the D3 value +store through the D4/D4b materializer, and the store persists VALUES ONLY — the +day counts are not in it and cannot be recovered from it. + +So the disclosure had exactly two possible fates: recompute it, or drop it. Dropping +a mandatory-by-construction disclosure because the plumbing changed is the silent +degradation this project forbids, so the channel exists and its cost is stated at +every gate it passes. + +WHAT IS PINNED HERE +* asking for diagnostics NEVER changes a value (the D4b-verified path is untouched); +* the sink is emit-window-scoped, so the denominators mean the same thing they meant + in the frozen artifacts (the days the run scored), not "the days that were loaded"; +* a factor with no diagnostics binding yields an EMPTY frame and answers False to + ``has_minute_diagnostics`` — an empty frame must never be readable as "no attrition"; +* a daily factor + a sink is a readable error, not an empty sink; +* the service refuses a shared sink for several factors (the frames carry no factor + label) and, on a WARM store, still materializes when a sink is asked for. + +Every invariance claim below carries recorded mutation evidence, and each mutation +was first asserted to change its target — a sink that silently stayed empty would +make "values unchanged" pass for the wrong reason, which is the exact failure shape +this repo keeps catching in itself. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from data.availability_policy import View +from data.clean.intraday_schema import normalize_intraday_bars +from factors import registry as factor_registry +from factors.compute.minute.binding import ( + _MINUTE_STREAM_BINDINGS, + has_minute_diagnostics, + minute_diagnostics_from_bars, +) +from factors.materialize import MaterializeSources, materialize_range +from factors.service import DecisionPoint, panel +from factors.store.values import FactorValueStore + +#: Factors whose disclosure needs the day-level frame. Asserted as a SET against +#: the binding table below, so adding a fourth without a decision goes red. +DIAGNOSTIC_FACTOR_IDS = ( + "valley_ridge_vwap_ratio_20", + "ridge_minute_return_20", + "peak_ridge_amount_ratio_20", +) + +N_SYMBOLS = 12 +SYMBOLS = [f"6000{i:02d}.SH" for i in range(N_SYMBOLS)] +DATES = pd.bdate_range("2021-01-04", periods=48) +EMIT_START, EMIT_END = DATES[44], DATES[47] +BARS_PER_DAY = 238 + + +def _bars() -> pd.DataFrame: + rng = np.random.RandomState(31) + rows: list[tuple] = [] + for s in SYMBOLS: + for d in DATES: + base = pd.Timestamp(d) + pd.Timedelta("09:31:00") + price = 100.0 + rng.normal(0, 2) + for i in range(BARS_PER_DAY): + t = base + pd.Timedelta(minutes=i) + price += rng.normal(0, 0.05) + slot = 1e4 * (1.0 + 0.3 * np.sin(i / 12.0)) + erupt = 6.0 if (rng.rand() < 0.06) else 1.0 + vol = slot * erupt * (1.0 + 0.1 * rng.rand()) + w = 0.15 * price * (1.0 + (2.0 if erupt > 1 else 0.0)) * (0.5 + rng.rand()) + hi, lo = price + abs(w) * rng.rand(), price - abs(w) * rng.rand() + cl = lo + (hi - lo) * rng.rand() + rows.append((t, s, price, hi, lo, cl, vol, cl * vol)) + cols = ["time", "symbol", "open", "high", "low", "close", "volume", "amount"] + return normalize_intraday_bars(pd.DataFrame(rows, columns=cols), freq="1min") + + +BARS = _bars() + + +class Prov: + """Cache-free minute provider that honours ``symbols`` and counts its calls.""" + + def __init__(self) -> None: + self.calls = 0 + + def minute_bars(self, symbols, start, end): + self.calls += 1 + names = [str(s) for s in symbols] + if not names: + return BARS.iloc[0:0] + t = BARS.index.get_level_values("time") + sym = pd.Index(BARS.index.get_level_values("symbol")) + return BARS[ + (t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end)) & sym.isin(names) + ] + + def earliest_available(self, symbols): + return DATES[0] + + +def _materialize(factor_id, *, diagnostics=None, provider=None): + return materialize_range( + factor_registry.build(factor_id), + view=View.CLOSE, + symbols=list(SYMBOLS), + emit_start=EMIT_START, + emit_end=EMIT_END, + sources=MaterializeSources(minute=provider or Prov()), + diagnostics=diagnostics, + ) + + +# --------------------------------------------------------------------------- # +# which factors publish a frame at all +# --------------------------------------------------------------------------- # +def test_the_diagnostic_set_is_exactly_the_declared_three(): + """Derived from the binding table, not from this list — a fourth goes red.""" + declared = { + cls().name + for cls, binding in _MINUTE_STREAM_BINDINGS.items() + if binding.per_symbol_diagnostics is not None + } + assert declared == set(DIAGNOSTIC_FACTOR_IDS) + + +def test_a_factor_without_a_binding_answers_false_and_yields_an_empty_frame(): + """An empty frame is NOT "no attrition" — the caller must ask the other question.""" + factor = factor_registry.build("volume_peak_count_20") + assert has_minute_diagnostics(factor) is False + assert minute_diagnostics_from_bars(factor, BARS).empty + + +@pytest.mark.parametrize("factor_id", DIAGNOSTIC_FACTOR_IDS) +def test_a_declared_factor_answers_true(factor_id): + assert has_minute_diagnostics(factor_registry.build(factor_id)) is True + + +# --------------------------------------------------------------------------- # +# the values do not move +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("factor_id", DIAGNOSTIC_FACTOR_IDS) +def test_asking_for_diagnostics_does_not_move_a_single_value(factor_id): + """MUTATION EVIDENCE, and the pre-assertion that makes it non-vacuous. + + "Values unchanged with a sink" is trivially true if the sink is never filled — + that is the shape of an unfailable test. So the sink is asserted NON-EMPTY + first: the comparison only means something once the diagnostics path has + actually run. Recorded mutation: returning ``pd.DataFrame()`` from + ``_sink_diagnostics`` makes the pre-assertion FAIL (rc=1) rather than letting + the equality pass for the wrong reason. + """ + without = _materialize(factor_id) + sink: list[pd.DataFrame] = [] + with_sink = _materialize(factor_id, diagnostics=sink) + + assert sink, "the diagnostics sink stayed empty — the comparison below is vacuous" + assert sum(len(f) for f in sink) > 0 + pd.testing.assert_series_equal(without, with_sink, check_exact=True) + + +@pytest.mark.parametrize("factor_id", DIAGNOSTIC_FACTOR_IDS) +def test_the_sink_is_scoped_to_the_emit_window(factor_id): + """The denominator must mean "the days this run scored", not "the days loaded". + + The engine loads warm-up / saturation history far behind ``emit_start``; letting + those days into the frame would inflate ``symbol_days`` against the frozen + artifacts' figure for no reason a reader could see. + """ + sink: list[pd.DataFrame] = [] + _materialize(factor_id, diagnostics=sink) + index = pd.DatetimeIndex(pd.concat(sink).index) + assert index.min() >= EMIT_START and index.max() <= EMIT_END + + +@pytest.mark.parametrize("factor_id", DIAGNOSTIC_FACTOR_IDS) +def test_the_frames_carry_the_symbol_label_the_summarizers_read(factor_id): + sink: list[pd.DataFrame] = [] + _materialize(factor_id, diagnostics=sink) + frame = pd.concat(sink) + assert "symbol" in frame.columns + assert set(frame["symbol"]).issubset(set(SYMBOLS)) + assert "classifiable_bars" in frame.columns + + +def test_a_factor_without_diagnostics_leaves_the_sink_empty_rather_than_erroring(): + """Uniform call shape; whether a factor HAS a disclosure stays a separate question.""" + sink: list[pd.DataFrame] = [] + _materialize("volume_peak_count_20", diagnostics=sink) + assert sink == [] + + +def test_a_daily_factor_with_a_sink_is_a_readable_error(): + """Refusing beats returning an empty sink a caller would read as "no attrition".""" + + class DailyProv: + def daily_panel(self, symbols, start, end): + idx = pd.MultiIndex.from_product( + [pd.bdate_range("2021-01-04", periods=60), SYMBOLS[:2]], + names=["date", "symbol"], + ) + return pd.DataFrame({"close": np.linspace(10.0, 20.0, len(idx))}, index=idx) + + with pytest.raises(ValueError, match="daily factor"): + materialize_range( + factor_registry.build("volatility_20"), + view=View.CLOSE, + symbols=SYMBOLS[:2], + emit_start=EMIT_START, + emit_end=EMIT_END, + sources=MaterializeSources(daily=DailyProv()), + diagnostics=[], + ) + + +# --------------------------------------------------------------------------- # +# the service-level contract +# --------------------------------------------------------------------------- # +def _decisions(): + return [DecisionPoint(date=d) for d in [EMIT_START, EMIT_END]] + + +def test_a_shared_sink_for_several_factors_is_refused(tmp_path): + store = FactorValueStore(tmp_path / "store") + with pytest.raises(ValueError, match="exactly ONE factor_id"): + panel( + ["ridge_minute_return_20", "valley_ridge_vwap_ratio_20"], + SYMBOLS, + _decisions(), + store=store, + sources=MaterializeSources(minute=Prov()), + diagnostics=[], + ) + + +def test_a_warm_store_serves_values_but_a_sink_forces_the_recompute(tmp_path): + """The disclosed cost, pinned as behaviour rather than left in a docstring. + + Warm read with NO sink: zero provider calls (the store answers). Warm read WITH + a sink: the engine runs again, because the day counts are not in the store. + """ + store = FactorValueStore(tmp_path / "store") + src_cold = MaterializeSources(minute=Prov()) + cold = panel( + ["ridge_minute_return_20"], SYMBOLS, _decisions(), + store=store, sources=src_cold, + ) + assert src_cold.minute.calls > 0 + + warm_prov = Prov() + warm = panel( + ["ridge_minute_return_20"], SYMBOLS, _decisions(), + store=store, sources=MaterializeSources(minute=warm_prov), + ) + assert warm_prov.calls == 0, "a warm store must not re-read minute bars" + pd.testing.assert_frame_equal(cold, warm, check_exact=True) + + diag_prov = Prov() + sink: list[pd.DataFrame] = [] + with_diag = panel( + ["ridge_minute_return_20"], SYMBOLS, _decisions(), + store=store, sources=MaterializeSources(minute=diag_prov), + diagnostics=sink, + ) + assert diag_prov.calls > 0, "asking for diagnostics must re-run the engine" + assert sink, "the sink must be filled by that re-run" + # and the recompute reproduces the stored values exactly + pd.testing.assert_frame_equal(cold, with_diag, check_exact=True)