diff --git a/config/factor_eval_csi500.yaml b/config/factor_eval_csi500.yaml new file mode 100644 index 0000000..dc197e0 --- /dev/null +++ b/config/factor_eval_csi500.yaml @@ -0,0 +1,110 @@ +# D5 C4 — UNIFIED exec-only factor evaluation runner (run-factor-eval). +# +# ONE config for all eleven minute-derived factors: the SUBJECT factor comes +# from the CLI (--factor , e.g. jump_amount_corr_20), not from +# this file. The evaluation is EXEC-ONLY: the factor's decision-view (14:50) +# values are scored on the 14:51-VWAP exec_to_exec forward return, through the +# factor SERVICE (store read-through + the single materializer engine); the +# reports are written as factor_eval_{factor_id}_exec_{no,with}_book.* (a stem +# the legacy eval_{name}_* artifacts never collide with). +# +# CACHE-ONLY: the minute read is provably live-call-free (the minute store has +# no fetch closure); daily / universe / covariate endpoints go through the +# read-through cache (warm -> 0 gap fetches). +# +# Universe = CSI500 (000905.SH), PIT membership; window = 2021-07-01 .. +# 2026-06-30; daily rebalance (contract default); fee 0.001; quantiles 5; +# SW-L1 industry + size neutralization — the same cell every legacy eval +# config declared (catalogue section 4: the 11 configs differed in exactly two +# keys, project.name and data.output_name). + +project: + name: quantitative_trading_factor_eval + timezone: Asia/Shanghai +data: + source: tushare + freq: D + start: '2021-07-01' + end: '2026-06-30' + external_secret_file: /home/shaofl/Projects/financial_projects/.config.json + tushare_token_key: tushare.token + output_name: factor_eval_csi500 + cache: + enabled: true + root_dir: artifacts/cache/tushare/v1 + refresh_recent_days: 14 + refresh_dimension_days: 30 + force_refresh: [] +universe: + type: index + index_code: 000905.SH + symbols: [] + min_listing_days: 60 + filters: + missing_close: true + suspended: false + st: false + limit_up_down: false +# factors = EMPTY ON PURPOSE (catalogue BUG 5): the subject factor comes from +# --factor and the evaluated BOOK is fixed inside the runner (value_ep / +# value_bp / volatility_20, lead ruling Q1). A non-empty block must match that +# effective book exactly or the runner raises — this file must never silently +# describe a factor set the run does not use. +factors: [] +processing: + drop_missing: true + standardize: + enabled: true + method: zscore + # winsorize is a P0 no-op in this codebase; the EvalConfig declares + # winsorize=None accordingly (nothing is clipped), so the report never + # overstates preprocessing. + winsorize: + enabled: false + method: mad + n: 3.0 + # Industry + market-cap neutralization (SW-L1), matching the report's + # neutral column and the EvalConfig neutralization declaration. + neutralize: + enabled: true + industry_col: industry + size_col: market_cap + industry_level: L1 +alpha: + model: equal_weight + params: {} +portfolio: + # Required by the schema but unused: this runner evaluates a factor, it does + # not build/execute a portfolio. + constructor: topn_equal_weight + top_n: 50 + long_only: true + max_weight: null + turnover_cap: null +backtest: + # Required by the schema but unused (no backtest is run). The EvalConfig + # uses a DAILY rebalance (contract default), independent of this field. + initial_nav: 1.0 + rebalance: monthly + event_order: close_to_next_period + cash_return: 0.0 +cost: + fee_rate: 0.001 + slippage_rate: 0.0 + turnover_formula: l1 +analytics: + forward_return_periods: + - 1 + quantiles: 5 + benchmark: null +# OOS split (window midpoint) so the OOS section runs and the Predictive axis +# can be assessed (sign consistency across both holdout subperiods). +oos: + split_date: '2024-01-01' +output: + root_dir: artifacts + data_dir: artifacts/data + factor_dir: artifacts/factors + report_dir: artifacts/reports + log_dir: artifacts/logs + overwrite: true diff --git a/docs/factors/d5_runner_difference_catalogue.md b/docs/factors/d5_runner_difference_catalogue.md index d6801fe..fb0d0b6 100644 --- a/docs/factors/d5_runner_difference_catalogue.md +++ b/docs/factors/d5_runner_difference_catalogue.md @@ -317,3 +317,64 @@ unknown-never-convicts / N_eff CI / exploratory 封顶 Watch 的全部规则。 ⚠️ 因此 C5 对账在**逐字节**层面对 22 份 md/json 一定不成立，能成立的是 **IC / ICIR / 分位价差 / verdict 的值级对账**加上「JSON 差异恰为上表三项 + §七 的 spec 四键」这个 **结构性**断言。把「逐字节相同」当作 C5 的通过条件会让人在这里误判为回归。 + +### 七之三、C4 首轮对账（3 因子 × 3 模式）暴露的判据缺口 —— 补登记（2026-07-27） + +C4 harness（`qt/factor_eval_reconcile.py`）首轮对账 minute_ideal_amp_10 / jump_amount_corr_20 / +volume_peak_count_20 三模式全红。独立调查（证据脚本 `/tmp/q1_dump.py` / `/tmp/q1_twogeom.py` / +`/tmp/q1b_tail.py`，首轮日志 `/tmp/c4_reconcile_run.log`）把**全部**差异归到两个根因，均为 +**加载几何/浮点层面、非取值 bug**。本节把判据补全为具名类，每条都有边界与反例约束 +（类外差异仍非零退出——未编目的差异算失败）。 + +**认知修正（本节最重要的一条）**：C4 handoff §3 ① 曾预测「bounded 因子 vs D1 baseline 预期 +**零差异**，因为旧 runner 也逐票读全窗口」。**这个预测错了**：旧 runner 的「全窗口」左端同样 +锚在 `data.start`（2021-07-01）——即 **anchor 截断**；新 materializer 的饱和加载左延到分钟缓存 +真实起点（2015-01-05）。首轮「预期全绿却没绿」的源头是这个预测，**不是引擎 bug**。 + +**根因一：warmup 左延漂移（设计内）**，具名类 `warmup_left_extension`，bounded 与 pooled 通用， +覆盖**全部三个方向**（frozen-NaN→new-finite、finite→finite 即 partial pool→full pool、 +new-only finite 行——旧 runner 网格根本没发射的那些行）： + +| 形态 | 边界（实测精确） | 首轮实测 | +|---|---|---| +| bounded（minute_ideal_amp_10 / jump_amount_corr_20） | 冻结网格**前 w−1 个交易日**（w=该因子 `lookback_depth`；第 w 日已完全 warmup）。边界取**网格左端**而非 per-symbol 首日：旧 runner 对每个 symbol 的加载都锚在 `data.start`，只有评估窗左端欠 warmup；晚上市/停牌票的首日两种几何看到同一批 bar，不可能有差异 | NaN→finite 3644 + 180；finite→finite 4554 + 17109 | +| pooled（valid-day 池化，volume_peak_count_20） | 早区窗 [2021-07-01, 2021-10-31]，且**按月计数非递增**（违反即失败） | NaN→finite 8189；finite→finite 17193（07-01..08-24 按月衰减）；new-only finite 9059（07-01..07-14，旧网格未发射） | + +anchors 腿同判据：hand 侧按旧几何算、service 侧按新几何算，失败行全在 warmup 区。原 harness +只给 pooled 备了早区类、bounded 没有 → **判据不对称**（minute_ideal_amp 与 jump 的首轮 anchors +失败即此）。bounded 的 warmup 边界与 panels 腿相同（冻结网格前 w−1 个交易日）；jump 的 +**非** warmup 行仍必须 reconcile（它证明 service 携带截断定义，首轮 3 个 random 行 rel ~1e-15 +已绿），warmup 区外的 jump mismatch 仍 FAIL。 + +**根因二：两个浮点/阈值尾部（非 bug）**，各带**双边界**（不调大全局容差，保住对真回归的牙）： + +| 类 | 机制 | 边界 | 实测 | +|---|---|---|---| +| `float_reordering_tail` | rolling 相关求和顺序差异（JC1 已裁 1e-12 为可归因浮点重排地板，这是地板之上的实测尾部） | rel ≤ **5e-12** 且 cell 数 ≤ **101**（超任一即失败） | jump 101 cell，5 票散在，rel 1.0e-12–2.9e-12 | +| `threshold_flip_tail` | rolling σ 浮点噪声（~4e-10）× 整数成交量恰压阈值 → 计数翻转 | 幅度**恰 ±1 count**、rel ≤ 1e-2、cell 数 ≤ **25**（超任一即失败） | volume_peak 20 cell，600623.SH 2023-06-15..07-14 每天恰 −1（整数成交量 13300.0） | + +**reports 腿两处口径修正 + 三个 JSON 登记项**： + +1. **MD 按 key 配对成行级 change 再分类**。裸行集合差把每个值变化报成一对（一条 removal + + 一条同 `- key:` 头的 addition）——首轮 83–96 条「removal」全是这种幻影。配对后跟随与 JSON + 相同的数值归因梯。配对变化的分类：**数字承载**（任一侧含数字，散文行的数字可能在冒号 + **之前**）才进 `warmup_aggregate_effect`；纯标签翻转（verdict PASS→FAIL 不含数字）仍 + unregistered 失败。**配对是两遍的**：先按精确 `- key:` 头配对，再对剩余行按**数字归一化** + 的头配对（实测必要：incremental 轴理由行的变化数字在首个冒号之前，精确头永远配不上； + 且该行头长 ~103 字符，配对 key 上限须 ≥200——改了散文**词**而非数字的行归一化后仍配不上， + 照失败，有反向测试锁定）。 +2. `warmup_aggregate_effect`（JSON 与 MD 共用）：聚合指标叶子（`sections[*]` / + `verdict.reasons[*]` / `verdict.axes.*`）的数值/散文变化是 panels 腿已登记 warmup 差异的 + **下游**（聚合一个 warmup cell 变了的面板；panels 腿才是值级闸门）。**牙仍在**：聚合路径 + 之外（verdict 标签、spec、eval_config、criteria）任何变化仍失败；无数字的标签翻转仍失败。 +3. `spec.requires[0..n]` 由精确匹配改**前缀**匹配（§七 已登记 spec 16→20 键，`requires` 是 + repr 字符串**列表**，展平后是索引叶子）。 +4. `registered_sanity_stem_rename`：`sections[5].payload.sanity_report`（及 MD 同名行）的值从 + `eval__exec_basis_sanity.md` 改为 `factor_eval__exec_basis_sanity.md`—— + runner 有意的防碰撞改名，双侧都以 `_exec_basis_sanity.md` 收尾才认。 +5. `registered_run_order_artifact`：`sections[5].payload.exec_price_artifact_reused` + False→True——同会话 run-order 产物（首个 run 建 exec 价格 artifact，后续 run 复用）。 + **反向（True→False）不登记，仍失败**。 +6. jump 的 `spec.description` 与 `sections[7].payload.factor_version`（连同原已特判的 + `spec.version`）纳入 `registered_correction_effect`——仅当新 JSON 带 `corrections` + 结构化更正承载（契约 v1.1）时成立。 diff --git a/factors/materialize.py b/factors/materialize.py index 282e327..ee30271 100644 --- a/factors/materialize.py +++ b/factors/materialize.py @@ -829,6 +829,15 @@ def make_recompute_fn( (``emit_start is None`` -> from ``data_start``). The same trailing-trim floor as ``materialize_range`` keeps the incremental overlap bit-identical to a full column (the D3 batch=incremental invariant). + + CROSS-SECTIONAL FACTORS (``stores_intermediate``) are NOT WIRED here: this + fn returns the VALUE Series, while the store persists the per-symbol + intermediate for them, so ``tail_recompute`` refuses their payload up front + with ``factors.store.incremental.IntermediatePayloadNotWiredError`` — a + deliberate D5 C4 deferral, not an oversight. Wiring it (an + intermediate-returning recompute fn, footprint semantics in the overlap + validation, and a post-combine value-level check) is its own engine work, + tracked for the 21:00-update story. """ def _recompute(emit_start, end, warmup) -> pd.Series: diff --git a/factors/store/__init__.py b/factors/store/__init__.py index 267a151..9c719be 100644 --- a/factors/store/__init__.py +++ b/factors/store/__init__.py @@ -35,6 +35,7 @@ from factors.store.incremental import ( CacheHorizonConfig, IncrementalResult, + IntermediatePayloadNotWiredError, endpoint_horizons, factor_lookback_depth, overlap_window, @@ -56,6 +57,7 @@ "CacheHorizonConfig", "FactorValueStore", "IncrementalResult", + "IntermediatePayloadNotWiredError", "RunRecord", "RunRegistry", "StoreKey", diff --git a/factors/store/incremental.py b/factors/store/incremental.py index 22792a7..9963524 100644 --- a/factors/store/incremental.py +++ b/factors/store/incremental.py @@ -29,7 +29,14 @@ re-computed (not locally patched), classified by the factor's ``adjustment``: ``price_level`` mismatches are EXPECTED (an ex-date re-based the level — the key/fingerprint invalidation path); ``returns_invariant`` / ``none`` mismatches -mean upstream data actually changed and are recorded LOUDLY. +mean upstream data actually changed and are recorded LOUDLY. A stored NaN cell +recomputing to finite is NOT a mismatch: it is a footprint fill (the D4c fill +recorded the cell as covered-but-empty and the cache later warmed), counted in +the result notes and absorbed, never ``revision_detected``. + +A CROSS-SECTIONAL factor's stored payload (its per-symbol intermediate, D4c) is +refused up front with :class:`IntermediatePayloadNotWiredError`: the incremental +path is deliberately not wired for that shape (D5 C4 deferral). Layering: ``factors.store`` never imports ``qt`` (red line #10); this uses stdlib + numpy/pandas + the availability-policy leaf + the sibling store modules. @@ -48,7 +55,7 @@ from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL from factors.base import Factor from factors.store.keys import StoreKey -from factors.store.values import FactorValueStore +from factors.store.values import FactorValueStore, payload_columns #: recompute(emit_start, end, warmup) -> Series over [emit_start, end], having #: loaded ``warmup`` trading days of input history before emit_start. emit_start @@ -56,6 +63,22 @@ RecomputeFn = Callable[[pd.Timestamp | None, pd.Timestamp, int], pd.Series] +class IntermediatePayloadNotWiredError(ValueError): + """``tail_recompute`` reached a CROSS-SECTIONAL factor's stored payload. + + Such a factor stores its universe-independent per-symbol INTERMEDIATE + (several columns), not its value (D4c). The incremental path is NOT WIRED + for that shape: the recompute callable returns a value Series, the overlap + validation is value-level, and the footprint semantics of an intermediate + payload are unhandled. That is a DELIBERATE deferral (D5 C4 — the eval + runner reaches these factors through the service's read-through, which + never calls tail_recompute), tracked for the 21:00-update story; wiring it + (an intermediate-returning recompute fn, footprint-aware overlap + validation, post-combine value validation) is its own engine work. See + ``factors.materialize.make_recompute_fn``. + """ + + @dataclass(frozen=True) class CacheHorizonConfig: """The LIVE cache-config values the revision horizon is derived from (R5). @@ -132,8 +155,28 @@ def _symbols(series: pd.Series) -> pd.Index: return series.index.get_level_values(SYMBOL_LEVEL) -def _mismatched_symbols(stored_overlap: pd.Series, new_overlap: pd.Series) -> list[str]: - """Symbols whose overlap values differ bit-for-bit (NaN-aware) between the two.""" +def _classify_overlap_differences( + stored_overlap: pd.Series, new_overlap: pd.Series +) -> tuple[list[str], list[str]]: + """Split NaN-aware overlap differences into REVISIONS vs footprint FILLS. + + Returns ``(revised_symbols, filled_symbols)``: + + * REVISED — a stored FINITE value recomputed to a different value or to + NaN. The store asserted a value and the recompute contradicts it: that is + an upstream data revision (loud for ``returns_invariant``/``none``, + expected for ``price_level``). + * FILLED — a stored NaN recomputed to a FINITE value. A stored NaN asserts + NO value, so it cannot be contradicted: D4c writes explicit NaN + "footprint" rows for the cells a fill covered but the factor produced + nothing for (``factors.service._record_fill_footprint``), and those cells + legitimately turn finite once the upstream cache warms. Reading that as a + revision misdiagnoses the store's own bookkeeping as upstream data + churn, so it is classified separately (counted in the result notes, + never ``revision_detected``) and the recomputed value simply replaces + the NaN. An honestly under-warmed stored NaN (near the data start) + filling later is the same shape and gets the same treatment. + """ old = stored_overlap.sort_index(kind="mergesort") new = new_overlap.reindex(old.index) ov = old.to_numpy(dtype=float) @@ -142,9 +185,12 @@ def _mismatched_symbols(stored_overlap: pd.Series, new_overlap: pd.Series) -> li both_nan = np.isnan(ov) & np.isnan(nv) differs = (~(ov == nv)) & ~both_nan if not differs.any(): - return [] - syms = old.index.get_level_values(SYMBOL_LEVEL)[differs] - return sorted(set(map(str, syms))) + return [], [] + old_nan = np.isnan(ov) + syms = old.index.get_level_values(SYMBOL_LEVEL) + revised = sorted(set(map(str, syms[differs & ~old_nan]))) + filled = sorted(set(map(str, syms[differs & old_nan]))) + return revised, filled def tail_recompute( @@ -165,7 +211,19 @@ def tail_recompute( (classified by ``adjustment``), and append the new tail. """ w_dep = factor_lookback_depth(factor) - stored = store.read(key) + stored_frame = store.read_frame(key) + if stored_frame is not None and not stored_frame.empty: + columns = payload_columns(stored_frame) + if columns != (key.factor_id,): + raise IntermediatePayloadNotWiredError( + f"{key.factor_id}: the stored payload carries {list(columns)}, not the " + f"single value column {key.factor_id!r} — a cross-sectional factor " + f"stores its per-symbol INTERMEDIATE, and the incremental path is NOT " + f"WIRED for it (deliberate deferral, D5 C4; tracked for the " + f"21:00-update story). The eval runner reaches this factor through the " + f"service's read-through, which never calls tail_recompute." + ) + stored = None if stored_frame is None else stored_frame[key.factor_id].rename(key.factor_id) if stored is None or stored.empty: full = recompute(None, today, w_dep) store.write(key, full, fingerprint=fingerprint) @@ -213,12 +271,23 @@ def tail_recompute( tail = recomputed[rec_dates > last] stored_overlap = stored[_dates(stored) >= emit_start] - mismatched = _mismatched_symbols(stored_overlap, new_overlap) + mismatched, filled = _classify_overlap_differences(stored_overlap, new_overlap) is_price_level = factor.spec.adjustment is Adjustment.PRICE_LEVEL notes: list[str] = [] recolumned: tuple[str, ...] = () + if filled: + # NOT a revision: a stored NaN asserts no value (D4c fill footprint or an + # honestly under-warmed row), so it cannot be contradicted. The combined + # assembly below already replaces those cells with the recomputed values. + notes.append( + f"filled_after_footprint: {len(filled)} symbol(s) had stored NaN " + f"cells (fill footprint / under-warm) recompute to finite values — " + f"absorbing the recomputed values; this is NOT an upstream data " + f"revision." + ) + # assemble: untouched history + recomputed overlap/tail for normal symbols. head = stored[_dates(stored) < emit_start] combined = pd.concat([head, recomputed]) @@ -266,6 +335,7 @@ def tail_recompute( __all__ = [ "CacheHorizonConfig", "IncrementalResult", + "IntermediatePayloadNotWiredError", "RecomputeFn", "endpoint_horizons", "factor_lookback_depth", diff --git a/factors/store/values.py b/factors/store/values.py index 9ceae50..d9e312c 100644 --- a/factors/store/values.py +++ b/factors/store/values.py @@ -62,7 +62,8 @@ _FINGERPRINT_META_KEY = b"factor_store_fingerprint" -def _payload_columns(frame: pd.DataFrame) -> tuple[str, ...]: +def payload_columns(frame: pd.DataFrame) -> tuple[str, ...]: + """The payload columns of a stored frame (the artifact's shape signature).""" return tuple(str(c) for c in frame.columns) @@ -161,7 +162,7 @@ def upsert_frame(self, key: StoreKey, frame: pd.DataFrame, *, fingerprint: dict) if ( existing is None or existing.empty - or _payload_columns(existing) != _payload_columns(frame) + or payload_columns(existing) != payload_columns(frame) ): combined = frame else: @@ -205,7 +206,7 @@ def _value_column(frame: pd.DataFrame | None, key: StoreKey) -> pd.Series | None """ if frame is None: return None - columns = _payload_columns(frame) + columns = payload_columns(frame) if columns != (key.factor_id,): raise ValueError( f"{key.factor_id}: the stored payload carries {list(columns)}, not the " @@ -266,7 +267,7 @@ def read_valid_frame( frame = self._read_raw(key) if frame is None: return None - if _payload_columns(frame) != tuple(str(c) for c in columns): + if payload_columns(frame) != tuple(str(c) for c in columns): return None # different payload shape: a miss, never a partial read stored = self.stored_fingerprint(key) if stored is None: @@ -289,4 +290,4 @@ def read_valid_frame( return frame[symbols.isin(valid_symbols)] -__all__ = ["FactorValueStore"] +__all__ = ["FactorValueStore", "payload_columns"] diff --git a/qt/cli.py b/qt/cli.py index ca5c81d..95b3cbf 100644 --- a/qt/cli.py +++ b/qt/cli.py @@ -515,19 +515,15 @@ def _cmd_run_eval_valley_price_quantile(args: argparse.Namespace) -> int: ) or "n/a" ) - cov = result.neutralization print( f"OK run-eval-valley-price-quantile: covered={result.covered_symbols}/" f"{result.requested_symbols}, stk_mins_live_calls={result.minute_live_calls}, " f"factor_rows={result.factor_rows} ({result.elapsed:.1f}s)\n" # The reversal neutralization is the structural novelty of this factor; a # neutralization that silently ate the panel shows up here as a number. - f"neutralization (T-1 rev20): raw_rows={cov.raw_rows} " - f"rev_paired={cov.rev_rows} residual_rows={cov.residual_rows} " - f"dates={cov.dates_residualized}/{cov.dates_total} " - f"cross_section min/med/max={cov.cross_section_min}/" - f"{cov.cross_section_median:.1f}/{cov.cross_section_max} " - f"mean_spearman(raw,rev20)={cov.raw_rev_spearman_mean:+.4f}\n" + # render() is the single home of this line (catalogue section 3 one-site + # normalization; it used to be an inline f-string here). + f"{result.neutralization.render()}\n" f"no-book: {nb['deployment']} (predictive={nb['predictive']}) " f"ic_mean={nb['ic_mean']:.4f} ic_ir={nb['ic_ir']:.3f} N_eff={nb['effective_samples']:.1f}\n" f"with-book: {wb['deployment']} (incremental={wb['incremental']}) " @@ -580,6 +576,117 @@ def _cmd_run_eval_valley_ridge_vwap_ratio(args: argparse.Namespace) -> int: return 0 +def _cmd_run_factor_eval(args: argparse.Namespace) -> int: + """Run the unified exec-only factor evaluation (D5 C4).""" + from qt.exec_basis_eval import format_exec_basis_line + from qt.factor_eval_runner import run_factor_eval + + try: + result = run_factor_eval(args.config, args.factor, book_mode=args.book_mode) + except (ConfigError, ValueError, FileNotFoundError, NotImplementedError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + nb = result.exec_basis.no_book_metrics + wb = result.exec_basis.with_book_metrics + lines = [ + f"OK run-factor-eval: factor={result.factor_id} book_mode={result.book_mode} " + f"covered={result.covered_symbols}/{result.requested_symbols}, " + f"stk_mins_live_calls={result.minute_live_calls}, " + f"factor_rows={result.factor_rows} ({result.elapsed:.1f}s)" + ] + if result.coverage is not None: + lines.append(result.coverage.render()) + lines.append( + f"exec no-book: {nb['deployment']} (predictive={nb['predictive']}) " + f"ic_mean={_fmt_metric(nb['ic_mean'])} ic_ir={_fmt_metric(nb['ic_ir'], '.3f')} " + f"N_eff={_fmt_metric(nb['effective_samples'], '.1f')}\n" + f"exec with-book: {wb['deployment']} (incremental={wb['incremental']}) " + f"incr_ic_ir={_fmt_metric(wb['incremental_ic_ir'], '.3f')}\n" + f"reports: {result.exec_basis.no_book_md} | {result.exec_basis.with_book_md}\n" + f"dashboards: {result.exec_basis.no_book_dashboard} | " + f"{result.exec_basis.with_book_dashboard}" + ) + print("\n".join(lines)) + print(format_exec_basis_line(result.exec_basis)) + return 0 + + +def _cmd_run_factor_eval_reconcile(args: argparse.Namespace) -> int: + """Reconcile the unified runner against the frozen baselines (D5 C4/C5).""" + from pathlib import Path + + from qt.factor_eval_reconcile import ReconciliationError + + repo_root = Path(".").resolve() + try: + if args.mode == "panels": + from qt.factor_eval_reconcile import run_panels_mode + + result = run_panels_mode(args.config, args.factor, repo_root) + unclassified = [ + d for d in result.diffs + if d.classification.startswith(("unclassified", "unregistered")) + ] + print( + f"{'OK' if result.ok else 'FAIL'} reconcile panels {result.factor_id}: " + f"frozen={result.rows_frozen} new={result.rows_new} " + f"equal={result.equal} within_tol={result.within_tolerance} " + f"warmup={len(result.by_class('warmup_left_extension'))} " + f"float_tail={len(result.by_class('float_reordering_tail'))} " + f"threshold_flip={len(result.by_class('threshold_flip_tail'))} " + f"nan_footprint={result.nan_footprint_rows} " + f"unclassified={len(unclassified)} " + f"max_rel_diff={result.max_rel_diff:.3e}" + ) + if result.warmup_by_direction: + print(f" warmup by direction: {result.warmup_by_direction}") + if result.warmup_by_month: + print(f" warmup by month: {result.warmup_by_month} " + f"(monotonic={result.warmup_monotonic})") + for d in unclassified[:10]: + print(f" UNCLASSIFIED {d.classification} {d.date} {d.symbol} " + f"frozen={d.frozen} new={d.new}") + return 0 if result.ok else 1 + if args.mode == "reports": + from qt.factor_eval_reconcile import run_reports_mode + + results = run_reports_mode(args.config, args.factor, repo_root) + ok = all(r.ok for r in results) + for r in results: + counts: dict[str, int] = {} + for d in r.diffs: + counts[d.classification] = counts.get(d.classification, 0) + 1 + print( + f"{'OK' if r.ok else 'FAIL'} reconcile reports {r.name}: " + f"diffs={len(r.diffs)} {counts} " + f"max_numeric_rel_diff={r.max_numeric_rel_diff:.3e}" + ) + for d in r.diffs: + if d.classification.startswith("unregistered"): + print(f" UNREGISTERED {d.classification} {d.path} " + f"old={d.old!r} new={d.new!r}") + return 0 if ok else 1 + from qt.factor_eval_reconcile import run_anchors_mode + + result = run_anchors_mode(args.config, args.factor, repo_root) + for row in result.rows: + print( + f"{'OK ' if row.classification == 'ok' else row.classification.upper():24s} " + f"{row.cls:16s} {row.date} {row.symbol} hand={row.hand!r} " + f"service={row.service!r} rel={row.rel_diff:.2e}" + ) + print( + f"{'OK' if result.ok else 'FAIL'} reconcile anchors {result.factor_id}: " + f"{len(result.rows)} rows, ok={len(result.by_class('ok'))} " + f"warmup={len(result.by_class('warmup_left_extension'))} " + f"failed={len(result.by_class('failed'))}" + ) + return 0 if result.ok else 1 + except (ConfigError, ValueError, FileNotFoundError, ReconciliationError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + def _cmd_data_update(args: argparse.Namespace) -> int: """Warm/update the tushare caches (P4-3); never runs a backtest.""" from qt.data_updater import format_summary, run_data_update @@ -788,6 +895,46 @@ def build_parser() -> argparse.ArgumentParser: p_pra.add_argument("--config", required=True, help="Path to the YAML config.") p_pra.set_defaults(func=_cmd_run_eval_peak_ridge_amount_ratio) + p_fe = sub.add_parser( + "run-factor-eval", + help="Run the UNIFIED exec-only factor evaluation (D5 C4, cache-only).", + ) + p_fe.add_argument("--config", required=True, help="Path to the YAML config.") + p_fe.add_argument( + "--factor", + required=True, + help="Subject factor id (registry name, e.g. jump_amount_corr_20).", + ) + p_fe.add_argument( + "--book-mode", + choices=("decision", "close"), + default="decision", + help="How the confirmed book is built: through the decision-view service " + "panel (default), or the legacy close-view direct compute.", + ) + p_fe.set_defaults(func=_cmd_run_factor_eval) + + p_fr = sub.add_parser( + "run-factor-eval-reconcile", + help="Reconcile the unified runner against the frozen baselines " + "(D5 C4 harness, C5 audit; hard-gates on the 77/77 exec baseline).", + ) + p_fr.add_argument("--config", required=True, help="Path to the YAML config.") + p_fr.add_argument( + "--factor", + required=True, + help="Subject factor id (registry name, e.g. jump_amount_corr_20).", + ) + p_fr.add_argument( + "--mode", + required=True, + choices=("panels", "reports", "anchors"), + help="panels: service panel vs the frozen D1 panel; reports: new " + "factor_eval_* artifacts vs the frozen exec artifacts; anchors: service " + "values vs the hand-computed anchor rows.", + ) + p_fr.set_defaults(func=_cmd_run_factor_eval_reconcile) + for name, func, help_text in ( ("fetch-data", _cmd_fetch_data, "Run the spine, report data fetch."), ("compute-factors", _cmd_compute_factors, "Run the spine, report factor compute."), diff --git a/qt/eval_peak_ridge_amount_ratio.py b/qt/eval_peak_ridge_amount_ratio.py index ca50f08..799bd5c 100644 --- a/qt/eval_peak_ridge_amount_ratio.py +++ b/qt/eval_peak_ridge_amount_ratio.py @@ -61,7 +61,6 @@ from dataclasses import dataclass from pathlib import Path -import numpy as np import pandas as pd from analytics.eval import ( @@ -93,6 +92,9 @@ from factors.spec import FactorSpec from qt.config import RootConfig, load_config from qt.exec_basis_eval import ExecBasisEvaluation, run_exec_basis_evaluation +# The peak-scarcity disclosure MOVED to qt.factor_eval_disclosures (the single +# home, D5 C4); re-exported here so historical import paths keep working. +from qt.factor_eval_disclosures import PeakCoverage, summarize_peak_coverage from qt.pipeline import ( _build_cache, _build_universe, @@ -107,142 +109,6 @@ _LOGGER_NAME = "qt.eval_peak_ridge_amount_ratio" _REPORT_STEM = "eval_peak_ridge_amount_ratio" -# Percentiles reported for the realized peak-bar distribution (the scarcity disclosure). -_PEAK_PCTL = (0, 10, 25, 50, 75, 90, 100) - -# The counterfactual peak floor the task card asks to quantify: how many days would still -# be valid if the PEAK leg were held to the RIDGE leg's floor instead of its own. -_COUNTERFACTUAL_PEAK_FLOOR = PEAK_RIDGE_MIN_RIDGE_BARS - - -# --------------------------------------------------------------------------- # -# Peak-scarcity coverage (measured, never assumed) -# --------------------------------------------------------------------------- # -@dataclass(frozen=True) -class PeakCoverage: - """Realized peak-bar distribution + day-validity rate over the whole universe. - - Built from the per-day diagnostics the factor emits, so the numbers describe the days - the factor actually saw. ``symbol_days`` counts EVERY symbol-day with visible bars, - including the leading warm-up days that have no same-slot baseline yet; - ``classifiable_days`` counts those that clear PR-F's classifiable floor. The headline - ``validity_rate`` is taken over ``classifiable_days``, because a day with no baseline - fails for a PR-F warm-up reason rather than a peak-scarcity one and would otherwise - make the peak gate look worse than it is — both denominators are reported so the reader - can check that framing. The gate-failure counts are NOT mutually exclusive (a thin day - can fail several gates at once) and are reported for shape, not as a partition. - """ - - symbol_days: int - classifiable_days: int - valid_days: int - peak_percentiles: tuple[tuple[int, float], ...] - peak_mean: float - ridge_median: float - days_below_peak_gate: int - days_below_ridge_gate: int - days_below_classifiable_gate: int - # Counterfactual: how many days would survive if the PEAK leg were held to the RIDGE - # floor. Quantifies exactly what the lowered threshold buys. - valid_days_at_ridge_floor: int - # The gates this run actually applied, so the disclosure can never describe the module - # defaults while the run used something else. - min_peak_bars: int = PEAK_RIDGE_MIN_PEAK_BARS - min_ridge_bars: int = PEAK_RIDGE_MIN_RIDGE_BARS - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE - counterfactual_peak_floor: int = _COUNTERFACTUAL_PEAK_FLOOR - - @property - def validity_rate(self) -> float: - """Valid days as a share of CLASSIFIABLE days (see the class docstring).""" - if not self.classifiable_days: - return float("nan") - return self.valid_days / self.classifiable_days - - def render(self) -> str: - """One-line, secret-free summary for the run log and the CLI.""" - pctl = " ".join(f"p{p}={v:.0f}" for p, v in self.peak_percentiles) - return ( - f"peak scarcity: symbol_days={self.symbol_days} " - f"classifiable_days={self.classifiable_days} " - f"valid_days={self.valid_days} ({self.validity_rate:.1%} of classifiable) " - f"peak_bars[{pctl} mean={self.peak_mean:.1f}] " - f"ridge_bars_median={self.ridge_median:.0f} " - f"below_peak_gate({self.min_peak_bars})={self.days_below_peak_gate} " - f"below_ridge_gate({self.min_ridge_bars})={self.days_below_ridge_gate} " - f"below_classifiable_gate({self.min_classifiable})=" - f"{self.days_below_classifiable_gate} " - f"valid_if_peak_floor_were_{self.counterfactual_peak_floor}=" - f"{self.valid_days_at_ridge_floor}" - ) - - -def summarize_peak_coverage( - frames: list[pd.DataFrame], - *, - min_peak_bars: int = PEAK_RIDGE_MIN_PEAK_BARS, - min_ridge_bars: int = PEAK_RIDGE_MIN_RIDGE_BARS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - counterfactual_peak_floor: int = _COUNTERFACTUAL_PEAK_FLOOR, -) -> PeakCoverage: - """Reduce the per-symbol day-level diagnostics to the scarcity disclosure. - - The three floors must be the ones the RUN applied, not the module defaults — otherwise - the disclosure would describe gates that were never enforced. - """ - gates = dict( - min_peak_bars=min_peak_bars, - min_ridge_bars=min_ridge_bars, - min_classifiable=min_classifiable, - counterfactual_peak_floor=counterfactual_peak_floor, - ) - empty = tuple((p, float("nan")) for p in _PEAK_PCTL) - if not frames: - return PeakCoverage( - symbol_days=0, - classifiable_days=0, - valid_days=0, - peak_percentiles=empty, - peak_mean=float("nan"), - ridge_median=float("nan"), - days_below_peak_gate=0, - days_below_ridge_gate=0, - days_below_classifiable_gate=0, - valid_days_at_ridge_floor=0, - **gates, - ) - diag = pd.concat(frames, ignore_index=True) - classifiable = diag["classifiable_bars"].to_numpy(dtype=float) - valid = diag["valid"].to_numpy(dtype=bool) - # The bar-count distributions describe the days that had a fair chance: a warm-up day - # with no same-slot baseline has zero of everything and would only drag the percentiles - # towards zero for a reason that has nothing to do with peak scarcity. - scored = classifiable >= min_classifiable - peak = diag.loc[scored, "peak_bars"].to_numpy(dtype=float) - ridge = diag.loc[scored, "ridge_bars"].to_numpy(dtype=float) - # The counterfactual raises the PEAK floor, leaving every other gate exactly as it was. - at_ridge_floor = valid & ( - diag["peak_bars"].to_numpy(dtype=float) >= counterfactual_peak_floor - ) - return PeakCoverage( - symbol_days=int(len(diag)), - classifiable_days=int(scored.sum()), - valid_days=int(valid.sum()), - peak_percentiles=( - tuple((p, float(np.percentile(peak, p))) for p in _PEAK_PCTL) - if peak.size - else empty - ), - peak_mean=float(peak.mean()) if peak.size else float("nan"), - ridge_median=float(np.median(ridge)) if ridge.size else float("nan"), - days_below_peak_gate=int((peak < min_peak_bars).sum()), - days_below_ridge_gate=int((ridge < min_ridge_bars).sum()), - days_below_classifiable_gate=int((~scored).sum()), - valid_days_at_ridge_floor=int(at_ridge_floor.sum()), - **gates, - ) - - # --------------------------------------------------------------------------- # # Minute loading (cache-only, per-symbol -> memory-bounded) # --------------------------------------------------------------------------- # diff --git a/qt/eval_ridge_minute_return.py b/qt/eval_ridge_minute_return.py index adaa704..7c7e151 100644 --- a/qt/eval_ridge_minute_return.py +++ b/qt/eval_ridge_minute_return.py @@ -61,7 +61,6 @@ from dataclasses import dataclass from pathlib import Path -import numpy as np import pandas as pd from analytics.eval import ( @@ -92,6 +91,12 @@ from factors.spec import FactorSpec from qt.config import RootConfig, load_config from qt.exec_basis_eval import ExecBasisEvaluation, run_exec_basis_evaluation +# The ridge-scarcity disclosure MOVED to qt.factor_eval_disclosures (the single +# home, D5 C4); re-exported here so historical import paths keep working. +from qt.factor_eval_disclosures import ( + RidgeReturnCoverage, + summarize_ridge_return_coverage, +) from qt.pipeline import ( _build_cache, _build_universe, @@ -106,153 +111,6 @@ _LOGGER_NAME = "qt.eval_ridge_minute_return" _REPORT_STEM = "eval_ridge_minute_return" -# Percentiles reported for the realized ridge-bar distribution (the scarcity disclosure). -_RIDGE_PCTL = (0, 10, 25, 50, 75, 90, 100) - -# The counterfactual floor the coverage disclosure also reports, so PR-K's ridge coverage -# is directly comparable to PR-J's (which used 20 for its VALLEY leg). -_COMPARISON_FLOOR = 20 - - -# --------------------------------------------------------------------------- # -# Ridge-scarcity coverage (measured, never assumed) -# --------------------------------------------------------------------------- # -@dataclass(frozen=True) -class RidgeReturnCoverage: - """Realized ridge-bar distribution + day-validity rate over the whole universe. - - Built from the per-day diagnostics the factor emits, so the numbers describe the days - the factor actually saw. ``symbol_days`` counts EVERY symbol-day with visible bars, - including the leading warm-up days that have no same-slot baseline yet; - ``classifiable_days`` counts those that clear PR-F's classifiable floor. The headline - ``validity_rate`` is taken over ``classifiable_days``, because a day with no baseline - fails for a PR-F warm-up reason rather than a ridge-scarcity one and would otherwise - make the ridge gate look worse than it is — both denominators are reported so the - reader can check that framing. - - TWO ridge counts are tracked, because this factor gates on the narrower one: total - ``ridge_bars`` and the ``ridge_return_bars`` subset that carries a valid minute return - (the day's first visible bar is excluded by the within-day lag even when it is a - ridge). Reporting both makes the return-guard attrition visible instead of implicit. - The gate-failure counts are NOT mutually exclusive and are reported for shape, not as - a partition. - """ - - symbol_days: int - classifiable_days: int - valid_days: int - ridge_return_percentiles: tuple[tuple[int, float], ...] - ridge_return_mean: float - ridge_bars_mean: float - ridge_bars_median: float - days_below_ridge_gate: int - days_below_classifiable_gate: int - # Counterfactual: how many days would survive at the higher floor PR-J used for its - # VALLEY leg. Quantifies exactly what the scarcity-driven threshold buys. - valid_days_at_comparison_floor: int - # The gates this run actually applied, so the disclosure can never describe the module - # defaults while the run used something else. - min_ridge_bars: int = RIDGE_RETURN_MIN_RIDGE_BARS - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE - comparison_floor: int = _COMPARISON_FLOOR - - @property - def validity_rate(self) -> float: - """Valid days as a share of CLASSIFIABLE days (see the class docstring).""" - if not self.classifiable_days: - return float("nan") - return self.valid_days / self.classifiable_days - - @property - def return_guard_attrition(self) -> float: - """Share of ridge bars LOST to the return guard (mean over classifiable days).""" - if not np.isfinite(self.ridge_bars_mean) or self.ridge_bars_mean <= 0.0: - return float("nan") - return 1.0 - self.ridge_return_mean / self.ridge_bars_mean - - def render(self) -> str: - """One-line, secret-free summary for the run log and the CLI.""" - pctl = " ".join(f"p{p}={v:.0f}" for p, v in self.ridge_return_percentiles) - return ( - f"ridge scarcity: symbol_days={self.symbol_days} " - f"classifiable_days={self.classifiable_days} " - f"valid_days={self.valid_days} ({self.validity_rate:.1%} of classifiable) " - f"ridge_return_bars[{pctl} mean={self.ridge_return_mean:.1f}] " - f"ridge_bars_mean={self.ridge_bars_mean:.1f} " - f"ridge_bars_median={self.ridge_bars_median:.0f} " - f"return_guard_attrition={self.return_guard_attrition:.1%} " - f"below_ridge_gate({self.min_ridge_bars})={self.days_below_ridge_gate} " - f"below_classifiable_gate({self.min_classifiable})=" - f"{self.days_below_classifiable_gate} " - f"valid_if_floor_were_{self.comparison_floor}=" - f"{self.valid_days_at_comparison_floor}" - ) - - -def summarize_ridge_return_coverage( - frames: list[pd.DataFrame], - *, - min_ridge_bars: int = RIDGE_RETURN_MIN_RIDGE_BARS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - comparison_floor: int = _COMPARISON_FLOOR, -) -> RidgeReturnCoverage: - """Reduce the per-symbol day-level diagnostics to the scarcity disclosure. - - The floors must be the ones the RUN applied, not the module defaults — otherwise the - disclosure would describe gates that were never enforced. - """ - gates = dict( - min_ridge_bars=min_ridge_bars, - min_classifiable=min_classifiable, - comparison_floor=comparison_floor, - ) - empty = tuple((p, float("nan")) for p in _RIDGE_PCTL) - if not frames: - return RidgeReturnCoverage( - symbol_days=0, - classifiable_days=0, - valid_days=0, - ridge_return_percentiles=empty, - ridge_return_mean=float("nan"), - ridge_bars_mean=float("nan"), - ridge_bars_median=float("nan"), - days_below_ridge_gate=0, - days_below_classifiable_gate=0, - valid_days_at_comparison_floor=0, - **gates, - ) - diag = pd.concat(frames, ignore_index=True) - classifiable = diag["classifiable_bars"].to_numpy(dtype=float) - valid = diag["valid"].to_numpy(dtype=bool) - # The bar-count distributions describe the days that had a fair chance: a warm-up day - # with no same-slot baseline has zero of everything and would only drag the - # percentiles towards zero for a reason that has nothing to do with ridge scarcity. - scored = classifiable >= min_classifiable - ridge_ret = diag.loc[scored, "ridge_return_bars"].to_numpy(dtype=float) - ridge_all = diag.loc[scored, "ridge_bars"].to_numpy(dtype=float) - # The counterfactual raises the ridge floor, leaving every other gate exactly as it was. - at_comparison = valid & ( - diag["ridge_return_bars"].to_numpy(dtype=float) >= comparison_floor - ) - return RidgeReturnCoverage( - symbol_days=int(len(diag)), - classifiable_days=int(scored.sum()), - valid_days=int(valid.sum()), - ridge_return_percentiles=( - tuple((p, float(np.percentile(ridge_ret, p))) for p in _RIDGE_PCTL) - if ridge_ret.size - else empty - ), - ridge_return_mean=float(ridge_ret.mean()) if ridge_ret.size else float("nan"), - ridge_bars_mean=float(ridge_all.mean()) if ridge_all.size else float("nan"), - ridge_bars_median=float(np.median(ridge_all)) if ridge_all.size else float("nan"), - days_below_ridge_gate=int((ridge_ret < min_ridge_bars).sum()), - days_below_classifiable_gate=int((~scored).sum()), - valid_days_at_comparison_floor=int(at_comparison.sum()), - **gates, - ) - - # --------------------------------------------------------------------------- # # Minute loading (cache-only, per-symbol -> memory-bounded) # --------------------------------------------------------------------------- # diff --git a/qt/eval_valley_price_quantile.py b/qt/eval_valley_price_quantile.py index aef1f8b..4703e7c 100644 --- a/qt/eval_valley_price_quantile.py +++ b/qt/eval_valley_price_quantile.py @@ -65,7 +65,6 @@ from dataclasses import dataclass from pathlib import Path -import numpy as np import pandas as pd from analytics.eval import ( @@ -100,6 +99,12 @@ from factors.spec import FactorSpec from qt.config import RootConfig, load_config from qt.exec_basis_eval import ExecBasisEvaluation, run_exec_basis_evaluation +# The neutralization disclosure MOVED to qt.factor_eval_disclosures (the single +# home, D5 C4); re-exported here so historical import paths keep working. +from qt.factor_eval_disclosures import ( + NeutralizationCoverage, + summarize_neutralization, +) from qt.pipeline import ( _build_cache, _build_universe, @@ -115,73 +120,6 @@ _REPORT_STEM = "eval_valley_price_quantile" -# --------------------------------------------------------------------------- # -# Neutralization coverage (the factor-specific diagnostic) -# --------------------------------------------------------------------------- # -@dataclass(frozen=True) -class NeutralizationCoverage: - """What the reversal neutralization actually did, measured rather than assumed. - - A neutralization can fail quietly in two ways: the reversal can be unavailable for - most of the panel (so most residuals are NaN), or the cross-section can be too thin - to regress on many dates. Both are counted here and logged, so a coverage regression - is a number in the run record instead of an unexplained drop in sample size. - """ - - raw_rows: int # finite RAW qbar values - rev_rows: int # finite rev20 values on those rows - residual_rows: int # finite residuals (the shipped factor) - dates_total: int - dates_residualized: int # dates that cleared min_cross_section AND were non-degenerate - cross_section_min: int - cross_section_median: float - cross_section_max: int - raw_rev_spearman_mean: float # mean per-date exposure of the RAW factor to rev20 - - -def summarize_neutralization( - raw: pd.Series, - rev: pd.Series, - residual: pd.Series, - *, - min_cross_section: int, -) -> NeutralizationCoverage: - """Reduce the raw / reversal / residual panels to the coverage diagnostic.""" - raw_finite = raw.dropna() - rev_on_raw = rev.reindex(raw.index) - paired = pd.DataFrame({"f": raw, "r": rev_on_raw}).dropna() - - sizes: list[int] = [] - exposures: list[float] = [] - for _, g in paired.groupby(level=DATE_LEVEL, sort=True): - sizes.append(len(g)) - if len(g) >= min_cross_section: - f = g["f"].to_numpy(dtype=float) - r = g["r"].to_numpy(dtype=float) - fr = pd.Series(f).rank().to_numpy() - rr = pd.Series(r).rank().to_numpy() - if fr.std() > 0.0 and rr.std() > 0.0: - exposures.append(float(np.corrcoef(fr, rr)[0, 1])) - - resid_finite = residual.dropna() - dates_resid = int( - resid_finite.index.get_level_values(DATE_LEVEL).unique().size - ) - return NeutralizationCoverage( - raw_rows=int(len(raw_finite)), - rev_rows=int(len(paired)), - residual_rows=int(len(resid_finite)), - dates_total=int(raw.index.get_level_values(DATE_LEVEL).unique().size), - dates_residualized=dates_resid, - cross_section_min=int(min(sizes)) if sizes else 0, - cross_section_median=float(np.median(sizes)) if sizes else float("nan"), - cross_section_max=int(max(sizes)) if sizes else 0, - raw_rev_spearman_mean=( - float(np.mean(exposures)) if exposures else float("nan") - ), - ) - - # --------------------------------------------------------------------------- # # Minute loading (cache-only, per-symbol stats -> ONE cross-sectional residualization) # --------------------------------------------------------------------------- # diff --git a/qt/eval_valley_ridge_vwap_ratio.py b/qt/eval_valley_ridge_vwap_ratio.py index 25fe0c8..6cecce7 100644 --- a/qt/eval_valley_ridge_vwap_ratio.py +++ b/qt/eval_valley_ridge_vwap_ratio.py @@ -54,7 +54,6 @@ from dataclasses import dataclass from pathlib import Path -import numpy as np import pandas as pd from analytics.eval import ( @@ -86,6 +85,9 @@ from factors.spec import FactorSpec from qt.config import RootConfig, load_config from qt.exec_basis_eval import ExecBasisEvaluation, run_exec_basis_evaluation +# The ridge-scarcity disclosure MOVED to qt.factor_eval_disclosures (the single +# home, D5 C4); re-exported here so historical import paths keep working. +from qt.factor_eval_disclosures import RidgeCoverage, summarize_ridge_coverage from qt.pipeline import ( _build_cache, _build_universe, @@ -100,138 +102,6 @@ _LOGGER_NAME = "qt.eval_valley_ridge_vwap_ratio" _REPORT_STEM = "eval_valley_ridge_vwap_ratio" -# Percentiles reported for the realized ridge-bar distribution (the scarcity disclosure). -_RIDGE_PCTL = (0, 10, 25, 50, 75, 90, 100) - - -# --------------------------------------------------------------------------- # -# Ridge-scarcity coverage (measured, never assumed) -# --------------------------------------------------------------------------- # -@dataclass(frozen=True) -class RidgeCoverage: - """Realized ridge-bar distribution + day-validity rate over the whole universe. - - Built from the per-day diagnostics the factor emits, so the numbers describe the - days the factor actually saw. ``symbol_days`` counts EVERY symbol-day with visible - bars, including the leading warm-up days that have no same-slot baseline yet; - ``classifiable_days`` counts those that clear PR-F's classifiable floor. The - headline ``validity_rate`` is taken over ``classifiable_days``, because a day with - no baseline fails for a PR-F warm-up reason rather than a ridge-scarcity one and - would otherwise make the ridge gate look worse than it is — both denominators are - reported so the reader can check that framing. The gate-failure counts are NOT - mutually exclusive (a thin day can fail several gates at once) and are reported for - shape, not as a partition. - """ - - symbol_days: int - classifiable_days: int - valid_days: int - ridge_percentiles: tuple[tuple[int, float], ...] - ridge_mean: float - valley_median: float - days_below_ridge_gate: int - days_below_valley_gate: int - days_below_classifiable_gate: int - # Counterfactual: how many days would survive if the ridge leg were held to the - # VALLEY floor. Quantifies exactly what the lowered threshold buys. - valid_days_at_valley_floor: int - # The gates this run actually applied, so the disclosure can never describe the - # module defaults while the run used something else. - min_ridge_bars: int = VALLEY_RIDGE_MIN_RIDGE_BARS - min_valley_bars: int = VALLEY_RIDGE_MIN_VALLEY_BARS - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE - - @property - def validity_rate(self) -> float: - """Valid days as a share of CLASSIFIABLE days (see the class docstring).""" - if not self.classifiable_days: - return float("nan") - return self.valid_days / self.classifiable_days - - def render(self) -> str: - """One-line, secret-free summary for the run log and the CLI.""" - pctl = " ".join(f"p{p}={v:.0f}" for p, v in self.ridge_percentiles) - return ( - f"ridge scarcity: symbol_days={self.symbol_days} " - f"classifiable_days={self.classifiable_days} " - f"valid_days={self.valid_days} ({self.validity_rate:.1%} of classifiable) " - f"ridge_bars[{pctl} mean={self.ridge_mean:.1f}] " - f"valley_bars_median={self.valley_median:.0f} " - f"below_ridge_gate({self.min_ridge_bars})=" - f"{self.days_below_ridge_gate} " - f"below_valley_gate({self.min_valley_bars})=" - f"{self.days_below_valley_gate} " - f"below_classifiable_gate({self.min_classifiable})=" - f"{self.days_below_classifiable_gate} " - f"valid_if_ridge_floor_were_{self.min_valley_bars}=" - f"{self.valid_days_at_valley_floor}" - ) - - -def summarize_ridge_coverage( - frames: list[pd.DataFrame], - *, - min_ridge_bars: int = VALLEY_RIDGE_MIN_RIDGE_BARS, - min_valley_bars: int = VALLEY_RIDGE_MIN_VALLEY_BARS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, -) -> RidgeCoverage: - """Reduce the per-symbol day-level diagnostics to the scarcity disclosure. - - The three floors must be the ones the RUN applied, not the module defaults — - otherwise the disclosure would describe gates that were never enforced. - """ - gates = dict( - min_ridge_bars=min_ridge_bars, - min_valley_bars=min_valley_bars, - min_classifiable=min_classifiable, - ) - empty = tuple((p, float("nan")) for p in _RIDGE_PCTL) - if not frames: - return RidgeCoverage( - symbol_days=0, - classifiable_days=0, - valid_days=0, - ridge_percentiles=empty, - ridge_mean=float("nan"), - valley_median=float("nan"), - days_below_ridge_gate=0, - days_below_valley_gate=0, - days_below_classifiable_gate=0, - valid_days_at_valley_floor=0, - **gates, - ) - diag = pd.concat(frames, ignore_index=True) - classifiable = diag["classifiable_bars"].to_numpy(dtype=float) - valid = diag["valid"].to_numpy(dtype=bool) - # The bar-count distributions describe the days that had a fair chance: a warm-up day - # with no same-slot baseline has zero of everything and would only drag the - # percentiles towards zero for a reason that has nothing to do with ridge scarcity. - scored = classifiable >= min_classifiable - ridge = diag.loc[scored, "ridge_bars"].to_numpy(dtype=float) - valley = diag.loc[scored, "valley_bars"].to_numpy(dtype=float) - # The counterfactual holds the ridge leg to the valley floor, leaving every other - # gate exactly as it was. - at_valley_floor = valid & ( - diag["ridge_bars"].to_numpy(dtype=float) >= min_valley_bars - ) - return RidgeCoverage( - symbol_days=int(len(diag)), - classifiable_days=int(scored.sum()), - valid_days=int(valid.sum()), - ridge_percentiles=( - tuple((p, float(np.percentile(ridge, p))) for p in _RIDGE_PCTL) - if ridge.size - else empty - ), - ridge_mean=float(ridge.mean()) if ridge.size else float("nan"), - valley_median=float(np.median(valley)) if valley.size else float("nan"), - days_below_ridge_gate=int((ridge < min_ridge_bars).sum()), - days_below_valley_gate=int((valley < min_valley_bars).sum()), - days_below_classifiable_gate=int((~scored).sum()), - valid_days_at_valley_floor=int(at_valley_floor.sum()), - **gates, - ) - # --------------------------------------------------------------------------- # # Minute loading (cache-only, per-symbol -> memory-bounded) diff --git a/qt/exec_basis_eval.py b/qt/exec_basis_eval.py index ef9e65c..3802f91 100644 --- a/qt/exec_basis_eval.py +++ b/qt/exec_basis_eval.py @@ -30,6 +30,7 @@ from __future__ import annotations import time +from collections.abc import Sequence from dataclasses import dataclass, replace from pathlib import Path @@ -41,6 +42,7 @@ FactorEvalReport, StandardFactorEvaluator, ) +from analytics.eval.sections import SectionLike from data.availability_policy import ReturnBasis, View from factors import registry as factor_registry from factors.compute.minute.binding import is_decision_cutoff_safe @@ -218,6 +220,25 @@ def _write_report(report: FactorEvalReport, report_dir: Path, stem: str) -> tupl return md_path, json_path +def _with_extra_sections( + report: FactorEvalReport, extra_sections: Sequence[SectionLike] +) -> FactorEvalReport: + """Re-assemble ``report`` with add-Sections appended (contract §3.6). + + ``assemble`` rejects any name collision with a mandatory section, and the + verdict reads only the mandatory payloads, so the mandatory sections and + the verdict of the re-assembled report are the originals — pinned by test + (the D5 C4 unified runner's coverage disclosures ride on this). + """ + augmented = FactorEvalReport.assemble( + report.spec, + report.cfg, + [*report.sections, *extra_sections], + thresholds=report.thresholds, + ) + return augmented.with_verdict() + + def _extract_metrics(report: FactorEvalReport) -> dict: """Headline verdict + gated metrics, mirroring the runners' ``extract_metrics``.""" verdict = report.require_verdict() @@ -260,6 +281,7 @@ def run_exec_basis_evaluation( stem: str, force_rebuild: bool = False, book_view: str = View.CLOSE.value, + extra_sections: Sequence[SectionLike] | None = None, ) -> ExecBasisEvaluation: """Build the exec-to-exec returns, sanity-check them, evaluate twice, report. @@ -273,6 +295,13 @@ def run_exec_basis_evaluation( defect design §1.1 records and D7 closes. The decision-view book path passes ``decision`` explicitly. Only the caller knows this, so it is a parameter rather than something guessed here. + + ``extra_sections`` (default None -> the legacy behavior, byte-for-byte) are + add-Sections appended to BOTH exec reports before they are written — the + contract's §3.6 extension point ("may ADD sections but never drop a + mandatory one"). The verdict reads only the mandatory payloads, so appended + sections cannot move it; the unified D5 C4 runner's coverage disclosures + travel this way. """ started = time.monotonic() params = ExecBasisParams.from_config(cfg) @@ -392,6 +421,9 @@ def run_exec_basis_evaluation( report_with_book, ir_with_book = evaluator.evaluate_with_ir( factor_panel, exec_spec, cfg_with_book, ctx_with_book ) + if extra_sections: + report_no_book = _with_extra_sections(report_no_book, extra_sections) + report_with_book = _with_extra_sections(report_with_book, extra_sections) nb_md, nb_json = _write_report(report_no_book, report_dir, f"{stem}_exec_no_book") wb_md, wb_json = _write_report( diff --git a/qt/factor_eval_disclosures.py b/qt/factor_eval_disclosures.py new file mode 100644 index 0000000..346d0a0 --- /dev/null +++ b/qt/factor_eval_disclosures.py @@ -0,0 +1,647 @@ +"""The SINGLE home of the heterogeneous factor-eval coverage disclosures (D5 C4). + +Four of the eleven legacy eval runners each carried a scarcity / neutralization +disclosure of a DIFFERENT shape (catalogue +``docs/factors/d5_runner_difference_catalogue.md`` §三): no shared base class, +four independent frozen dataclasses, each with its own summarizer and +``render()``. They are deliberately NOT unified into one object — these are the +measured facts of each factor's OWN gates, with different units and semantics, +and forcing one shape would lose information. This module is where they now +live, MOVED VERBATIM from: + +* ``RidgeCoverage`` + ``summarize_ridge_coverage`` (from + ``qt/eval_valley_ridge_vwap_ratio.py``), +* ``RidgeReturnCoverage`` + ``summarize_ridge_return_coverage`` (from + ``qt/eval_ridge_minute_return.py``), +* ``PeakCoverage`` + ``summarize_peak_coverage`` (from + ``qt/eval_peak_ridge_amount_ratio.py``), +* ``NeutralizationCoverage`` + ``summarize_neutralization`` (from + ``qt/eval_valley_price_quantile.py``). + +The four legacy runners re-export from here, so every historical import path +keeps working (their test files are untouched). + +The TWO intentional additions on top of the verbatim move, both catalogue +items, and both small: + +1. ``NeutralizationCoverage.render()`` — the one disclosure the CLI rendered + as an INLINE f-string (catalogue §三 "一处归一": PR-L at ``qt/cli.py`` + while its three siblings already went through ``.render()``). The format is + exactly the line the CLI used to inline. +2. :func:`to_section` — packs a coverage dataclass into an add-Section, the + contract's §3.6 extension point ("may ADD sections but never drop a + mandatory one"). The renderer natively supports extras + (``analytics/eval/render.py::canonical_sections``: mandatory first, extras + sorted by name after), and the verdict reads only the mandatory payloads, + so appending an extra section is VERDICT-LAZY by construction (pinned by + test). + +GATE-DEFAULT SINGLE SOURCE: the summarizers' floor defaults are the same +``data.clean`` module constants the minute-binding's compute calls apply +(``factors/compute/minute/binding.py`` calls every compute function with all +gate parameters at their module defaults). The disclosure therefore reports +the floors the run ACTUALLY applied by construction — and a test pins the +defaults against those constants so a hand-edit on one side cannot drift. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import asdict, dataclass + +import numpy as np +import pandas as pd + +from analytics.eval.sections import Section +from data.clean.intraday_amount_ratio import ( + PEAK_RIDGE_MIN_PEAK_BARS, + PEAK_RIDGE_MIN_RIDGE_BARS, +) +from data.clean.intraday_ridge_return import RIDGE_RETURN_MIN_RIDGE_BARS +from data.clean.intraday_valley_ridge_vwap import ( + VALLEY_RIDGE_MIN_RIDGE_BARS, + VALLEY_RIDGE_MIN_VALLEY_BARS, +) +from data.clean.intraday_volume_prv import VOLUME_PRV_MIN_CLASSIFIABLE +from data.clean.schema import DATE_LEVEL +from factors.compute.minute.peak_ridge_amount_ratio import PeakRidgeAmountRatioFactor +from factors.compute.minute.ridge_minute_return import RidgeMinuteReturnFactor +from factors.compute.minute.valley_ridge_vwap_ratio import ValleyRidgeVwapRatioFactor + +# Percentiles reported for the realized ridge/peak-bar distributions. +_RIDGE_PCTL = (0, 10, 25, 50, 75, 90, 100) +_PEAK_PCTL = (0, 10, 25, 50, 75, 90, 100) + +# The counterfactual floor the ridge-return coverage disclosure also reports, so +# PR-K's ridge coverage is directly comparable to PR-J's (which used 20 for its +# VALLEY leg). +_COMPARISON_FLOOR = 20 + +# The counterfactual peak floor the disclosure quantifies: how many days would +# still be valid if the PEAK leg were held to the RIDGE leg's floor instead of +# its own. DERIVED from the ridge gate, not hardcoded. +_COUNTERFACTUAL_PEAK_FLOOR = PEAK_RIDGE_MIN_RIDGE_BARS + +# --------------------------------------------------------------------------- # +# Ridge-scarcity coverage (measured, never assumed) — PR-J valley/ridge VWAP +# --------------------------------------------------------------------------- # + +@dataclass(frozen=True) +class RidgeCoverage: + """Realized ridge-bar distribution + day-validity rate over the whole universe. + + Built from the per-day diagnostics the factor emits, so the numbers describe the + days the factor actually saw. ``symbol_days`` counts EVERY symbol-day with visible + bars, including the leading warm-up days that have no same-slot baseline yet; + ``classifiable_days`` counts those that clear PR-F's classifiable floor. The + headline ``validity_rate`` is taken over ``classifiable_days``, because a day with + no baseline fails for a PR-F warm-up reason rather than a ridge-scarcity one and + would otherwise make the ridge gate look worse than it is — both denominators are + reported so the reader can check that framing. The gate-failure counts are NOT + mutually exclusive (a thin day can fail several gates at once) and are reported for + shape, not as a partition. + """ + + symbol_days: int + classifiable_days: int + valid_days: int + ridge_percentiles: tuple[tuple[int, float], ...] + ridge_mean: float + valley_median: float + days_below_ridge_gate: int + days_below_valley_gate: int + days_below_classifiable_gate: int + # Counterfactual: how many days would survive if the ridge leg were held to the + # VALLEY floor. Quantifies exactly what the lowered threshold buys. + valid_days_at_valley_floor: int + # The gates this run actually applied, so the disclosure can never describe the + # module defaults while the run used something else. + min_ridge_bars: int = VALLEY_RIDGE_MIN_RIDGE_BARS + min_valley_bars: int = VALLEY_RIDGE_MIN_VALLEY_BARS + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE + + @property + def validity_rate(self) -> float: + """Valid days as a share of CLASSIFIABLE days (see the class docstring).""" + if not self.classifiable_days: + return float("nan") + return self.valid_days / self.classifiable_days + + def render(self) -> str: + """One-line, secret-free summary for the run log and the CLI.""" + pctl = " ".join(f"p{p}={v:.0f}" for p, v in self.ridge_percentiles) + return ( + f"ridge scarcity: symbol_days={self.symbol_days} " + f"classifiable_days={self.classifiable_days} " + f"valid_days={self.valid_days} ({self.validity_rate:.1%} of classifiable) " + f"ridge_bars[{pctl} mean={self.ridge_mean:.1f}] " + f"valley_bars_median={self.valley_median:.0f} " + f"below_ridge_gate({self.min_ridge_bars})=" + f"{self.days_below_ridge_gate} " + f"below_valley_gate({self.min_valley_bars})=" + f"{self.days_below_valley_gate} " + f"below_classifiable_gate({self.min_classifiable})=" + f"{self.days_below_classifiable_gate} " + f"valid_if_ridge_floor_were_{self.min_valley_bars}=" + f"{self.valid_days_at_valley_floor}" + ) + + + +def summarize_ridge_coverage( + frames: list[pd.DataFrame], + *, + min_ridge_bars: int = VALLEY_RIDGE_MIN_RIDGE_BARS, + min_valley_bars: int = VALLEY_RIDGE_MIN_VALLEY_BARS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, +) -> RidgeCoverage: + """Reduce the per-symbol day-level diagnostics to the scarcity disclosure. + + The three floors must be the ones the RUN applied, not the module defaults — + otherwise the disclosure would describe gates that were never enforced. + """ + gates = dict( + min_ridge_bars=min_ridge_bars, + min_valley_bars=min_valley_bars, + min_classifiable=min_classifiable, + ) + empty = tuple((p, float("nan")) for p in _RIDGE_PCTL) + if not frames: + return RidgeCoverage( + symbol_days=0, + classifiable_days=0, + valid_days=0, + ridge_percentiles=empty, + ridge_mean=float("nan"), + valley_median=float("nan"), + days_below_ridge_gate=0, + days_below_valley_gate=0, + days_below_classifiable_gate=0, + valid_days_at_valley_floor=0, + **gates, + ) + diag = pd.concat(frames, ignore_index=True) + classifiable = diag["classifiable_bars"].to_numpy(dtype=float) + valid = diag["valid"].to_numpy(dtype=bool) + # The bar-count distributions describe the days that had a fair chance: a warm-up day + # with no same-slot baseline has zero of everything and would only drag the + # percentiles towards zero for a reason that has nothing to do with ridge scarcity. + scored = classifiable >= min_classifiable + ridge = diag.loc[scored, "ridge_bars"].to_numpy(dtype=float) + valley = diag.loc[scored, "valley_bars"].to_numpy(dtype=float) + # The counterfactual holds the ridge leg to the valley floor, leaving every other + # gate exactly as it was. + at_valley_floor = valid & ( + diag["ridge_bars"].to_numpy(dtype=float) >= min_valley_bars + ) + return RidgeCoverage( + symbol_days=int(len(diag)), + classifiable_days=int(scored.sum()), + valid_days=int(valid.sum()), + ridge_percentiles=( + tuple((p, float(np.percentile(ridge, p))) for p in _RIDGE_PCTL) + if ridge.size + else empty + ), + ridge_mean=float(ridge.mean()) if ridge.size else float("nan"), + valley_median=float(np.median(valley)) if valley.size else float("nan"), + days_below_ridge_gate=int((ridge < min_ridge_bars).sum()), + days_below_valley_gate=int((valley < min_valley_bars).sum()), + days_below_classifiable_gate=int((~scored).sum()), + valid_days_at_valley_floor=int(at_valley_floor.sum()), + **gates, + ) + + + +# --------------------------------------------------------------------------- # +# Ridge-scarcity coverage (measured, never assumed) — PR-K ridge minute return +# --------------------------------------------------------------------------- # + +@dataclass(frozen=True) +class RidgeReturnCoverage: + """Realized ridge-bar distribution + day-validity rate over the whole universe. + + Built from the per-day diagnostics the factor emits, so the numbers describe the days + the factor actually saw. ``symbol_days`` counts EVERY symbol-day with visible bars, + including the leading warm-up days that have no same-slot baseline yet; + ``classifiable_days`` counts those that clear PR-F's classifiable floor. The headline + ``validity_rate`` is taken over ``classifiable_days``, because a day with no baseline + fails for a PR-F warm-up reason rather than a ridge-scarcity one and would otherwise + make the ridge gate look worse than it is — both denominators are reported so the + reader can check that framing. + + TWO ridge counts are tracked, because this factor gates on the narrower one: total + ``ridge_bars`` and the ``ridge_return_bars`` subset that carries a valid minute return + (the day's first visible bar is excluded by the within-day lag even when it is a + ridge). Reporting both makes the return-guard attrition visible instead of implicit. + The gate-failure counts are NOT mutually exclusive and are reported for shape, not as + a partition. + """ + + symbol_days: int + classifiable_days: int + valid_days: int + ridge_return_percentiles: tuple[tuple[int, float], ...] + ridge_return_mean: float + ridge_bars_mean: float + ridge_bars_median: float + days_below_ridge_gate: int + days_below_classifiable_gate: int + # Counterfactual: how many days would survive at the higher floor PR-J used for its + # VALLEY leg. Quantifies exactly what the scarcity-driven threshold buys. + valid_days_at_comparison_floor: int + # The gates this run actually applied, so the disclosure can never describe the module + # defaults while the run used something else. + min_ridge_bars: int = RIDGE_RETURN_MIN_RIDGE_BARS + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE + comparison_floor: int = _COMPARISON_FLOOR + + @property + def validity_rate(self) -> float: + """Valid days as a share of CLASSIFIABLE days (see the class docstring).""" + if not self.classifiable_days: + return float("nan") + return self.valid_days / self.classifiable_days + + @property + def return_guard_attrition(self) -> float: + """Share of ridge bars LOST to the return guard (mean over classifiable days).""" + if not np.isfinite(self.ridge_bars_mean) or self.ridge_bars_mean <= 0.0: + return float("nan") + return 1.0 - self.ridge_return_mean / self.ridge_bars_mean + + def render(self) -> str: + """One-line, secret-free summary for the run log and the CLI.""" + pctl = " ".join(f"p{p}={v:.0f}" for p, v in self.ridge_return_percentiles) + return ( + f"ridge scarcity: symbol_days={self.symbol_days} " + f"classifiable_days={self.classifiable_days} " + f"valid_days={self.valid_days} ({self.validity_rate:.1%} of classifiable) " + f"ridge_return_bars[{pctl} mean={self.ridge_return_mean:.1f}] " + f"ridge_bars_mean={self.ridge_bars_mean:.1f} " + f"ridge_bars_median={self.ridge_bars_median:.0f} " + f"return_guard_attrition={self.return_guard_attrition:.1%} " + f"below_ridge_gate({self.min_ridge_bars})={self.days_below_ridge_gate} " + f"below_classifiable_gate({self.min_classifiable})=" + f"{self.days_below_classifiable_gate} " + f"valid_if_floor_were_{self.comparison_floor}=" + f"{self.valid_days_at_comparison_floor}" + ) + + + +def summarize_ridge_return_coverage( + frames: list[pd.DataFrame], + *, + min_ridge_bars: int = RIDGE_RETURN_MIN_RIDGE_BARS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + comparison_floor: int = _COMPARISON_FLOOR, +) -> RidgeReturnCoverage: + """Reduce the per-symbol day-level diagnostics to the scarcity disclosure. + + The floors must be the ones the RUN applied, not the module defaults — otherwise the + disclosure would describe gates that were never enforced. + """ + gates = dict( + min_ridge_bars=min_ridge_bars, + min_classifiable=min_classifiable, + comparison_floor=comparison_floor, + ) + empty = tuple((p, float("nan")) for p in _RIDGE_PCTL) + if not frames: + return RidgeReturnCoverage( + symbol_days=0, + classifiable_days=0, + valid_days=0, + ridge_return_percentiles=empty, + ridge_return_mean=float("nan"), + ridge_bars_mean=float("nan"), + ridge_bars_median=float("nan"), + days_below_ridge_gate=0, + days_below_classifiable_gate=0, + valid_days_at_comparison_floor=0, + **gates, + ) + diag = pd.concat(frames, ignore_index=True) + classifiable = diag["classifiable_bars"].to_numpy(dtype=float) + valid = diag["valid"].to_numpy(dtype=bool) + # The bar-count distributions describe the days that had a fair chance: a warm-up day + # with no same-slot baseline has zero of everything and would only drag the + # percentiles towards zero for a reason that has nothing to do with ridge scarcity. + scored = classifiable >= min_classifiable + ridge_ret = diag.loc[scored, "ridge_return_bars"].to_numpy(dtype=float) + ridge_all = diag.loc[scored, "ridge_bars"].to_numpy(dtype=float) + # The counterfactual raises the ridge floor, leaving every other gate exactly as it was. + at_comparison = valid & ( + diag["ridge_return_bars"].to_numpy(dtype=float) >= comparison_floor + ) + return RidgeReturnCoverage( + symbol_days=int(len(diag)), + classifiable_days=int(scored.sum()), + valid_days=int(valid.sum()), + ridge_return_percentiles=( + tuple((p, float(np.percentile(ridge_ret, p))) for p in _RIDGE_PCTL) + if ridge_ret.size + else empty + ), + ridge_return_mean=float(ridge_ret.mean()) if ridge_ret.size else float("nan"), + ridge_bars_mean=float(ridge_all.mean()) if ridge_all.size else float("nan"), + ridge_bars_median=float(np.median(ridge_all)) if ridge_all.size else float("nan"), + days_below_ridge_gate=int((ridge_ret < min_ridge_bars).sum()), + days_below_classifiable_gate=int((~scored).sum()), + valid_days_at_comparison_floor=int(at_comparison.sum()), + **gates, + ) + + + +# --------------------------------------------------------------------------- # +# Peak-scarcity coverage (measured, never assumed) — PR-M peak/ridge amount +# --------------------------------------------------------------------------- # + +@dataclass(frozen=True) +class PeakCoverage: + """Realized peak-bar distribution + day-validity rate over the whole universe. + + Built from the per-day diagnostics the factor emits, so the numbers describe the days + the factor actually saw. ``symbol_days`` counts EVERY symbol-day with visible bars, + including the leading warm-up days that have no same-slot baseline yet; + ``classifiable_days`` counts those that clear PR-F's classifiable floor. The headline + ``validity_rate`` is taken over ``classifiable_days``, because a day with no baseline + fails for a PR-F warm-up reason rather than a peak-scarcity one and would otherwise + make the peak gate look worse than it is — both denominators are reported so the reader + can check that framing. The gate-failure counts are NOT mutually exclusive (a thin day + can fail several gates at once) and are reported for shape, not as a partition. + """ + + symbol_days: int + classifiable_days: int + valid_days: int + peak_percentiles: tuple[tuple[int, float], ...] + peak_mean: float + ridge_median: float + days_below_peak_gate: int + days_below_ridge_gate: int + days_below_classifiable_gate: int + # Counterfactual: how many days would survive if the PEAK leg were held to the RIDGE + # floor. Quantifies exactly what the lowered threshold buys. + valid_days_at_ridge_floor: int + # The gates this run actually applied, so the disclosure can never describe the module + # defaults while the run used something else. + min_peak_bars: int = PEAK_RIDGE_MIN_PEAK_BARS + min_ridge_bars: int = PEAK_RIDGE_MIN_RIDGE_BARS + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE + counterfactual_peak_floor: int = _COUNTERFACTUAL_PEAK_FLOOR + + @property + def validity_rate(self) -> float: + """Valid days as a share of CLASSIFIABLE days (see the class docstring).""" + if not self.classifiable_days: + return float("nan") + return self.valid_days / self.classifiable_days + + def render(self) -> str: + """One-line, secret-free summary for the run log and the CLI.""" + pctl = " ".join(f"p{p}={v:.0f}" for p, v in self.peak_percentiles) + return ( + f"peak scarcity: symbol_days={self.symbol_days} " + f"classifiable_days={self.classifiable_days} " + f"valid_days={self.valid_days} ({self.validity_rate:.1%} of classifiable) " + f"peak_bars[{pctl} mean={self.peak_mean:.1f}] " + f"ridge_bars_median={self.ridge_median:.0f} " + f"below_peak_gate({self.min_peak_bars})={self.days_below_peak_gate} " + f"below_ridge_gate({self.min_ridge_bars})={self.days_below_ridge_gate} " + f"below_classifiable_gate({self.min_classifiable})=" + f"{self.days_below_classifiable_gate} " + f"valid_if_peak_floor_were_{self.counterfactual_peak_floor}=" + f"{self.valid_days_at_ridge_floor}" + ) + + + +def summarize_peak_coverage( + frames: list[pd.DataFrame], + *, + min_peak_bars: int = PEAK_RIDGE_MIN_PEAK_BARS, + min_ridge_bars: int = PEAK_RIDGE_MIN_RIDGE_BARS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + counterfactual_peak_floor: int = _COUNTERFACTUAL_PEAK_FLOOR, +) -> PeakCoverage: + """Reduce the per-symbol day-level diagnostics to the scarcity disclosure. + + The three floors must be the ones the RUN applied, not the module defaults — otherwise + the disclosure would describe gates that were never enforced. + """ + gates = dict( + min_peak_bars=min_peak_bars, + min_ridge_bars=min_ridge_bars, + min_classifiable=min_classifiable, + counterfactual_peak_floor=counterfactual_peak_floor, + ) + empty = tuple((p, float("nan")) for p in _PEAK_PCTL) + if not frames: + return PeakCoverage( + symbol_days=0, + classifiable_days=0, + valid_days=0, + peak_percentiles=empty, + peak_mean=float("nan"), + ridge_median=float("nan"), + days_below_peak_gate=0, + days_below_ridge_gate=0, + days_below_classifiable_gate=0, + valid_days_at_ridge_floor=0, + **gates, + ) + diag = pd.concat(frames, ignore_index=True) + classifiable = diag["classifiable_bars"].to_numpy(dtype=float) + valid = diag["valid"].to_numpy(dtype=bool) + # The bar-count distributions describe the days that had a fair chance: a warm-up day + # with no same-slot baseline has zero of everything and would only drag the percentiles + # towards zero for a reason that has nothing to do with peak scarcity. + scored = classifiable >= min_classifiable + peak = diag.loc[scored, "peak_bars"].to_numpy(dtype=float) + ridge = diag.loc[scored, "ridge_bars"].to_numpy(dtype=float) + # The counterfactual raises the PEAK floor, leaving every other gate exactly as it was. + at_ridge_floor = valid & ( + diag["peak_bars"].to_numpy(dtype=float) >= counterfactual_peak_floor + ) + return PeakCoverage( + symbol_days=int(len(diag)), + classifiable_days=int(scored.sum()), + valid_days=int(valid.sum()), + peak_percentiles=( + tuple((p, float(np.percentile(peak, p))) for p in _PEAK_PCTL) + if peak.size + else empty + ), + peak_mean=float(peak.mean()) if peak.size else float("nan"), + ridge_median=float(np.median(ridge)) if ridge.size else float("nan"), + days_below_peak_gate=int((peak < min_peak_bars).sum()), + days_below_ridge_gate=int((ridge < min_ridge_bars).sum()), + days_below_classifiable_gate=int((~scored).sum()), + valid_days_at_ridge_floor=int(at_ridge_floor.sum()), + **gates, + ) + + + +# --------------------------------------------------------------------------- # +# Neutralization coverage (the reversal-neutralized factor's diagnostic) — PR-L +# --------------------------------------------------------------------------- # + +@dataclass(frozen=True) +class NeutralizationCoverage: + """What the reversal neutralization actually did, measured rather than assumed. + + A neutralization can fail quietly in two ways: the reversal can be unavailable for + most of the panel (so most residuals are NaN), or the cross-section can be too thin + to regress on many dates. Both are counted here and logged, so a coverage regression + is a number in the run record instead of an unexplained drop in sample size. + """ + + raw_rows: int # finite RAW qbar values + rev_rows: int # finite rev20 values on those rows + residual_rows: int # finite residuals (the shipped factor) + dates_total: int + dates_residualized: int # dates that cleared min_cross_section AND were non-degenerate + cross_section_min: int + cross_section_median: float + cross_section_max: int + raw_rev_spearman_mean: float # mean per-date exposure of the RAW factor to rev20 + def render(self) -> str: + """One-line, secret-free summary for the run log and the CLI. + + THE one intentional addition on top of the verbatim move: this disclosure + was the only one of the four the CLI rendered as an INLINE f-string + (catalogue section 3 "one-site normalization"), while its three siblings + already went through ``.render()``. The format below is exactly the line + the CLI used to inline, so the rendered output cannot drift from what + runs already print. + """ + return ( + f"neutralization (T-1 rev20): raw_rows={self.raw_rows} " + f"rev_paired={self.rev_rows} residual_rows={self.residual_rows} " + f"dates={self.dates_residualized}/{self.dates_total} " + f"cross_section min/med/max={self.cross_section_min}/" + f"{self.cross_section_median:.1f}/{self.cross_section_max} " + f"mean_spearman(raw,rev20)={self.raw_rev_spearman_mean:+.4f}" + ) + + + +def summarize_neutralization( + raw: pd.Series, + rev: pd.Series, + residual: pd.Series, + *, + min_cross_section: int, +) -> NeutralizationCoverage: + """Reduce the raw / reversal / residual panels to the coverage diagnostic.""" + raw_finite = raw.dropna() + rev_on_raw = rev.reindex(raw.index) + paired = pd.DataFrame({"f": raw, "r": rev_on_raw}).dropna() + + sizes: list[int] = [] + exposures: list[float] = [] + for _, g in paired.groupby(level=DATE_LEVEL, sort=True): + sizes.append(len(g)) + if len(g) >= min_cross_section: + f = g["f"].to_numpy(dtype=float) + r = g["r"].to_numpy(dtype=float) + fr = pd.Series(f).rank().to_numpy() + rr = pd.Series(r).rank().to_numpy() + if fr.std() > 0.0 and rr.std() > 0.0: + exposures.append(float(np.corrcoef(fr, rr)[0, 1])) + + resid_finite = residual.dropna() + dates_resid = int( + resid_finite.index.get_level_values(DATE_LEVEL).unique().size + ) + return NeutralizationCoverage( + raw_rows=int(len(raw_finite)), + rev_rows=int(len(paired)), + residual_rows=int(len(resid_finite)), + dates_total=int(raw.index.get_level_values(DATE_LEVEL).unique().size), + dates_residualized=dates_resid, + cross_section_min=int(min(sizes)) if sizes else 0, + cross_section_median=float(np.median(sizes)) if sizes else float("nan"), + cross_section_max=int(max(sizes)) if sizes else 0, + raw_rev_spearman_mean=( + float(np.mean(exposures)) if exposures else float("nan") + ), + ) + + + +# --------------------------------------------------------------------------- # +# The add-Section bridge (contract §3.6: may ADD, never drop a mandatory one) +# --------------------------------------------------------------------------- # +def to_section(name: str, coverage) -> Section: + """Pack a coverage disclosure dataclass into an add-Section. + + The payload is the dataclass's fields (asdict) plus its derived properties + (``validity_rate`` / ``return_guard_attrition`` where defined); the note is + the disclosure's one-line ``render()``, so the artifact and the run log can + never state different numbers. The verdict reads ONLY the mandatory section + payloads (``analytics/eval/report.py::extract_verdict_inputs``), so an extra + section added this way cannot move a verdict — pinned by test. + """ + payload: dict[str, object] = dict(asdict(coverage)) + for prop in ("validity_rate", "return_guard_attrition"): + if hasattr(coverage, prop): + payload[prop] = getattr(coverage, prop) + return Section(name=name, payload=payload, note=coverage.render()) + + +@dataclass(frozen=True) +class DisclosureBinding: + """Which disclosure a factor publishes: the section name + its summarizer.""" + + section_name: str + summarize: Callable[[list[pd.DataFrame]], object] + + +#: factor class -> its day-level gate-attrition disclosure. Only the factors +#: that HAVE such a disclosure appear (catalogue §三 mechanism A: the +#: diagnostics sink); every other factor publishes NO per-day disclosure, which +#: is stated (``None``) rather than inferred from an empty frame. +_DISCLOSURE_BY_CLASS: dict[type, DisclosureBinding] = { + ValleyRidgeVwapRatioFactor: DisclosureBinding( + "ridge_scarcity_coverage", summarize_ridge_coverage + ), + RidgeMinuteReturnFactor: DisclosureBinding( + "ridge_scarcity_coverage", summarize_ridge_return_coverage + ), + PeakRidgeAmountRatioFactor: DisclosureBinding( + "peak_scarcity_coverage", summarize_peak_coverage + ), +} + + +def disclosure_binding_for(factor) -> DisclosureBinding | None: + """The disclosure binding for ``factor``, or None when it publishes none. + + Keyed by the factor CLASS (the same keying + ``factors/compute/minute/binding.py`` uses for its diagnostics bindings), so + the two tables agree by construction; a factor with a diagnostics binding + but no summarizer here is a readable error at the call site, never a + silently reduced mixture. + """ + return _DISCLOSURE_BY_CLASS.get(type(factor)) + + +__all__ = [ + "DisclosureBinding", + "NeutralizationCoverage", + "PeakCoverage", + "RidgeCoverage", + "RidgeReturnCoverage", + "disclosure_binding_for", + "summarize_neutralization", + "summarize_peak_coverage", + "summarize_ridge_coverage", + "summarize_ridge_return_coverage", + "to_section", +] diff --git a/qt/factor_eval_providers.py b/qt/factor_eval_providers.py new file mode 100644 index 0000000..19ef47b --- /dev/null +++ b/qt/factor_eval_providers.py @@ -0,0 +1,202 @@ +"""Real-cache provider wiring for the unified factor-eval runner (D5 C4, commit 1). + +The materializer (``factors.materialize``) is factor-agnostic and takes its data +access INJECTED (layering red line #3/#10: ``factors`` never touches a feed, a +token, or qt). This module is the qt side of that injection for the evaluation +plane: it wires the persistent tushare caches and the pipeline's daily panel +into the provider protocols, and assembles the service bundle the unified +exec-only ``FactorEvalRunner`` (the next C4 commit) consumes. + +* :class:`CacheMinuteProvider` — cache-only 1min bars, MOVED here from + ``qt.factor_hotpath_smoke`` (single source; the smoke and the probes import + it from here now). Zero live calls: ``IntradayParquetStore.read_range`` has + no fetch closure, so a missing month is an empty read, never a warm. +* :class:`DailyEvalPanelProvider` — serves the pipeline's loaded daily panel + through the ``DailyPanelProvider`` protocol. +* :func:`build_eval_service` / :class:`EvalServiceBundle` — the one wiring of + cache -> universe -> panel -> enrichments -> store + sources, in the SAME + call order the legacy eval runners used (``qt/eval_jump_amount_corr.py``). +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import pandas as pd + +from data.cache.intraday_cache import ENDPOINT as INTRADAY_ENDPOINT +from data.cache.intraday_cache import READ_COLUMNS +from data.cache.intraday_parquet_store import IntradayParquetStore +from data.clean.intraday_schema import ( + RAW_INTRADAY_FREQ, + empty_intraday_bars, + normalize_intraday_bars, +) +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL +from factors.materialize import MaterializeSources +from factors.store import FactorValueStore +from qt.pipeline import ( + _build_cache, + _build_universe, + _load_panel, + _log_run_cache_stats, + _maybe_enrich_covariates, + _maybe_enrich_value, +) + +if TYPE_CHECKING: + from qt.config import RootConfig + +#: The intraday cache's DECLARED earliest bar date (measured on the real cache: +#: several CSI500 names carry 1min bars from 2015-01-05). The pooled saturation +#: loop needs a declared floor; it must never infer one from row counts. +CACHE_MINUTE_DATA_START = "2015-01-05" + +#: The factor value store's runtime artifact root (design §3.4 R22: runtime +#: artifacts live under artifacts/, gitignored, never in the source tree). +DEFAULT_STORE_ROOT = "artifacts/factor_store" + + +class CacheMinuteProvider: + """Cache-only MinuteBarProvider: per-symbol read + normalize, zero live calls.""" + + def __init__(self, root: str) -> None: + self._store = IntradayParquetStore(root) + self.calls = 0 + self.live_calls = 0 # provably 0 — read_range has no fetch closure + + def earliest_available(self, symbols): + """The cache's DECLARED minute-data floor (measured: bars from 2015-01-05). + + Declared, never inferred from row counts — a long mid-history no-bar gap + (a suspension) is indistinguishable from exhaustion by row count, which + is exactly the unsound signal the pooled saturation loop refuses. + """ + return pd.Timestamp(CACHE_MINUTE_DATA_START) + + def minute_bars(self, symbols, start, end): + self.calls += 1 + if not symbols: + return empty_intraday_bars() + parts = [] + s = pd.Timestamp(start) + e = pd.Timestamp(end) + for sym in symbols: + part = self._store.read_range(INTRADAY_ENDPOINT, sym, RAW_INTRADAY_FREQ, s, e) + if part.empty: + continue + parts.append( + normalize_intraday_bars( + part.rename(columns={"bar_end": "time"})[READ_COLUMNS], freq=RAW_INTRADAY_FREQ + ) + ) + if not parts: + return empty_intraday_bars() + return pd.concat(parts).sort_index(kind="mergesort") + + +class DailyEvalPanelProvider: + """``DailyPanelProvider`` over the pipeline's loaded evaluation panel. + + CLOSE-VIEW, NOT LAGGED — and that is exactly right: the materializer's own + daily path applies ``factors.view_lag.daily_decision_lag`` for the decision + view (the prev-day shift with the field-level ``open`` exception, R18), so + the provider must hand it values dated at their natural close date. A + pre-lagged panel would be shifted TWICE. ``qt.pipeline._load_panel``'s + product (raw bars enriched with tradability flags, front-adjusted in + memory) is precisely such an un-lagged close-view panel, so this provider + is a thin window/symbol slicer over it. + + WINDOW SEMANTICS: the panel covers the configured ``[data.start, data.end]`` + window (the same window the legacy runners loaded). A materializer load + request reaching before the panel's first date is served what exists — + the trailing-trading-day trim then treats the panel's left edge like the + data start (honest under-warm NaN), which reproduces the legacy runners' + warmup geometry rather than silently inventing deeper history. + """ + + def __init__(self, panel: pd.DataFrame) -> None: + self._panel = panel + + def daily_panel(self, symbols, start, end): + if self._panel.empty: + return self._panel + dates = self._panel.index.get_level_values(DATE_LEVEL) + mask = (dates >= pd.Timestamp(start)) & (dates <= pd.Timestamp(end)) + out = self._panel[mask] + keep = [str(s) for s in symbols] + syms = out.index.get_level_values(SYMBOL_LEVEL) + return out[syms.isin(keep)] + + +@dataclass(frozen=True) +class EvalServiceBundle: + """The wired evaluation-plane service inputs (immutable, no secrets). + + ``store`` — the factor value store (design §3.4 R22 root). + ``sources`` — the materializer's injected data access (daily + minute). + ``panel`` — the enriched close-view daily panel (book factors + + neutralization covariates ride on it, as in the legacy runners). + ``symbols`` — the union of historical constituents from ``_build_universe``. + ``cache`` — the shared read-through cache (None when caching is disabled). + + The universe object itself is deliberately NOT carried yet: this bundle has + no consumer until the unified runner lands (the next C4 commit), and what + it needs beyond these five fields is that commit's to add. + """ + + store: FactorValueStore + sources: MaterializeSources + panel: pd.DataFrame + symbols: list[str] + cache: object + + +def build_eval_service( + cfg: RootConfig, + logger: logging.Logger, + *, + value_factors=(), + store_root: str = DEFAULT_STORE_ROOT, +) -> EvalServiceBundle: + """Wire the evaluation service bundle from the live config. + + SAME CALL ORDER as the legacy eval runners (``qt/eval_jump_amount_corr.py``): + shared cache -> PIT universe -> raw panel -> value (pe/pb) enrichment -> + neutralization covariates -> one cache-stats log line. ``value_factors`` + are the factors needing ``daily_basic`` pe/pb (the book's value_ep / + value_bp); empty means the enrichment is a no-op passthrough. + + The minute provider reads ``cfg.data.cache.root_dir`` directly: it is + cache-only by construction (never a live call), so it is wired whether or + not the read-through cache is enabled. + """ + cache = _build_cache(cfg) + _universe, symbols = _build_universe(cfg, logger, cache) + panel = _load_panel(cfg, symbols, logger, cache) + panel = _maybe_enrich_value(cfg, panel, symbols, list(value_factors), logger, cache) + panel = _maybe_enrich_covariates(cfg, panel, symbols, logger, cache) + _log_run_cache_stats(cache, logger) + sources = MaterializeSources( + daily=DailyEvalPanelProvider(panel), + minute=CacheMinuteProvider(cfg.data.cache.root_dir), + ) + return EvalServiceBundle( + store=FactorValueStore(store_root), + sources=sources, + panel=panel, + symbols=list(symbols), + cache=cache, + ) + + +__all__ = [ + "CACHE_MINUTE_DATA_START", + "DEFAULT_STORE_ROOT", + "CacheMinuteProvider", + "DailyEvalPanelProvider", + "EvalServiceBundle", + "build_eval_service", +] diff --git a/qt/factor_eval_reconcile.py b/qt/factor_eval_reconcile.py new file mode 100644 index 0000000..d802849 --- /dev/null +++ b/qt/factor_eval_reconcile.py @@ -0,0 +1,1048 @@ +"""run-factor-eval-reconcile: the four-leg reconciliation harness (D5 C4, commit 3). + +Built in C4, consumed by C5 (the full four-leg audit). Three modes plus ONE hard +gate that runs at every mode's entry: + +* HARD GATE — the frozen exec baseline (``qt.exec_baseline_freeze``) must verify + **77/77**. Baseline bytes that are unreadable or modified are a HARD ERROR, + never a skip: without the baseline there is nothing to reconcile against, and + a reconcile that "passes" without one is the compare_postmerge failure shape. + +* ``--mode panels`` — the raw factor panel from the factor SERVICE + (``factors.service.panel``, decision view x exec_to_exec) vs the frozen D1 + panel, cell by cell. ``jump_amount_corr_20`` compares against the + ``pr_c_cutoff_fix`` reference instead (its D1 panel encodes the pre-cutoff + definition and is kept untouched). Differences are only allowed in the + NAMED, machine-checkable classes registered in catalogue §七之三: + + 1. ``warmup_left_extension`` — the old runners anchored their left edge at + ``data.start`` (2021-07-01) and never extended the window, so the frozen + panel's first w-1 trading dates are under-warmed (NaN or a partial pool); + the materializer saturates left to the minute cache's real start + (2015-01-05). This is a loading-geometry difference, not a value bug: + * bounded factor: the differing row sits in the grid's first + ``lookback_depth - 1`` trading dates — ALL THREE directions allowed + (frozen-NaN -> new-finite, finite -> finite as a partial pool fills, + and a finite value on a row the frozen panel does not have); + * valid-day-POOLED factor: the early region [2021-07-01, 2021-10-31], + same three directions, and the per-month counts must be + non-increasing ("按月递减至零"; a violation fails the mode). + 2. ``float_reordering_tail`` — scattered finite-vs-finite cells with rel + diff <= 5e-12 (rolling-correlation summation order; the JC1 1e-12 gate + is the attributable floor, this is the measured tail above it). The + class is CAPPED at 101 cells (the measured jump count); more fails. + 3. ``threshold_flip_tail`` — count factors (volume_peak): rolling-sigma + float noise (~4e-10) times an integer volume sitting on the peak + threshold flips the count by EXACTLY +/-1 on a sparse (symbol, day) + cluster (measured: 20 cells, 600623.SH 2023-06-15..07-14). Bounds: + |delta| == 1 exactly, rel <= 1e-2, at most 25 cells; more fails. + 4. the jump cutoff reference — handled by the reference-path selection above. + + Anything else (finite->NaN, finite-vs-finite beyond every named tail, a + finite value on a row the frozen panel does not have outside the warmup + boundary) is UNCLASSIFIED and fails the mode. Extra all-NaN rows in the + served panel are the D4c NaN footprint (registered drift #3) and are + counted, not failed. + +* ``--mode reports`` — the new ``factor_eval_*`` JSON/Markdown artifacts vs the + frozen exec artifacts, VALUE-LEVEL (never byte-level: §七/§七之二 registered + the contract-v1.0/v1.1 additions, so byte equality is known-false and must + not be the pass condition). Every flattened leaf difference must be one of + the registered items; everything else must be EQUAL (numerics within + METRIC_REL_TOL). Registered relaxations (catalogue §七之三): + + * ``warmup_aggregate_effect`` — aggregate-metric leaves (``sections[*]``, + ``verdict.reasons[*]``, ``verdict.axes.*``) legitimately move: they + aggregate a panel whose warmup cells moved (the panels leg is the value + gate). Only DIGIT-CARRYING changes qualify — a pure label flip (verdict + PASS -> FAIL carries no digit) stays unregistered and fails. + * ``registered_sanity_stem_rename`` — the runner's deliberate + ``eval__`` -> ``factor_eval__`` sanity-report rename + (anti-collision). + * ``registered_run_order_artifact`` — ``exec_price_artifact_reused`` + False -> True within one session (later runs reuse the artifact the + first run built). + * the with-book run in the default ``decision`` book mode carries the + decision-view book — its Incremental-axis numerics legitimately differ from + the frozen close-view-book artifact. Those leaves are REPORTED in full + (class ``book_view_effect``), not gated; the strict with-book gate lives on + the ``_bookclose`` artifact (legacy-faithful book), which isolates the + engine from the intended book-view change (the handoff's (a)/(b) split). + * ``jump_amount_corr_20``: the frozen exec artifact is PRE-cutoff-fix (its + IC -0.030840 encodes the lookahead), so value differences ARE the declared + correction — including ``spec.version`` / ``spec.description`` / + ``sections[7].payload.factor_version``. They are only accepted when the + new JSON carries the ``corrections`` block (contract v1.1); without it + they fail. Jump's value-level verification lives in the panels leg + (cutoff reference) and in the post-fix restated numbers, not here. + + Markdown is diffed as a line set, then same-key lines are PAIRED into + row-level changes before classification (a bare set diff reports every + value change twice — one removal + one addition with the same ``- key:`` + head). + + Cross-check within the NEW pair: no-book vs with-book ``eval_config`` may + differ ONLY in ``book_view`` (None vs the run's book mode) — the first time + the two exec artifacts legally differ (§七之二), not a regression. + +* ``--mode anchors`` — the service path produces engine values for the + ``hand_anchors_d2.json`` rows (same real-cache bars) and reconciles against + the HAND side. The hand side was computed with the OLD loading geometry + (left edge anchored at 2021-07-01), so a mismatch inside the warmup + boundary is the registered ``warmup_left_extension`` class — for bounded + factors the frozen grid's first ``lookback_depth - 1`` trading dates (the + harness never had this class for bounded factors before §七之三; the + first-run failures were exactly this asymmetry). EXPECTED SIGNAL (do not + misread as a regression): jump's NON-warmup rows must reconcile — the + service carries the corrected, truncated definition, so the frozen-engine + mismatches on random dates go GREEN here (rel ~1e-15). A jump mismatch + outside the warmup boundary FAILS this mode. + +Layering: this module is qt-side orchestration (it may wire caches and the +service); the classification rules are pure functions so the unit tests need +no cache, no network, and no real baseline. +""" + +from __future__ import annotations + +import json +import math +import re +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +import pandas as pd + +from factors import registry as factor_registry +from factors import service as factor_service +from factors.compute.minute.binding import is_valid_day_pooled +from qt.exec_baseline_freeze import ( + DEFAULT_FROZEN_ROOT, + DEFAULT_MANIFEST, + FACTORS as REPORT_FACTOR_NAMES, + FrozenExecBaseline, +) + +# --------------------------------------------------------------------------- # +# Constants (each pre-registered in the handoff §3 / catalogue §七/§七之二) +# --------------------------------------------------------------------------- # +FROZEN_PANELS_DIR = Path("artifacts/refactor_baseline/panels") +JUMP_CUTOFF_REFERENCE_DIR = Path("artifacts/refactor_baseline/pr_c_cutoff_fix/panels") +ANCHORS_JSON = Path("artifacts/refactor_baseline/hand_anchors_d2.json") + +#: JC1 ruling: rolling-mechanism factors reconcile to <= 1e-12 relative +#: (attributable float reordering; the NaN mask must match EXACTLY). +PANEL_REL_TOL = 1e-12 +#: Aggregated metrics (IC / ICIR / spreads) pass through more summation stages +#: than a single factor cell; the gate is still a hard number, reported per run. +METRIC_REL_TOL = 1e-9 + +#: ``float_reordering_tail`` (catalogue §七之三): scattered finite-vs-finite +#: cells just above the JC1 floor (measured jump tail: 101 cells, rel +#: 1.0e-12..2.9e-12, rolling-correlation summation order). Bounded on BOTH +#: axes — the global tolerance is NOT widened, so a real regression keeps +#: failing; a flood of float-noise cells fails the cap. +FLOAT_TAIL_REL_TOL = 5e-12 +FLOAT_TAIL_MAX_CELLS = 101 + +#: ``threshold_flip_tail`` (catalogue §七之三): count factors (volume_peak) — +#: rolling-sigma float noise (~4e-10) times an integer volume sitting on the +#: peak threshold flips the count by EXACTLY +/-1. Measured: 20 cells, +#: 600623.SH 2023-06-15..07-14. An amplitude > 1 or a larger cluster fails. +THRESHOLD_FLIP_REL_TOL = 1e-2 +THRESHOLD_FLIP_MAX_CELLS = 25 + +EARLY_REGION_LO = pd.Timestamp("2021-07-01") +EARLY_REGION_HI = pd.Timestamp("2021-10-31") + +#: factor_id -> the frozen exec artifact's report name (qt.exec_baseline_freeze +#: FACTORS). Closed map: an unknown factor id is a readable error, never a guess. +_FACTOR_TO_REPORT_NAME: dict[str, str] = { + "jump_amount_corr_20": "jump_amount_corr", + "minute_ideal_amp_10": "minute_ideal_amplitude", + "amp_marginal_anomaly_vol_20": "amp_marginal_anomaly_vol", + "volume_peak_count_20": "volume_peak_count", + "intraday_amp_cut_10": "intraday_amp_cut", + "peak_interval_kurtosis_20": "peak_interval_kurtosis", + "valley_relative_vwap_20": "valley_relative_vwap", + "valley_ridge_vwap_ratio_20": "valley_ridge_vwap_ratio", + "ridge_minute_return_20": "ridge_minute_return", + "valley_price_quantile_20": "valley_price_quantile", + "peak_ridge_amount_ratio_20": "peak_ridge_amount_ratio", +} +assert set(_FACTOR_TO_REPORT_NAME.values()) == set(REPORT_FACTOR_NAMES) + +#: Registered JSON additions (catalogue §七 spec 16->20 keys; §七之二 contract +#: v1.0/v1.1). ``corrections`` and ``spec.requires`` are matched as PREFIXES +#: (their leaves are indexed: ``corrections[0]...``, ``spec.requires[0]...``). +ALLOWED_ADDED_JSON_PATHS: frozenset[str] = frozenset({ + "eval_config.view", + "eval_config.return_basis", + "eval_config.book_view", + "eval_contract_version", + "spec.adjustment", + "spec.overnight_boundary", + "spec.lookback_depth", +}) +ALLOWED_ADDED_JSON_PREFIXES: tuple[str, ...] = ("corrections", "spec.requires") + +#: Aggregate-metric leaves: numeric/prose changes HERE are the downstream +#: effect of the registered panel-level ``warmup_left_extension`` differences +#: (the panels leg is the value gate; reports are aggregates of the panel). +#: Anything OUTSIDE these paths (verdict labels, spec, eval_config, criteria) +#: still fails on change. +AGGREGATE_CHANGE_PATH_PREFIXES: tuple[str, ...] = ( + "sections[", + "verdict.reasons[", + "verdict.axes.", +) + +#: Registered Markdown additions (§七之二 #3: the four provenance lines; #5: one +#: corrections provenance line per declared correction). +ALLOWED_ADDED_MD_PREFIXES: tuple[str, ...] = ( + "- evaluation contract:", + "- requires (endpoint inputs):", + "- adjustment / overnight boundary:", + "- lookback depth (trailing trading days):", + "- ⚠️ CORRECTION (", +) + +MAX_EXAMPLES = 10 + + +class ReconciliationError(RuntimeError): + """A reconciliation precondition failed (baseline unreadable, bad input).""" + + +# --------------------------------------------------------------------------- # +# Hard gate +# --------------------------------------------------------------------------- # +def require_baseline_verified(baseline: FrozenExecBaseline) -> None: + """The 77/77 hard gate. Any problem is a hard error, never a skip.""" + ok, problems = baseline.verify_all() + if problems or ok != baseline.file_count: + raise ReconciliationError( + f"frozen exec baseline did NOT verify: {ok}/{baseline.file_count} ok; " + f"{len(problems)} problem(s): {problems[:5]}. Reconciliation without " + "the baseline is meaningless — fix the baseline first." + ) + + +def load_verified_baseline(repo_root: Path) -> FrozenExecBaseline: + """Construct the frozen-baseline reader and run the hard gate.""" + baseline = FrozenExecBaseline( + Path(repo_root) / DEFAULT_FROZEN_ROOT, Path(repo_root) / DEFAULT_MANIFEST + ) + require_baseline_verified(baseline) + return baseline + + +# --------------------------------------------------------------------------- # +# Shared leaf diff helpers +# --------------------------------------------------------------------------- # +def _flatten(obj: object, prefix: str = "") -> dict[str, object]: + """Flatten a JSON-like structure to {dotted.path[index]: leaf}.""" + out: dict[str, object] = {} + if isinstance(obj, dict): + for key, value in obj.items(): + out.update(_flatten(value, f"{prefix}.{key}" if prefix else str(key))) + elif isinstance(obj, list): + for i, value in enumerate(obj): + out.update(_flatten(value, f"{prefix}[{i}]")) + else: + out[prefix] = obj + return out + + +def _numeric_rel_diff(old: object, new: object) -> float | None: + """Relative difference for genuine numerics (bool is NOT numeric here).""" + if isinstance(old, bool) or isinstance(new, bool): + return None + if isinstance(old, (int, float)) and isinstance(new, (int, float)): + if math.isnan(old) and math.isnan(new): + return 0.0 + denom = max(abs(old), abs(new)) + return abs(old - new) / denom if denom > 0 else 0.0 + return None + + +# --------------------------------------------------------------------------- # +# reports mode: value-level JSON/Markdown diff with the registered whitelist +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class LeafDiff: + path: str + old: object + new: object + classification: str # registered_* | unregistered_* | book_view_effect + + +@dataclass +class ReportDiff: + """One JSON pair diff: every differing leaf, classified; ok = gate result.""" + + name: str + strict: bool + diffs: list[LeafDiff] = field(default_factory=list) + max_numeric_rel_diff: float = 0.0 + ok: bool = True + + def by_class(self, classification: str) -> list[LeafDiff]: + return [d for d in self.diffs if d.classification == classification] + + +def _is_registered_addition(path: str) -> bool: + return path in ALLOWED_ADDED_JSON_PATHS or any( + path == p or path.startswith(f"{p}.") or path.startswith(f"{p}[") + for p in ALLOWED_ADDED_JSON_PREFIXES + ) + + +def _carries_digit(value: object) -> bool: + return any(ch.isdigit() for ch in str(value)) + + +def _classify_value_change( + path: str, + old_v: object, + new_v: object, + *, + correction_expected: bool, + corrections_present: bool, + strict: bool, +) -> str: + """Classify one changed leaf (both sides present, values differ). + + Ladder (catalogue §七之三): + 1. the runner's deliberate ``sanity_report`` stem rename + (``eval__`` -> ``factor_eval__``) — anti-collision; + 2. ``exec_price_artifact_reused`` False -> True — a same-session run-order + artifact (the first run builds the exec-price artifact, later runs + reuse it). True -> False is NOT registered; + 3. jump: any value change while the structured ``corrections`` carrier is + present IS the declared correction (contract v1.1) — this covers + ``spec.version``, ``spec.description``, + ``sections[7].payload.factor_version`` and the restated numerics; + 4. the decision-view with-book pair: registered numeric drift, reported + not gated (``book_view_effect``); + 5. digit-carrying changes on AGGREGATE paths — the downstream effect of + the registered panel-level warmup differences + (``warmup_aggregate_effect``). The digit requirement keeps a pure + label flip (a verdict PASS -> FAIL carries no digit) UNREGISTERED. + 6. everything else: unregistered. + """ + if ( + path.endswith(".sanity_report") + and isinstance(old_v, str) + and isinstance(new_v, str) + and old_v.endswith("_exec_basis_sanity.md") + and new_v.endswith("_exec_basis_sanity.md") + ): + return "registered_sanity_stem_rename" + if path.endswith(".exec_price_artifact_reused") and old_v is False and new_v is True: + return "registered_run_order_artifact" + if correction_expected and corrections_present: + return "registered_correction_effect" + if not strict: + return "book_view_effect" + if path.startswith(AGGREGATE_CHANGE_PATH_PREFIXES) and ( + _carries_digit(old_v) or _carries_digit(new_v) + ): + return "warmup_aggregate_effect" + return "unregistered_change" + + +def diff_report_json( + old: dict, new: dict, *, name: str, strict: bool, correction_expected: bool +) -> ReportDiff: + """Diff one (frozen, new) JSON pair leaf by leaf against the registered list. + + ``strict`` gates numeric/string equality (no-book; with-book bookclose). + Non-strict mode (decision-view book) reports numeric differences as + ``book_view_effect`` without failing them — the book-view change is + registered; hiding it would be, gating it would false-positive. + ``correction_expected`` (jump): value differences are the declared + correction — accepted ONLY if the new JSON carries a ``corrections`` block. + """ + result = ReportDiff(name=name, strict=strict) + old_flat, new_flat = _flatten(old), _flatten(new) + corrections_present = "corrections" in new and bool(new["corrections"]) + for path in sorted(set(old_flat) | set(new_flat)): + in_old, in_new = path in old_flat, path in new_flat + if in_old and in_new: + old_v, new_v = old_flat[path], new_flat[path] + rel = _numeric_rel_diff(old_v, new_v) + if rel is not None: + result.max_numeric_rel_diff = max(result.max_numeric_rel_diff, rel) + if rel <= METRIC_REL_TOL: + continue + elif old_v == new_v: + continue + cls = _classify_value_change( + path, old_v, new_v, + correction_expected=correction_expected, + corrections_present=corrections_present, + strict=strict, + ) + result.diffs.append(LeafDiff(path, old_v, new_v, cls)) + elif in_new: + cls = ( + "registered_addition" + if _is_registered_addition(path) + else "unregistered_addition" + ) + result.diffs.append(LeafDiff(path, None, new_flat[path], cls)) + else: + result.diffs.append(LeafDiff(path, old_flat[path], None, "unregistered_removal")) + result.ok = not any(d.classification.startswith("unregistered") for d in result.diffs) + if correction_expected and not corrections_present: + # Value drift without the structured correction carrier is unexplained. + result.ok = False + return result + + +#: Pairing-key length cap. Long prose bullets legitimately pair by their +#: (digit-normalized) head — e.g. the incremental-axis reason line is ~103 +#: chars before its first colon — so the cap only excludes pathological +#: lines, not real prose. +_MD_KEY_MAX_CHARS = 200 + + +def _md_line_key(line: str) -> str | None: + """The pairing key of a ``- key: value`` Markdown line (None = unpairable).""" + head, sep, _value = line.partition(":") + return head if sep and len(head) <= _MD_KEY_MAX_CHARS else None + + +def _md_line_key_digit_normalized(line: str) -> str | None: + """Fallback key: some prose lines carry the changed number BEFORE the + first colon (``- [incremental FAIL] orthogonalized ICIR +0.120 is ~ 0 + (redundant with the book): after residualizing ...``), so the exact head + can never pair them. Normalizing digit runs in the head pairs exactly + those; a change in the prose WORDS still does not pair (the normalized + heads differ).""" + key = _md_line_key(line) + return re.sub(r"\d+", "#", key) if key is not None else None + + +def _pair_md_lines( + removed: list[str], added: list[str] +) -> tuple[list[tuple[str, str]], list[str], list[str]]: + """Pair removals with additions into row-level changes. + + Two passes: exact ``- key:`` head first, then the digit-normalized head + for the leftovers. Returns (pairs, unpaired_removals, unpaired_additions). + """ + pairs: list[tuple[str, str]] = [] + rem, add = list(removed), list(added) + for keyfn in (_md_line_key, _md_line_key_digit_normalized): + next_rem: list[str] = [] + next_add: list[str] = [] + rem_by: dict[str, list[str]] = {} + add_by: dict[str, list[str]] = {} + for line in rem: + key = keyfn(line) + (rem_by.setdefault(key, []) if key is not None else next_rem).append(line) + for line in add: + key = keyfn(line) + (add_by.setdefault(key, []) if key is not None else next_add).append(line) + for key in sorted(set(rem_by) | set(add_by)): + olds, news = rem_by.get(key, []), add_by.get(key, []) + for old_line, new_line in zip(olds, news): + pairs.append((old_line, new_line)) + next_rem.extend(olds[len(news):]) + next_add.extend(news[len(olds):]) + rem, add = next_rem, next_add + return pairs, rem, add + + +def _classify_md_change( + old_line: str, new_line: str, *, correction_expected: bool, strict: bool +) -> str: + """One KEY-PAIRED Markdown change; the ladder mirrors the JSON one.""" + old_v, new_v = old_line.partition(":")[2], new_line.partition(":")[2] + if ( + old_line.startswith("- sanity_report:") + and old_v.strip().endswith("_exec_basis_sanity.md") + and new_v.strip().endswith("_exec_basis_sanity.md") + ): + return "registered_sanity_stem_rename" + if correction_expected: + return "registered_correction_effect" + if not strict: + return "book_view_effect" + if _carries_digit(old_line) and _carries_digit(new_line): + # A change carrying numbers (in the key OR the value — prose bullets + # can carry the number before the colon) follows the numeric + # attribution of the registered panel-level warmup differences. A + # pure label flip (no digits on either side) stays UNREGISTERED. + return "warmup_aggregate_effect" + return "unregistered_change" + + +def diff_report_md( + old_text: str, new_text: str, *, name: str, strict: bool = True, correction_expected: bool +) -> ReportDiff: + """Markdown: line-set diff, then pair same-key lines into row-level CHANGES. + + A bare set difference reports every value change twice (one removal + one + addition with the same ``- key:`` head) — 83-96 phantom "removals" in the + first run were exactly that. Pairing first, then classifying the pair with + the numeric-attribution ladder (the same one as the JSON leg), keeps the + teeth where they belong: UNPAIRED additions must be registered additions, + UNPAIRED removals are never registered (except jump's correction prose). + """ + result = ReportDiff(name=name, strict=strict) + old_counts = Counter(old_text.splitlines()) + new_counts = Counter(new_text.splitlines()) + removed = sorted((old_counts - new_counts).elements()) + added = sorted((new_counts - old_counts).elements()) + + pairs, unpaired_removals, unpaired_additions = _pair_md_lines(removed, added) + for old_line, new_line in pairs: + cls = _classify_md_change( + old_line, new_line, + correction_expected=correction_expected, strict=strict, + ) + result.diffs.append(LeafDiff(f"", old_line, new_line, cls)) + for line in unpaired_removals: + cls = "registered_correction_effect" if correction_expected else "unregistered_removal" + result.diffs.append(LeafDiff("", line, None, cls)) + for line in unpaired_additions: + cls = ( + "registered_addition" + if line.startswith(ALLOWED_ADDED_MD_PREFIXES) + else ("registered_correction_effect" if correction_expected else "unregistered_addition") + ) + result.diffs.append(LeafDiff("", None, line, cls)) + result.ok = not any(d.classification.startswith("unregistered") for d in result.diffs) + return result + + +def check_new_pair_consistency(no_book: dict, with_book: dict) -> list[LeafDiff]: + """The new no-book vs with-book eval_config may differ ONLY in book_view.""" + problems: list[LeafDiff] = [] + old_cfg = _flatten(no_book.get("eval_config", {}), "eval_config") + new_cfg = _flatten(with_book.get("eval_config", {}), "eval_config") + for path in sorted(set(old_cfg) | set(new_cfg)): + if old_cfg.get(path) != new_cfg.get(path) and path != "eval_config.book_view": + problems.append( + LeafDiff(path, old_cfg.get(path), new_cfg.get(path), "unregistered_change") + ) + return problems + + +# --------------------------------------------------------------------------- # +# panels mode: cell-by-cell with the named classes (catalogue §七之三) +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class PanelCellDiff: + date: str + symbol: str + frozen: object + new: object + classification: str # warmup_left_extension | float_reordering_tail | + # threshold_flip_tail | unclassified_* | unregistered_* + + +@dataclass +class PanelDiff: + factor_id: str + rows_frozen: int = 0 + rows_new: int = 0 + equal: int = 0 + within_tolerance: int = 0 + nan_footprint_rows: int = 0 + max_rel_diff: float = 0.0 + diffs: list[PanelCellDiff] = field(default_factory=list) + warmup_by_month: dict[str, int] = field(default_factory=dict) + warmup_by_direction: dict[str, int] = field(default_factory=dict) + warmup_monotonic: bool = True + ok: bool = True + + def by_class(self, classification: str) -> list[PanelCellDiff]: + return [d for d in self.diffs if d.classification == classification] + + +def classify_panel_differences( + new_values: pd.Series, + frozen: pd.DataFrame, + *, + factor_id: str, + is_pooled: bool, + lookback_depth: int, + tol: float = PANEL_REL_TOL, + early_lo: pd.Timestamp = EARLY_REGION_LO, + early_hi: pd.Timestamp = EARLY_REGION_HI, + float_tail_tol: float = FLOAT_TAIL_REL_TOL, + float_tail_max: int = FLOAT_TAIL_MAX_CELLS, + flip_rel_tol: float = THRESHOLD_FLIP_REL_TOL, + flip_max: int = THRESHOLD_FLIP_MAX_CELLS, +) -> PanelDiff: + """Classify every cell difference between the served and the frozen panel. + + ``new_values``: (date, symbol) -> raw value from the service (MultiIndex). + ``frozen``: the frozen parquet with columns date / symbol / . + Pure function — the unit tests drive it with synthetic frames. + + The warmup boundary is the LEFT-EXTENSION geometry difference: the old + runners anchored at ``data.start`` (the frozen grid's first w-1 trading + dates are under-warmed), the materializer saturates to the cache's real + start. Bounded: the grid's first ``lookback_depth - 1`` trading dates. + Pooled: the early region [early_lo, early_hi] with non-increasing + per-month counts. + """ + result = PanelDiff(factor_id=factor_id) + frozen = frozen.copy() + frozen["date"] = pd.to_datetime(frozen["date"]) + frozen_s = frozen.set_index(["date", "symbol"])[factor_id] + result.rows_frozen = int(len(frozen_s)) + result.rows_new = int(len(new_values)) + + # Bounded warmup boundary: the frozen grid's first (lookback_depth - 1) + # trading dates (measured: exactly where the under-warmed cells live; the + # w-th date is already fully warmed). The boundary is the GRID's left + # edge, not per-symbol: the old runners anchored every symbol's load at + # ``data.start``, so only the evaluation window's left edge is + # under-warmed — a late-starting symbol's first dates see the same bars + # in both geometries and cannot differ there. + grid_dates = sorted(frozen["date"].unique()) + n_warm = max(int(lookback_depth) - 1, 0) + warmup_dates = set(grid_dates[:n_warm]) + + def _in_warmup(date: pd.Timestamp, symbol: str) -> bool: + if is_pooled: + return early_lo <= date <= early_hi + return date in warmup_dates + + def _warmup_cell(date, symbol, frozen_v, new_v, direction: str) -> None: + result.warmup_by_direction[direction] = ( + result.warmup_by_direction.get(direction, 0) + 1 + ) + if is_pooled: + month = str(date.to_period("M")) + result.warmup_by_month[month] = result.warmup_by_month.get(month, 0) + 1 + result.diffs.append( + PanelCellDiff( + str(date.date()), str(symbol), + None if frozen_v is None or pd.isna(frozen_v) else float(frozen_v), + None if new_v is None or pd.isna(new_v) else float(new_v), + "warmup_left_extension", + ) + ) + + new_idx = new_values.index + frozen_idx = frozen_s.index + frozen_symbols = set(frozen["symbol"].unique()) + + # Extra served rows: allowed as the all-NaN D4c footprint (drift #3), or + # as new-only finite rows INSIDE the warmup boundary for a symbol the + # frozen panel knows (a grid gap the old runner did not emit). A finite + # value for a symbol the frozen panel does not have at all is never + # registered. + extra = new_idx.difference(frozen_idx) + for key in extra: + value = new_values.loc[key] + if pd.isna(value): + result.nan_footprint_rows += 1 + elif key[1] in frozen_symbols and _in_warmup(key[0], key[1]): + _warmup_cell(key[0], key[1], None, value, "new_only_finite") + else: + result.diffs.append( + PanelCellDiff(str(key[0].date()), str(key[1]), None, float(value), + "unregistered_new_finite_row") + ) + + for (date, symbol), frozen_v in frozen_s.items(): + new_v = new_values.loc[(date, symbol)] if (date, symbol) in new_idx else float("nan") + if pd.isna(frozen_v) and pd.isna(new_v): + result.equal += 1 + continue + if pd.notna(frozen_v) and pd.notna(new_v): + denom = max(abs(frozen_v), abs(new_v)) + rel = abs(frozen_v - new_v) / denom if denom > 0 else 0.0 + result.max_rel_diff = max(result.max_rel_diff, rel) + if rel <= tol: + result.within_tolerance += 1 + elif _in_warmup(date, symbol): + _warmup_cell(date, symbol, frozen_v, new_v, "finite_to_finite") + elif rel <= float_tail_tol: + result.diffs.append( + PanelCellDiff(str(date.date()), str(symbol), float(frozen_v), + float(new_v), "float_reordering_tail") + ) + elif abs(frozen_v - new_v) == 1.0 and rel <= flip_rel_tol: + result.diffs.append( + PanelCellDiff(str(date.date()), str(symbol), float(frozen_v), + float(new_v), "threshold_flip_tail") + ) + else: + result.diffs.append( + PanelCellDiff(str(date.date()), str(symbol), float(frozen_v), + float(new_v), "unclassified_finite_vs_finite") + ) + continue + if pd.notna(frozen_v): # finite -> NaN: never in an allowed class + result.diffs.append( + PanelCellDiff(str(date.date()), str(symbol), float(frozen_v), None, + "unclassified_frozen_finite_new_nan") + ) + continue + # frozen NaN -> new finite: the warmup class or unclassified. + if _in_warmup(date, symbol): + _warmup_cell(date, symbol, frozen_v, new_v, "nan_to_finite") + else: + result.diffs.append( + PanelCellDiff(str(date.date()), str(symbol), None, float(new_v), + "unclassified_nan_to_finite") + ) + + counts = [result.warmup_by_month[m] for m in sorted(result.warmup_by_month)] + result.warmup_monotonic = all(b <= a for a, b in zip(counts, counts[1:])) + result.ok = ( + not any(d.classification.startswith("unclassified") or + d.classification.startswith("unregistered") for d in result.diffs) + and result.warmup_monotonic + and len(result.by_class("float_reordering_tail")) <= float_tail_max + and len(result.by_class("threshold_flip_tail")) <= flip_max + ) + return result + + +def frozen_panel_path(factor_id: str, repo_root: Path) -> Path: + """The frozen reference for the panels leg (jump -> the cutoff reference).""" + if factor_id == "jump_amount_corr_20": + return Path(repo_root) / JUMP_CUTOFF_REFERENCE_DIR / f"{factor_id}.parquet" + return Path(repo_root) / FROZEN_PANELS_DIR / f"{factor_id}.parquet" + + +# --------------------------------------------------------------------------- # +# anchors mode: the service path vs the hand-computed anchor rows +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class AnchorRowResult: + factor_id: str + cls: str + date: str + symbol: str + hand: float + service: float + rel_diff: float + classification: str # ok | warmup_left_extension | failed + + +@dataclass +class AnchorsDiff: + factor_id: str + rows: list[AnchorRowResult] = field(default_factory=list) + ok: bool = True + + def by_class(self, classification: str) -> list[AnchorRowResult]: + return [r for r in self.rows if r.classification == classification] + + +def classify_anchor_row( + *, + factor_id: str, + cls: str, + date: pd.Timestamp, + symbol: str, + hand: float, + service: float, + is_pooled: bool, + tol: float, + warmup_dates: frozenset | None = None, +) -> AnchorRowResult: + """One anchor row. The hand side was computed with the OLD loading + geometry (left edge anchored at 2021-07-01), the service with the new + (saturating) one — so a warmup-region row legitimately differs, for + bounded factors exactly as in the panels leg (``warmup_dates`` = the + frozen grid's first ``lookback_depth - 1`` trading dates). + + jump's NON-warmup rows MUST reconcile: they are the signal that the + service carries the corrected, truncated definition (the five frozen- + engine mismatches go green here); a miss outside the warmup boundary + means the truncation is not carried and fails the mode. + """ + if math.isnan(hand) and math.isnan(service): + rel = 0.0 + elif math.isnan(hand) or math.isnan(service): + rel = math.inf + else: + denom = max(abs(hand), abs(service)) + rel = abs(hand - service) / denom if denom > 0 else 0.0 + if rel <= tol: + classification = "ok" + elif warmup_dates is not None and date in warmup_dates: + classification = "warmup_left_extension" + elif factor_id == "jump_amount_corr_20": + classification = "failed" + elif is_pooled and EARLY_REGION_LO <= date <= EARLY_REGION_HI: + classification = "warmup_left_extension" + else: + classification = "failed" + return AnchorRowResult( + factor_id, cls, str(date.date()), symbol, hand, service, rel, classification + ) + + +def load_anchor_rows(factor_id: str, repo_root: Path) -> list[dict]: + """The factor's hand-anchor rows (placeholder rows carry no date/symbol).""" + path = Path(repo_root) / ANCHORS_JSON + if not path.exists(): + raise ReconciliationError(f"hand anchors not found: {path}") + payload = json.loads(path.read_text(encoding="utf-8")) + return [ + row + for row in payload.get("frozen14", []) + if row.get("factor_id") == factor_id and row.get("symbol") and row.get("date") + ] + + +# --------------------------------------------------------------------------- # +# The real-cache orchestration (qt-side wiring; NOT unit-tested) +# --------------------------------------------------------------------------- # +def _report_name(factor_id: str) -> str: + try: + return _FACTOR_TO_REPORT_NAME[factor_id] + except KeyError: + raise ReconciliationError( + f"{factor_id!r} has no frozen exec artifact name mapping." + ) from None + + +def _build_bundle(cfg, logger, symbols=None, value_factors=(), store_root=None): + """The service bundle for panels/anchors (same wiring as the runner).""" + from qt.factor_eval_providers import ( + DEFAULT_STORE_ROOT, + CacheMinuteProvider, + DailyEvalPanelProvider, + ) + from factors.materialize import MaterializeSources + from factors.store import FactorValueStore + from qt.pipeline import _build_cache, _load_panel, _maybe_enrich_value + + cache = _build_cache(cfg) + if symbols is None: + from qt.pipeline import _build_universe + + _universe, symbols = _build_universe(cfg, logger, cache) + panel = _load_panel(cfg, list(symbols), logger, cache) + panel = _maybe_enrich_value(cfg, panel, list(symbols), list(value_factors), logger, cache) + sources = MaterializeSources( + daily=DailyEvalPanelProvider(panel), + minute=CacheMinuteProvider(cfg.data.cache.root_dir), + ) + store = FactorValueStore(store_root or DEFAULT_STORE_ROOT) + return store, sources, panel, list(symbols), cache + + +def run_panels_mode(config_path: str, factor_id: str, repo_root: Path) -> PanelDiff: + from data.availability_policy import ReturnBasis, View + from qt.config import load_config + from qt.pipeline import _make_logger + + load_verified_baseline(repo_root) + cfg = load_config(config_path) + logger = _make_logger( + Path(cfg.output.log_dir) / f"factor_eval_reconcile_{factor_id}.log", + name="qt.factor_eval_reconcile", + ) + store, sources, daily_panel, symbols, _cache = _build_bundle(cfg, logger) + decisions = [ + factor_service.DecisionPoint(d) + for d in pd.Index(pd.unique(daily_panel.index.get_level_values("date"))).sort_values() + ] + served = factor_service.panel( + [factor_id], symbols, decisions, store=store, sources=sources, + view=View.DECISION, basis=ReturnBasis.EXEC_TO_EXEC, + ) + live_calls = int(getattr(sources.minute, "live_calls", 0)) + if live_calls != 0: + raise ReconciliationError( + f"cache-only violated: stk_mins_live_calls={live_calls}. ABORT." + ) + factor = factor_registry.build(factor_id) + frozen = pd.read_parquet(frozen_panel_path(factor_id, repo_root)) + result = classify_panel_differences( + served[factor_id], + frozen, + factor_id=factor_id, + is_pooled=is_valid_day_pooled(factor), + lookback_depth=int(factor.spec.lookback_depth), + ) + logger.info( + "panels %s: rows frozen=%d new=%d equal=%d tol=%d warmup=%d(%s) " + "float_tail=%d threshold_flip=%d footprint=%d unclassified=%d " + "max_rel=%.3e live_calls=%d ok=%s", + factor_id, result.rows_frozen, result.rows_new, result.equal, + result.within_tolerance, len(result.by_class("warmup_left_extension")), + result.warmup_by_direction, + len(result.by_class("float_reordering_tail")), + len(result.by_class("threshold_flip_tail")), + result.nan_footprint_rows, + len([d for d in result.diffs if d.classification.startswith("un")]), + result.max_rel_diff, live_calls, result.ok, + ) + return result + + +def run_reports_mode( + config_path: str, factor_id: str, repo_root: Path, *, report_dir: Path | None = None +) -> list[ReportDiff]: + from qt.config import load_config + + baseline = load_verified_baseline(repo_root) + cfg = load_config(config_path) + report_dir = report_dir or Path(cfg.output.report_dir) + report_name = _report_name(factor_id) + correction_expected = factor_id == "jump_amount_corr_20" + stem = f"factor_eval_{factor_id}" + + results: list[ReportDiff] = [] + new_no_book = json.loads((report_dir / f"{stem}_exec_no_book.json").read_text()) + new_with_book = json.loads((report_dir / f"{stem}_exec_with_book.json").read_text()) + + pairs = [ + ("no_book", new_no_book, True), + # decision-view book: registered numeric drift, reported not gated. + ("with_book(decision)", new_with_book, False), + ] + bookclose = report_dir / f"{stem}_exec_with_book_bookclose.json" + if bookclose.exists(): + pairs.append( + ("with_book(bookclose)", json.loads(bookclose.read_text()), True) + ) + for label, new_json, strict in pairs: + book = "no_book" if label == "no_book" else "with_book" + frozen_json = baseline.report_json(report_name, book) + results.append( + diff_report_json( + frozen_json, new_json, name=f"{stem}_exec_{book}.json[{label}]", + strict=strict, correction_expected=correction_expected, + ) + ) + new_md = (report_dir / f"{stem}_exec_{book}{'_bookclose' if 'bookclose' in label else ''}.md").read_text() + frozen_md = baseline.read_text(f"eval_{report_name}_exec_{book}.md") + results.append( + diff_report_md( + frozen_md, new_md, name=f"{stem}_exec_{book}.md[{label}]", + strict=strict, correction_expected=correction_expected, + ) + ) + problems = check_new_pair_consistency(new_no_book, new_with_book) + if problems: + results.append( + ReportDiff( + name="new-pair eval_config consistency", strict=True, + diffs=problems, ok=False, + ) + ) + return results + + +def run_anchors_mode(config_path: str, factor_id: str, repo_root: Path) -> AnchorsDiff: + from data.availability_policy import ReturnBasis, View + from qt.config import load_config + from qt.hand_anchors_d2 import TOL + from qt.pipeline import _make_logger + + load_verified_baseline(repo_root) + rows = load_anchor_rows(factor_id, repo_root) + if not rows: + raise ReconciliationError(f"no hand-anchor rows for {factor_id!r}.") + factor = factor_registry.build(factor_id) + try: + pooled = is_valid_day_pooled(factor) + is_minute = True + except KeyError: + pooled = False # daily factors are not in the minute partition sets + is_minute = False + warmup_dates: frozenset | None = None + if is_minute and not pooled: + # Bounded factors: the same warmup boundary as the panels leg — the + # frozen grid's first (lookback_depth - 1) trading dates. + panel_path = frozen_panel_path(factor_id, repo_root) + if panel_path.exists(): + grid = pd.Index( + pd.unique(pd.to_datetime(pd.read_parquet(panel_path)["date"])) + ).sort_values() + n_warm = max(int(factor.spec.lookback_depth) - 1, 0) + warmup_dates = frozenset(grid[:n_warm]) + cfg = load_config(config_path) + logger = _make_logger( + Path(cfg.output.log_dir) / f"factor_eval_reconcile_anchors_{factor_id}.log", + name="qt.factor_eval_reconcile", + ) + symbols = sorted({row["symbol"] for row in rows}) + value_factors = () + if factor_id in ("value_ep", "value_bp"): + from qt.factor_eval_runner import _build_book_factors + + value_factors = tuple(_build_book_factors()) + store, sources, _panel, symbols, _cache = _build_bundle( + cfg, logger, symbols=symbols, value_factors=value_factors + ) + decisions = [factor_service.DecisionPoint(pd.Timestamp(row["date"])) for row in rows] + served = factor_service.panel( + [factor_id], symbols, decisions, store=store, sources=sources, + view=View.DECISION, basis=ReturnBasis.EXEC_TO_EXEC, + ) + live_calls = int(getattr(sources.minute, "live_calls", 0)) + if live_calls != 0: + raise ReconciliationError( + f"cache-only violated: stk_mins_live_calls={live_calls}. ABORT." + ) + result = AnchorsDiff(factor_id=factor_id) + series = served[factor_id] + for row in rows: + key = (pd.Timestamp(row["date"]), row["symbol"]) + service_v = float(series.loc[key]) if key in series.index else float("nan") + result.rows.append( + classify_anchor_row( + factor_id=factor_id, cls=row["class"], date=key[0], symbol=key[1], + hand=float(row["hand"]), service=service_v, + is_pooled=pooled, tol=TOL, warmup_dates=warmup_dates, + ) + ) + result.ok = all(r.classification != "failed" for r in result.rows) + logger.info( + "anchors %s: %d rows, ok=%d warmup=%d failed=%d live_calls=%d -> %s", + factor_id, len(result.rows), len(result.by_class("ok")), + len(result.by_class("warmup_left_extension")), + len(result.by_class("failed")), live_calls, result.ok, + ) + return result + + +__all__ = [ + "AGGREGATE_CHANGE_PATH_PREFIXES", + "ALLOWED_ADDED_JSON_PATHS", + "ALLOWED_ADDED_MD_PREFIXES", + "AnchorsDiff", + "EARLY_REGION_HI", + "EARLY_REGION_LO", + "FLOAT_TAIL_MAX_CELLS", + "FLOAT_TAIL_REL_TOL", + "LeafDiff", + "METRIC_REL_TOL", + "PANEL_REL_TOL", + "PanelDiff", + "ReconciliationError", + "ReportDiff", + "THRESHOLD_FLIP_MAX_CELLS", + "THRESHOLD_FLIP_REL_TOL", + "check_new_pair_consistency", + "classify_anchor_row", + "classify_panel_differences", + "diff_report_json", + "diff_report_md", + "frozen_panel_path", + "load_anchor_rows", + "load_verified_baseline", + "require_baseline_verified", + "run_anchors_mode", + "run_panels_mode", + "run_reports_mode", +] diff --git a/qt/factor_eval_runner.py b/qt/factor_eval_runner.py new file mode 100644 index 0000000..00c7292 --- /dev/null +++ b/qt/factor_eval_runner.py @@ -0,0 +1,451 @@ +"""run-factor-eval: the UNIFIED exec-only factor evaluation runner (D5 C4, commit 2). + +ONE runner body for the eleven minute-derived factors, replacing the eleven +near-identical ``qt/eval_*.py`` main flows (the difference catalogue +``docs/factors/d5_runner_difference_catalogue.md`` §四 lists what was already +identical; §二/§三 list the four extension points this runner parameterizes): + +1. factor construction + params — the registry's pre-registered defaults + (``factors.registry.build``); the legacy runners passed the same values + explicitly. +2. the post-loop cross-sectional finalizer — the D4c read-assembly combine, + which ``factors.service`` runs over exactly the requested universe + (``intraday_amp_cut``'s z-score happens there, not in a runner). +3. the add-Section coverage disclosures — ``qt.factor_eval_disclosures`` + (mechanism A diagnostics sink through ``factors.service.panel``'s + ``diagnostics=``; the sink re-materializes the request so a warm store can + still serve the disclosure — see ``factors.service._ensure_coverage``). +4. metric keys — ``qt.exec_basis_eval``'s extraction. + +The subject factor's VALUES come from the factor SERVICE +(``factors.service.panel``, decision view x exec_to_exec basis), not from a +per-runner minute loader: the store read-through + the per-symbol streaming +materializer are the single engine (D4/D4b/D4c), which is what retires the +second factor-sourcing path (design decision 3). The evaluation itself reuses +``qt.exec_basis_eval.run_exec_basis_evaluation`` verbatim — the SAME evaluator, +the SAME two book settings, the SAME 14:51-VWAP execution anchor. + +EXEC-ONLY: this runner never produces close_to_close artifacts. Its EvalConfig +declares ``view="decision", return_basis="exec_to_exec"`` EXPLICITLY — the +EvalConfig default is still the legacy close pairing, and an exec-only runner +must not inherit it. + +BOOK MODES (``--book-mode``): + +* ``close`` — faithfully replicates the legacy book path: direct + ``Factor.compute`` on the close-view enriched panel + one + ``_process_factors`` (template ``qt/eval_jump_amount_corr.py``). This is the + live defect design §1.1 records (close(d) book values against an exec + holding window opening at 14:51(d)); the artifact says ``book_view=close``. +* ``decision`` (default) — the book comes through the SAME service panel as + the subject, so the materializer's ``daily_decision_lag`` applies to it; + the artifact says ``book_view=decision``. + +The book is the frozen confirmed trio value_ep / value_bp / volatility_20 +(lead ruling Q1). Catalogue BUG 5 closure: a config ``factors:`` block that +disagrees with that effective book is a READABLE ERROR, never silently +ignored; the honest declaration is ``factors: []``. + +ARTIFACT ISOLATION: the report stem is ``factor_eval_{factor_id}`` (vs the +legacy ``eval_{name}``), so new and legacy artifacts coexist and can never +overwrite each other (the compare_postmerge lesson). In ``close`` book mode +the with-book artifacts gain a ``_bookclose`` suffix for the same reason. + +RunRegistry appends are a DELIBERATE DEFERRAL to D7's governance surface: this +runner writes its artifacts but does not register runs. + +``valley_price_quantile`` is NOT served: its raw compute also needs the DAILY +close panel (reversal neutralization), so it has no minute binding yet — that +lands in PR-C4b. Asking for it is a readable error, never a silent mis-compute. +""" + +from __future__ import annotations + +import os +import time +from dataclasses import dataclass, replace +from pathlib import Path + +import pandas as pd + +from analytics.eval import EvalConfig +from data.availability_policy import ReturnBasis, View +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL +from factors import registry as factor_registry +from factors import service as factor_service +from factors.compute.minute.binding import has_minute_diagnostics, is_minute_bound +from factors.compute.minute.valley_price_quantile import ValleyPriceQuantileFactor +from factors.spec import FactorSpec +from qt.config import RootConfig, load_config +from qt.exec_basis_eval import ExecBasisEvaluation, run_exec_basis_evaluation +from qt.factor_eval_disclosures import disclosure_binding_for, to_section +from qt.factor_eval_providers import EvalServiceBundle, build_eval_service +from qt.pipeline import _make_logger, _process_factors + +_LOGGER_NAME = "qt.factor_eval_runner" + +#: The frozen confirmed book (lead ruling Q1) — the same trio the eleven legacy +#: runners hardcoded in ``_build_book_factors`` (11 AST-identical bodies). +BOOK_IDS: tuple[str, ...] = ("value_ep", "value_bp", "volatility_20") + +#: What the effective book IS, for the catalogue BUG 5 consistency check: a +#: config ``factors:`` block must either be empty (``factors: []``) or declare +#: exactly this — anything else would sit in the file looking live while the +#: runner never reads it. +_EFFECTIVE_BOOK: dict[str, dict[str, object]] = { + "value_ep": {}, + "value_bp": {}, + "volatility_20": {"window": 20, "price_col": "close"}, +} + +BOOK_MODES: tuple[str, ...] = (View.DECISION.value, View.CLOSE.value) + + +def _build_book_factors() -> list: + """Instantiate the confirmed book (value_ep / value_bp / volatility_20).""" + from factors.compute.candidates import ValueFactor, VolatilityFactor + + return [ + ValueFactor("value_ep"), + ValueFactor("value_bp"), + VolatilityFactor(window=20), + ] + + +# --------------------------------------------------------------------------- # +# Config gates (the eleven identical bodies, collapsed once — catalogue C1/C2) +# --------------------------------------------------------------------------- # +def _check_preconditions(cfg: RootConfig) -> None: + """Fail readably if the config cannot drive a real, cache-only CSI500 eval.""" + if cfg.data.source != "tushare": + raise ValueError( + "run-factor-eval needs data.source='tushare' (real cached A-share " + f"data); got {cfg.data.source!r}." + ) + if not cfg.data.cache.enabled: + raise ValueError( + "run-factor-eval needs data.cache.enabled=true (it reads the " + "persistent tushare cache and never warms live)." + ) + if cfg.universe.type != "index": + raise ValueError( + "run-factor-eval needs universe.type='index' (PIT membership, e.g. " + f"000905.SH for CSI500); got {cfg.universe.type!r}." + ) + if not cfg.processing.neutralize.enabled: + raise ValueError( + "run-factor-eval expects processing.neutralize.enabled=true " + "(industry + size neutralization, matching the report's neutral " + "column and the EvalConfig declaration)." + ) + + +def _check_config_book(cfg: RootConfig) -> None: + """Catalogue BUG 5 closure: the config may not declare a book the run ignores. + + The legacy configs carry a ``factors:`` block naming the book, but no runner + ever read ``cfg.factors`` — harmless only while the two agree. Here the + subject comes from ``--factor`` and the book is fixed, so a non-empty block + must match the effective book EXACTLY; a mismatch is a readable error, and + the honest declaration is ``factors: []``. + """ + declared = {f.name: dict(f.params) for f in cfg.factors if f.enabled} + if not declared: + return + if declared != _EFFECTIVE_BOOK: + raise ValueError( + "run-factor-eval does not read config 'factors:' — the subject comes " + "from --factor and the book is fixed at value_ep / value_bp / " + "volatility_20(window=20, price_col=close). The config declares " + f"{sorted(declared)}, which disagrees; remove the block (declare " + "'factors: []') or make it match exactly, so the file cannot " + "silently describe a factor set the run never used (catalogue BUG 5)." + ) + + +def _build_eval_config(cfg: RootConfig) -> EvalConfig: + """The per-run EvalConfig — the catalogue C1 17 shared kwargs + exec identity. + + The 17 kwargs are the ones the eleven legacy ``_build_eval_config`` bodies + shared character-for-character. This runner is EXEC-ONLY, so the identity + is declared explicitly: ``view="decision", return_basis="exec_to_exec"`` + (the EvalConfig default is still the legacy close pairing and must not be + inherited here). ``book_view`` is NOT set on this config — the no-book and + with-book runs carry DIFFERENT book views, so the identity is derived per + run by ``qt.exec_basis_eval.exec_identity`` (which also re-derives the + subject view from the measured cutoff-safety fact). + """ + if cfg.oos is None: + raise ValueError( + "run-factor-eval requires an 'oos' section (split_date) so the " + "Predictive axis has an out-of-sample split to assess; add e.g. " + "oos: {split_date: '2024-01-01'}." + ) + return EvalConfig( + universe=cfg.universe.index_code or cfg.universe.type, + universe_is_pit=cfg.universe.type == "index", + start=cfg.data.start, + end=cfg.data.end, + is_exploratory=True, + post_hoc_selected=False, + rebalance="daily", + n_quantiles=int(cfg.analytics.quantiles), + cost_scenarios=(1.0, 2.0, 4.0), + oos_split=cfg.oos.split_date, + # winsorize is a P0 no-op in this codebase -> declare None (nothing clipped). + winsorize=None, + standardize="zscore" if cfg.processing.standardize.enabled else None, + neutralization=("industry", "size") if cfg.processing.neutralize.enabled else (), + industry_level=cfg.processing.neutralize.industry_level, + tuned=False, + # We evaluate ONE pre-registered factor whose sign came from the report + # (not a screen of our own); the report's own screen is a caveat noted in + # the run's prose, not our multiple-testing background. + n_factors_screened=1, + data_snapshot_id=cfg.data.cache.root_dir, + view=View.DECISION.value, + return_basis=ReturnBasis.EXEC_TO_EXEC.value, + ) + + +def _check_subject_supported(factor) -> None: + """The one subject this runner cannot serve yet (PR-C4b), as a readable error.""" + if isinstance(factor, ValleyPriceQuantileFactor): + raise ValueError( + f"run-factor-eval cannot serve {factor.name!r} yet: its raw compute " + "also needs the DAILY close panel (its reversal neutralization), so " + "it has no minute binding in factors.compute.minute.binding — the " + "binding lands in PR-C4b. Asking for it here must fail loudly, never " + "silently mis-compute." + ) + + +# --------------------------------------------------------------------------- # +# The evaluation flow +# --------------------------------------------------------------------------- # +def _decisions(panel: pd.DataFrame) -> list[factor_service.DecisionPoint]: + """One DecisionPoint per daily-panel trading day (14:50 default cutoff).""" + dates = pd.Index( + pd.unique(panel.index.get_level_values(DATE_LEVEL)), name=DATE_LEVEL + ).sort_values() + return [factor_service.DecisionPoint(d) for d in dates] + + +def _load_subject_raw( + bundle: EvalServiceBundle, + factor_id: str, + decisions: list[factor_service.DecisionPoint], + *, + diagnostics: list | None, +) -> pd.DataFrame: + """The subject's raw values from the factor SERVICE (the single engine). + + Decision view x exec_to_exec basis (the pairing is enforced at the service + boundary). The store read-through fills any (date, symbol) it lacks via the + ONE materializer; the cross-sectional combine runs at read-assembly over + exactly this universe (D4c). + """ + return factor_service.panel( + [factor_id], + bundle.symbols, + decisions, + store=bundle.store, + sources=bundle.sources, + view=View.DECISION, + basis=ReturnBasis.EXEC_TO_EXEC, + diagnostics=diagnostics, + ) + + +def _load_book_raw( + bundle: EvalServiceBundle, + decisions: list[factor_service.DecisionPoint], + *, + book_mode: str, +) -> pd.DataFrame: + """The book's raw values, in one of the two declared modes. + + ``close`` replicates the legacy runners exactly (direct compute on the + close-view enriched panel); ``decision`` takes the book through the same + service panel as the subject, so the materializer's decision-view lag + applies (the §1.1 close-view book defect does not exist on this path). + """ + if book_mode == View.CLOSE.value: + return pd.concat( + [f.compute(bundle.panel).rename(f.name) for f in _build_book_factors()], + axis=1, + ) + return factor_service.panel( + list(BOOK_IDS), + bundle.symbols, + decisions, + store=bundle.store, + sources=bundle.sources, + view=View.DECISION, + basis=ReturnBasis.EXEC_TO_EXEC, + ) + + +def _coverage_split( + subject_raw: pd.DataFrame, factor_id: str, symbols: list[str] +) -> tuple[tuple[str, ...], tuple[str, ...]]: + """(symbols with >= 1 finite raw value, requested symbols with none).""" + by_symbol = subject_raw[factor_id].notna().groupby(level=SYMBOL_LEVEL).any() + covered = {str(s) for s in by_symbol[by_symbol].index} + empty = sorted(set(map(str, symbols)) - covered) + return tuple(sorted(covered)), tuple(empty) + + +def _apply_bookclose_suffix( + exec_basis: ExecBasisEvaluation, report_dir: Path, stem: str +) -> ExecBasisEvaluation: + """``book_mode='close'`` artifact isolation: ``..._exec_with_book_bookclose.*``. + + The no-book artifacts carry no book at all, so they keep the shared stem; + the sanity report is book-independent for the same reason. + """ + mapping = { + "with_book_md": report_dir / f"{stem}_exec_with_book_bookclose.md", + "with_book_json": report_dir / f"{stem}_exec_with_book_bookclose.json", + "with_book_dashboard": report_dir + / f"{stem}_exec_with_book_bookclose_dashboard.png", + } + for field_name, dst in mapping.items(): + os.replace(getattr(exec_basis, field_name), dst) + return replace(exec_basis, **mapping) + + +# --------------------------------------------------------------------------- # +# Result container + the full glue +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class FactorEvalResult: + """Immutable summary of one run-factor-eval run.""" + + config: RootConfig + spec: FactorSpec + factor_id: str + book_mode: str + requested_symbols: int + covered_symbols: int + empty_symbols: int + factor_rows: int + minute_live_calls: int + coverage: object | None # a qt.factor_eval_disclosures coverage, when published + exec_basis: ExecBasisEvaluation + log_path: Path + elapsed: float + + +def run_factor_eval( + config_path: str, factor_id: str, *, book_mode: str = View.DECISION.value +) -> FactorEvalResult: + """Run the unified exec-only evaluation of ``factor_id`` (cache-only).""" + if book_mode not in BOOK_MODES: + raise ValueError( + f"run-factor-eval --book-mode must be one of {BOOK_MODES}; got " + f"{book_mode!r}." + ) + cfg = load_config(config_path) + _check_preconditions(cfg) + _check_config_book(cfg) + factor = factor_registry.build(factor_id) + _check_subject_supported(factor) + spec = factor.spec + eval_cfg = _build_eval_config(cfg) + + log_path = Path(cfg.output.log_dir) / f"factor_eval_{factor_id}.log" + logger = _make_logger(log_path, name=_LOGGER_NAME) + started = time.monotonic() + logger.info( + "eval config: %s rebalance=daily oos_split=%s factor=%s book_mode=%s", + eval_cfg.universe, eval_cfg.oos_split, factor_id, book_mode, + ) + + # The book's value factors need daily_basic pe/pb on the panel in BOTH book + # modes (in 'decision' mode the materializer's daily provider serves this + # same enriched panel to ValueFactor.compute). + bundle = build_eval_service(cfg, logger, value_factors=_build_book_factors()) + decisions = _decisions(bundle.panel) + + # Subject: the service panel. The diagnostics sink rides along only for the + # factors that publish a day-level gate-attrition disclosure (catalogue §三 + # mechanism A); asking for it re-materializes the request (a warm store can + # serve values but not the disclosure), which is the disclosed cost. + binding = disclosure_binding_for(factor) + if binding is None and is_minute_bound(factor) and has_minute_diagnostics(factor): + raise ValueError( + f"{factor_id}: the minute binding publishes per-day diagnostics but " + "qt.factor_eval_disclosures has no summarizer for them — a disclosure " + "without a home must be loud, never silently reduced away." + ) + sink: list | None = [] if binding is not None else None + subject_raw = _load_subject_raw(bundle, factor_id, decisions, diagnostics=sink) + covered, empty = _coverage_split(subject_raw, factor_id, bundle.symbols) + + coverage = None + extra_sections = [] + if binding is not None: + coverage = binding.summarize(sink) + logger.info("%s", coverage.render()) + extra_sections.append(to_section(binding.section_name, coverage)) + + subject_processed = _process_factors( + cfg, subject_raw[[spec.factor_id]], bundle.panel + ) + factor_series = subject_processed[spec.factor_id] + book_processed = _process_factors( + cfg, _load_book_raw(bundle, decisions, book_mode=book_mode), bundle.panel + ) + + stem = f"factor_eval_{factor_id}" + exec_basis = run_exec_basis_evaluation( + factor_series, + spec, + eval_cfg, + book_processed, + cfg=cfg, + panel=bundle.panel, + symbols=bundle.symbols, + logger=logger, + report_dir=Path(cfg.output.report_dir), + stem=stem, + book_view=book_mode, + extra_sections=extra_sections or None, + ) + if book_mode == View.CLOSE.value: + exec_basis = _apply_bookclose_suffix( + exec_basis, Path(cfg.output.report_dir), stem + ) + + logger.info( + "verdict exec no-book: %s (predictive=%s); with-book: %s (incremental=%s)", + exec_basis.no_book_metrics["deployment"], + exec_basis.no_book_metrics["predictive"], + exec_basis.with_book_metrics["deployment"], + exec_basis.with_book_metrics["incremental"], + ) + + return FactorEvalResult( + config=cfg, + spec=spec, + factor_id=factor_id, + book_mode=book_mode, + requested_symbols=len(bundle.symbols), + covered_symbols=len(covered), + empty_symbols=len(empty), + factor_rows=int(len(factor_series)), + minute_live_calls=int(getattr(bundle.sources.minute, "live_calls", 0)), + coverage=coverage, + exec_basis=exec_basis, + log_path=log_path, + elapsed=time.monotonic() - started, + ) + + +__all__ = [ + "BOOK_IDS", + "BOOK_MODES", + "FactorEvalResult", + "run_factor_eval", +] diff --git a/qt/factor_hotpath_smoke.py b/qt/factor_hotpath_smoke.py index f41eb6d..335a78f 100644 --- a/qt/factor_hotpath_smoke.py +++ b/qt/factor_hotpath_smoke.py @@ -36,22 +36,13 @@ import pandas as pd from data.cache.intraday_cache import ENDPOINT as INTRADAY_ENDPOINT -from data.cache.intraday_cache import READ_COLUMNS -from data.cache.intraday_parquet_store import IntradayParquetStore -from data.clean.intraday_schema import ( - RAW_INTRADAY_FREQ, - empty_intraday_bars, - normalize_intraday_bars, -) +from data.clean.intraday_schema import RAW_INTRADAY_FREQ from factors import registry as factor_registry from factors.compute.minute.binding import minute_raw_from_bars from factors.view_lag import minute_decision_cutoff +from qt.factor_eval_providers import CacheMinuteProvider DEFAULT_CACHE_ROOT = "artifacts/cache/tushare/v1" -#: The intraday cache's DECLARED earliest bar date (measured on the real cache: -#: several CSI500 names carry 1min bars from 2015-01-05). The pooled saturation -#: loop needs a declared floor; it must never infer one from row counts. -CACHE_MINUTE_DATA_START = "2015-01-05" DEFAULT_FACTORS = ( "jump_amount_corr_20", "minute_ideal_amp_10", @@ -65,44 +56,6 @@ DEFAULT_N_SYMBOLS = 40 -class CacheMinuteProvider: - """Cache-only MinuteBarProvider: per-symbol read + normalize, zero live calls.""" - - def __init__(self, root: str) -> None: - self._store = IntradayParquetStore(root) - self.calls = 0 - self.live_calls = 0 # provably 0 — read_range has no fetch closure - - def earliest_available(self, symbols): - """The cache's DECLARED minute-data floor (measured: bars from 2015-01-05). - - Declared, never inferred from row counts — a long mid-history no-bar gap - (a suspension) is indistinguishable from exhaustion by row count, which - is exactly the unsound signal the pooled saturation loop refuses. - """ - return pd.Timestamp(CACHE_MINUTE_DATA_START) - - def minute_bars(self, symbols, start, end): - self.calls += 1 - if not symbols: - return empty_intraday_bars() - parts = [] - s = pd.Timestamp(start) - e = pd.Timestamp(end) - for sym in symbols: - part = self._store.read_range(INTRADAY_ENDPOINT, sym, RAW_INTRADAY_FREQ, s, e) - if part.empty: - continue - parts.append( - normalize_intraday_bars( - part.rename(columns={"bar_end": "time"})[READ_COLUMNS], freq=RAW_INTRADAY_FREQ - ) - ) - if not parts: - return empty_intraday_bars() - return pd.concat(parts).sort_index(kind="mergesort") - - def _sample_symbols(root: str, n: int) -> list[str]: """First ``n`` symbols the intraday store has, in a stable directory order.""" base = Path(root) / f"{INTRADAY_ENDPOINT}" / f"freq={RAW_INTRADAY_FREQ}" diff --git a/qt/panel_leg_probe.py b/qt/panel_leg_probe.py index b6d8cc9..e66612e 100644 --- a/qt/panel_leg_probe.py +++ b/qt/panel_leg_probe.py @@ -49,7 +49,7 @@ from data.availability_policy import View from factors import registry as factor_registry from factors.materialize import MaterializeSources, materialize_range -from qt.factor_hotpath_smoke import CacheMinuteProvider +from qt.factor_eval_providers import CacheMinuteProvider DEFAULT_CACHE_ROOT = "artifacts/cache/tushare/v1" DEFAULT_PANEL_DIR = "artifacts/refactor_baseline/panels" diff --git a/qt/saturation_probe.py b/qt/saturation_probe.py index 8ec2e93..c7ad213 100644 --- a/qt/saturation_probe.py +++ b/qt/saturation_probe.py @@ -63,7 +63,7 @@ from factors import registry as factor_registry from factors.compute.minute.binding import minute_raw_from_bars from factors.materialize import MaterializeSources, materialize_range -from qt.factor_hotpath_smoke import CACHE_MINUTE_DATA_START, CacheMinuteProvider +from qt.factor_eval_providers import CACHE_MINUTE_DATA_START, CacheMinuteProvider DEFAULT_CACHE_ROOT = "artifacts/cache/tushare/v1" #: The frozen D1 panel is on the exact evaluation data plane, so its symbol list diff --git a/tests/test_factor_eval_disclosures.py b/tests/test_factor_eval_disclosures.py new file mode 100644 index 0000000..ec77efe --- /dev/null +++ b/tests/test_factor_eval_disclosures.py @@ -0,0 +1,376 @@ +"""The moved coverage disclosures: summarizers, add-Section bridge, gate constants. + +The summarizer tests MOVED here from the three legacy runner test files (the +old files are untouched and still pass through the runners' re-exports); +catalogue BUG 6 closure: the two colliding +``test_summarize_ridge_coverage_handles_no_frames`` definitions are renamed +with factor prefixes in this home. PR-M's ``summarize_peak_coverage`` had ZERO +tests (catalogue BUG 3) — the counterfactual + empty-frames pair its three +siblings all have is NET-NEW here. +""" + +from __future__ import annotations + +import inspect + +import numpy as np +import pandas as pd +import pytest + +from analytics.eval import EvalConfig, MANDATORY_SECTIONS, Section +from analytics.eval.render import canonical_sections +from analytics.eval.report import FactorEvalReport +from data.clean.intraday_amount_ratio import ( + PEAK_RIDGE_MIN_PEAK_BARS, + PEAK_RIDGE_MIN_RIDGE_BARS, +) +from data.clean.intraday_ridge_return import RIDGE_RETURN_MIN_RIDGE_BARS +from data.clean.intraday_valley_ridge_vwap import ( + VALLEY_RIDGE_MIN_RIDGE_BARS, + VALLEY_RIDGE_MIN_VALLEY_BARS, +) +from factors import registry as factor_registry +from factors.compute.minute import ( + peak_ridge_amount_ratio as peak_module, +) +from factors.compute.minute import ( + ridge_minute_return as ridge_return_module, +) +from factors.compute.minute import ( + valley_ridge_vwap_ratio as valley_ridge_module, +) +from qt.exec_basis_eval import _with_extra_sections +from qt.factor_eval_disclosures import ( + NeutralizationCoverage, + disclosure_binding_for, + summarize_neutralization, + summarize_peak_coverage, + summarize_ridge_coverage, + summarize_ridge_return_coverage, + to_section, +) + + +def _diag(columns: dict, days: int) -> pd.DataFrame: + return pd.DataFrame( + columns, + index=pd.DatetimeIndex( + pd.bdate_range("2022-01-03", periods=days), name="trade_date" + ), + ) + + +# --------------------------------------------------------------------------- # +# Moved: valley/ridge VWAP-ratio ridge-scarcity summarizer (PR-J) +# --------------------------------------------------------------------------- # +def test_valley_ridge_summarize_counterfactual_at_the_valley_floor(): + """The disclosure quantifies exactly what the LOWERED ridge floor buys.""" + diag = _diag( + { + "classifiable_bars": [240, 240, 240, 240], + "valley_bars": [200, 200, 200, 200], + "ridge_bars": [4, 12, 25, 30], + # as the factor would mark them under the default floor of 10 + "valid": [False, True, True, True], + }, + 4, + ) + cov = summarize_ridge_coverage([diag]) + assert cov.symbol_days == 4 + assert cov.classifiable_days == 4 # every day clears PR-F's classifiable floor + assert cov.valid_days == 3 + # holding the ridge leg to the VALLEY floor (20) would keep only the 25 / 30 days + assert cov.valid_days_at_valley_floor == 2 + assert cov.days_below_ridge_gate == 1 + assert cov.days_below_valley_gate == 0 + assert cov.ridge_mean == pytest.approx((4 + 12 + 25 + 30) / 4) + # defaults are the PINNED production floors when the caller does not override + assert cov.min_ridge_bars == VALLEY_RIDGE_MIN_RIDGE_BARS == 10 + assert cov.min_valley_bars == VALLEY_RIDGE_MIN_VALLEY_BARS == 20 + assert f"below_ridge_gate({VALLEY_RIDGE_MIN_RIDGE_BARS})" in cov.render() + + +def test_valley_ridge_summarize_handles_no_frames(): + cov = summarize_ridge_coverage([]) + assert cov.symbol_days == 0 + assert cov.classifiable_days == 0 + assert cov.valid_days == 0 + assert np.isnan(cov.validity_rate) + assert cov.render() # renders without dividing by zero + + +# --------------------------------------------------------------------------- # +# Moved: ridge-minute-return ridge-scarcity summarizer (PR-K) +# --------------------------------------------------------------------------- # +def test_ridge_return_summarize_reports_the_return_guard_attrition(): + """Ridge bars lost to the within-day lag must be VISIBLE, not silently absorbed.""" + diag = _diag( + { + "classifiable_bars": [240, 240], + "ridge_bars": [10, 30], + # one ridge on each day is the day's first visible bar -> no return + "ridge_return_bars": [9, 29], + "valid": [False, True], + }, + 2, + ) + cov = summarize_ridge_return_coverage([diag]) + assert cov.ridge_bars_mean == pytest.approx(20.0) + assert cov.ridge_return_mean == pytest.approx(19.0) + assert cov.return_guard_attrition == pytest.approx(1.0 - 19.0 / 20.0) + # the gate is applied to the RETURN-carrying count, so the 9-ridge day falls below 10 + assert cov.days_below_ridge_gate == 1 + assert "return_guard_attrition" in cov.render() + + +def test_ridge_return_summarize_counterfactual_at_the_comparison_floor(): + """The disclosure quantifies exactly what the scarcity floor buys (vs PR-J's 20).""" + diag = _diag( + { + "classifiable_bars": [240, 240, 240, 240], + "ridge_bars": [4, 12, 25, 30], + "ridge_return_bars": [4, 12, 25, 30], + # as the factor would mark them under the default floor of 10 + "valid": [False, True, True, True], + }, + 4, + ) + cov = summarize_ridge_return_coverage([diag]) + assert cov.symbol_days == 4 + assert cov.classifiable_days == 4 # every day clears PR-F's classifiable floor + assert cov.valid_days == 3 + # raising the ridge floor to PR-J's 20 would keep only the 25 / 30 days + assert cov.valid_days_at_comparison_floor == 2 + assert cov.days_below_ridge_gate == 1 + assert cov.ridge_return_mean == pytest.approx((4 + 12 + 25 + 30) / 4) + # defaults are the PINNED production floors when the caller does not override + assert cov.min_ridge_bars == RIDGE_RETURN_MIN_RIDGE_BARS == 10 + assert f"below_ridge_gate({RIDGE_RETURN_MIN_RIDGE_BARS})" in cov.render() + + +def test_ridge_return_summarize_handles_no_frames(): + cov = summarize_ridge_return_coverage([]) + assert cov.symbol_days == 0 + assert cov.classifiable_days == 0 + assert cov.valid_days == 0 + assert np.isnan(cov.validity_rate) + assert np.isnan(cov.return_guard_attrition) + assert cov.render() # renders without dividing by zero + + +# --------------------------------------------------------------------------- # +# NET-NEW (catalogue BUG 3): peak/ridge amount-ratio peak-scarcity summarizer +# --------------------------------------------------------------------------- # +def test_peak_summarize_counterfactual_at_the_ridge_floor(): + """The disclosure quantifies exactly what the LOWERED peak floor buys.""" + diag = _diag( + { + "classifiable_bars": [240, 240, 240, 240], + "peak_bars": [2, 7, 12, 30], + "ridge_bars": [30, 30, 30, 30], + # as the factor would mark them under the default PEAK floor of 5 + "valid": [False, True, True, True], + }, + 4, + ) + cov = summarize_peak_coverage([diag]) + assert cov.symbol_days == 4 + assert cov.classifiable_days == 4 # every day clears PR-F's classifiable floor + assert cov.valid_days == 3 + # raising the PEAK leg to the RIDGE floor (10) would keep only the 12 / 30 days + assert cov.valid_days_at_ridge_floor == 2 + assert cov.days_below_peak_gate == 1 + assert cov.days_below_ridge_gate == 0 + assert cov.peak_mean == pytest.approx((2 + 7 + 12 + 30) / 4) + assert cov.ridge_median == pytest.approx(30.0) + # defaults are the PINNED production floors when the caller does not override + assert cov.min_peak_bars == PEAK_RIDGE_MIN_PEAK_BARS == 5 + assert cov.min_ridge_bars == PEAK_RIDGE_MIN_RIDGE_BARS == 10 + assert f"below_peak_gate({PEAK_RIDGE_MIN_PEAK_BARS})" in cov.render() + assert f"below_ridge_gate({PEAK_RIDGE_MIN_RIDGE_BARS})" in cov.render() + + +def test_peak_summarize_handles_no_frames(): + cov = summarize_peak_coverage([]) + assert cov.symbol_days == 0 + assert cov.classifiable_days == 0 + assert cov.valid_days == 0 + assert np.isnan(cov.validity_rate) + assert cov.render() # renders without dividing by zero + + +# --------------------------------------------------------------------------- # +# Moved: reversal-neutralization summarizer (PR-L) +# --------------------------------------------------------------------------- # +def _panel(dates, syms, values, name): + idx = pd.MultiIndex.from_product([dates, syms], names=["date", "symbol"]) + return pd.Series(np.asarray(values, dtype=float).reshape(-1), index=idx, name=name) + + +def test_neutralization_summarize_counts_missing_reversal_rows(): + dates = pd.bdate_range("2023-01-02", periods=2) + syms = ["A", "B", "C"] + raw = _panel(dates, syms, [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], "q") + rev = _panel(dates, syms, [[1.0, 2.0, 3.0], [1.0, np.nan, np.nan]], "r") + resid = _panel(dates, syms, [[0.0, 0.0, 0.0], [np.nan] * 3], "f") + cov = summarize_neutralization(raw, rev, resid, min_cross_section=3) + assert cov.raw_rows == 6 + assert cov.rev_rows == 4 # only four rows had BOTH + assert cov.residual_rows == 3 + assert cov.dates_total == 2 + assert cov.dates_residualized == 1 + assert cov.cross_section_min == 1 and cov.cross_section_max == 3 + + +def test_neutralization_summarize_handles_an_all_missing_reversal(): + dates = pd.bdate_range("2023-01-02", periods=1) + syms = ["A", "B"] + raw = _panel(dates, syms, [[0.1, 0.2]], "q") + rev = _panel(dates, syms, [[np.nan, np.nan]], "r") + resid = _panel(dates, syms, [[np.nan, np.nan]], "f") + cov = summarize_neutralization(raw, rev, resid, min_cross_section=3) + assert cov.rev_rows == 0 + assert cov.residual_rows == 0 + assert cov.dates_residualized == 0 + + +def test_neutralization_render_matches_the_former_cli_inline_line(): + """render() is the line the CLI used to inline (catalogue section 3 one-site normalization) — pinned.""" + cov = NeutralizationCoverage( + raw_rows=6, rev_rows=4, residual_rows=3, dates_total=2, dates_residualized=1, + cross_section_min=1, cross_section_median=2.0, cross_section_max=3, + raw_rev_spearman_mean=0.123456, + ) + assert cov.render() == ( + "neutralization (T-1 rev20): raw_rows=6 rev_paired=4 residual_rows=3 " + "dates=1/2 cross_section min/med/max=1/2.0/3 " + "mean_spearman(raw,rev20)=+0.1235" + ) + + +# --------------------------------------------------------------------------- # +# Gate-constant single source: summarizer defaults == the compute gates +# --------------------------------------------------------------------------- # +def _defaults(fn) -> dict: + return { + name: p.default + for name, p in inspect.signature(fn).parameters.items() + if p.default is not inspect.Parameter.empty + } + + +def test_summarizer_defaults_are_the_gates_the_binding_applies(): + """The disclosure must report the floors the RUN applied, not a hand copy. + + The minute binding (``factors/compute/minute/binding.py``) calls every + compute function with all gate parameters at their module defaults, so the + summarizer defaults and the compute-signature defaults must agree — and + both must be the SAME constants the factor modules define (not a second + literal that can drift, the #76/#78 lesson). + """ + ridge = _defaults(summarize_ridge_coverage) + compute = _defaults(valley_ridge_module.compute_valley_ridge_vwap_ratio) + for key in ("min_ridge_bars", "min_valley_bars", "min_classifiable"): + assert ridge[key] == compute[key] + + ret = _defaults(summarize_ridge_return_coverage) + compute = _defaults(ridge_return_module.compute_ridge_minute_return) + for key in ("min_ridge_bars", "min_classifiable"): + assert ret[key] == compute[key] + + peak = _defaults(summarize_peak_coverage) + compute = _defaults(peak_module.compute_peak_ridge_amount_ratio) + for key in ("min_peak_bars", "min_ridge_bars", "min_classifiable"): + assert peak[key] == compute[key] + + # the counterfactual floors: PR-K's comparison floor is PR-J's valley floor + # (a documented cross-factor anchor, hardcoded as 20 with the reason on it); + # PR-M's is DERIVED from the ridge gate, never a hardcoded duplicate. + assert ret["comparison_floor"] == 20 == VALLEY_RIDGE_MIN_VALLEY_BARS + assert peak["counterfactual_peak_floor"] == PEAK_RIDGE_MIN_RIDGE_BARS + + +def test_disclosure_binding_covers_exactly_the_three_publishing_factors(): + for fid in ( + "valley_ridge_vwap_ratio_20", + "ridge_minute_return_20", + "peak_ridge_amount_ratio_20", + ): + binding = disclosure_binding_for(factor_registry.build(fid)) + assert binding is not None, fid + assert binding.section_name.endswith("_coverage") + # every other factor publishes NO per-day disclosure — stated, not inferred + assert disclosure_binding_for(factor_registry.build("jump_amount_corr_20")) is None + assert disclosure_binding_for(factor_registry.build("value_ep")) is None + + +# --------------------------------------------------------------------------- # +# to_section: the add-Section bridge (contract section 3.6) +# --------------------------------------------------------------------------- # +def _verdicted_report() -> FactorEvalReport: + spec = factor_registry.build("jump_amount_corr_20").spec + cfg = EvalConfig( + universe="000905.SH", + universe_is_pit=True, + start="2021-07-01", + end="2026-06-30", + is_exploratory=True, + post_hoc_selected=False, + oos_split="2024-01-01", + view="decision", + return_basis="exec_to_exec", + ) + sections = [Section(name=n, payload={}) for n in MANDATORY_SECTIONS] + return FactorEvalReport.assemble(spec, cfg, sections).with_verdict() + + +def test_to_section_packs_fields_properties_and_the_render_note(): + cov = summarize_ridge_coverage([]) + section = to_section("ridge_scarcity_coverage", cov) + assert section.name == "ridge_scarcity_coverage" + # dataclass fields land in the payload... + assert section.payload["symbol_days"] == 0 + assert section.payload["min_ridge_bars"] == VALLEY_RIDGE_MIN_RIDGE_BARS + # ...the derived property rides along... + assert "validity_rate" in section.payload + # ...and the note is the one-line render, so artifact and log cannot disagree + assert section.note == cov.render() + # a disclosure without that property must not invent it + peak = to_section("peak_scarcity_coverage", summarize_peak_coverage([])) + assert "return_guard_attrition" not in peak.payload + assert "validity_rate" in peak.payload + + +def test_extra_section_never_moves_the_verdict_or_mandatory_sections(): + base = _verdicted_report() + extra = to_section("ridge_scarcity_coverage", summarize_ridge_coverage([])) + augmented = FactorEvalReport.assemble( + base.spec, base.cfg, [*base.sections, extra], thresholds=base.thresholds + ).with_verdict() + # the mandatory sections are the ORIGINALS, in order... + assert augmented.sections[: len(base.sections)] == base.sections + # ...the verdict is bit-identical (it reads only the mandatory payloads)... + assert augmented.verdict == base.verdict + # ...and canonical order is mandatory first, extras sorted by name after + ordered = [name for name, _ in canonical_sections(augmented, MANDATORY_SECTIONS)] + assert ordered == [*MANDATORY_SECTIONS, "ridge_scarcity_coverage"] + + +def test_exec_basis_augmentation_seam_preserves_report_and_verdict(): + base = _verdicted_report() + extra = to_section("peak_scarcity_coverage", summarize_peak_coverage([])) + augmented = _with_extra_sections(base, [extra]) + assert augmented.sections[: len(base.sections)] == base.sections + assert augmented.verdict == base.verdict + assert augmented.by_name()["peak_scarcity_coverage"] is extra or ( + augmented.by_name()["peak_scarcity_coverage"] == extra + ) + + +def test_an_extra_section_may_never_shadow_a_mandatory_name(): + base = _verdicted_report() + bad = Section(name="caveats", payload={"smuggled": True}) + with pytest.raises(ValueError, match="duplicate report section"): + FactorEvalReport.assemble( + base.spec, base.cfg, [*base.sections, bad], thresholds=base.thresholds + ) diff --git a/tests/test_factor_eval_providers.py b/tests/test_factor_eval_providers.py new file mode 100644 index 0000000..f0a5d1d --- /dev/null +++ b/tests/test_factor_eval_providers.py @@ -0,0 +1,219 @@ +"""Real-cache eval provider wiring (D5 C4, commit 1). + +Network-free. Pins: the CacheMinuteProvider MOVE (single source — the hotpath +smoke and the probes import it from ``qt.factor_eval_providers`` now), the +provider's cache-only/zero-live-call behavior against a real on-disk intraday +store, the DailyEvalPanelProvider's close-view window/symbol slicing, and +``build_eval_service``'s call order + bundle contents (pipeline helpers +monkeypatched, so no tushare client is ever built). +""" + +from __future__ import annotations + +import logging +from types import SimpleNamespace + +import pandas as pd + +import qt.factor_eval_providers as fep +import qt.factor_hotpath_smoke as hotpath_smoke +import qt.panel_leg_probe as panel_leg_probe +import qt.saturation_probe as saturation_probe +from data.cache.intraday_cache import ENDPOINT as INTRADAY_ENDPOINT +from data.cache.intraday_parquet_store import KEY_COLS, IntradayParquetStore +from data.clean.intraday_schema import RAW_INTRADAY_FREQ +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL +from factors.materialize import MaterializeSources +from factors.store import FactorValueStore, StoreKey +from qt.factor_eval_providers import ( + CACHE_MINUTE_DATA_START, + DEFAULT_STORE_ROOT, + CacheMinuteProvider, + DailyEvalPanelProvider, + EvalServiceBundle, + build_eval_service, +) + + +# --------------------------------------------------------------------------- # +# the MOVE: one source for the cache minute provider +# --------------------------------------------------------------------------- # +def test_cache_minute_provider_has_a_single_source(): + assert hotpath_smoke.CacheMinuteProvider is fep.CacheMinuteProvider + assert saturation_probe.CacheMinuteProvider is fep.CacheMinuteProvider + assert panel_leg_probe.CacheMinuteProvider is fep.CacheMinuteProvider + assert saturation_probe.CACHE_MINUTE_DATA_START is fep.CACHE_MINUTE_DATA_START + + +def test_earliest_available_is_the_declared_floor(): + provider = CacheMinuteProvider("does/not/exist") + assert provider.earliest_available(["AAA", "BBB"]) == pd.Timestamp("2015-01-05") + assert CACHE_MINUTE_DATA_START == "2015-01-05" + + +# --------------------------------------------------------------------------- # +# CacheMinuteProvider against a REAL on-disk intraday store (cache-only) +# --------------------------------------------------------------------------- # +def _write_bars(root, symbol, rows): + """rows: list of (bar_end, close). Upsert raw 1min bars the store's way.""" + store = IntradayParquetStore(str(root)) + frame = pd.DataFrame( + { + "symbol": [symbol] * len(rows), + "bar_end": [pd.Timestamp(r[0]) for r in rows], + "source_trade_time": [pd.Timestamp(r[0]) for r in rows], + "open": [r[1] for r in rows], + "high": [r[1] for r in rows], + "low": [r[1] for r in rows], + "close": [r[1] for r in rows], + "volume": [100.0] * len(rows), + "amount": [1000.0] * len(rows), + "freq": [RAW_INTRADAY_FREQ] * len(rows), + } + ) + store.upsert(INTRADAY_ENDPOINT, symbol, RAW_INTRADAY_FREQ, frame, list(KEY_COLS)) + + +def test_minute_bars_reads_normalized_bars_with_zero_live_calls(tmp_path): + _write_bars( + tmp_path, + "000001.SZ", + [("2024-01-02 09:31:00", 11.0), ("2024-01-02 09:32:00", 12.0)], + ) + _write_bars(tmp_path, "000002.SZ", [("2024-01-02 09:31:00", 21.0)]) + provider = CacheMinuteProvider(str(tmp_path)) + bars = provider.minute_bars( + ["000001.SZ", "000002.SZ"], "2024-01-02 09:00:00", "2024-01-02 10:00:00" + ) + assert provider.calls == 1 + assert provider.live_calls == 0 # read_range has no fetch closure + assert len(bars) == 3 + assert bars.index.names == ["time", "symbol"] + times = bars.index.get_level_values("time") + assert (times == times.normalize()).sum() == 0 # minute precision kept + closes = bars["close"].astype(float).tolist() + assert closes == [11.0, 21.0, 12.0] # sorted by (time, symbol) + + +def test_minute_bars_empty_and_missing_are_empty_never_live(tmp_path): + provider = CacheMinuteProvider(str(tmp_path)) + empty = provider.minute_bars([], "2024-01-02", "2024-01-03") + assert empty.empty + missing = provider.minute_bars(["NOPE.SZ"], "2024-01-02", "2024-01-03") + assert missing.empty # an absent month partition is an empty read, not a fetch + assert provider.live_calls == 0 + + +# --------------------------------------------------------------------------- # +# DailyEvalPanelProvider: close-view, un-lagged, window/symbol slicing +# --------------------------------------------------------------------------- # +def _panel(): + dates = pd.bdate_range("2024-01-01", periods=6) + idx = pd.MultiIndex.from_product([dates, ["AAA", "BBB"]], names=[DATE_LEVEL, SYMBOL_LEVEL]) + return pd.DataFrame( + { + "close": [float(i) for i in range(len(idx))], + "open": [float(i) + 100.0 for i in range(len(idx))], + }, + index=idx, + ) + + +def test_daily_panel_slices_window_and_symbols_without_lagging(): + provider = DailyEvalPanelProvider(_panel()) + dates = pd.bdate_range("2024-01-01", periods=6) + out = provider.daily_panel(["AAA"], dates[2], dates[4]) + assert out.index.get_level_values(SYMBOL_LEVEL).unique().tolist() == ["AAA"] + got_dates = pd.DatetimeIndex(pd.unique(out.index.get_level_values(DATE_LEVEL))) + assert got_dates.equals(dates[2:5]) # both bounds inclusive + # close-view, NOT lagged: the value dated d is the panel's own close at d. + full = _panel() + for d in dates[2:5]: + assert out.loc[(d, "AAA"), "close"] == full.loc[(d, "AAA"), "close"] + + +def test_daily_panel_unknown_symbols_and_empty_panel(): + provider = DailyEvalPanelProvider(_panel()) + assert provider.daily_panel(["NOPE"], "2024-01-01", "2024-01-31").empty + empty_provider = DailyEvalPanelProvider(pd.DataFrame()) + assert empty_provider.daily_panel(["AAA"], "2024-01-01", "2024-01-31").empty + + +# --------------------------------------------------------------------------- # +# build_eval_service: legacy call order + bundle contents (helpers faked) +# --------------------------------------------------------------------------- # +def _fake_cfg(): + return SimpleNamespace(data=SimpleNamespace(cache=SimpleNamespace(root_dir="cache/root"))) + + +def test_build_eval_service_calls_pipeline_helpers_in_legacy_order(tmp_path, monkeypatch): + calls: list[str] = [] + sentinel_cache = object() + panel = _panel() + + def rec(name, ret=None): + def _fn(*args, **kwargs): + calls.append(name) + return ret + + return _fn + + monkeypatch.setattr(fep, "_build_cache", rec("_build_cache", sentinel_cache)) + monkeypatch.setattr(fep, "_build_universe", rec("_build_universe", (object(), ["AAA", "BBB"]))) + monkeypatch.setattr(fep, "_load_panel", rec("_load_panel", panel)) + monkeypatch.setattr(fep, "_maybe_enrich_value", rec("_maybe_enrich_value", panel)) + monkeypatch.setattr(fep, "_maybe_enrich_covariates", rec("_maybe_enrich_covariates", panel)) + monkeypatch.setattr(fep, "_log_run_cache_stats", rec("_log_run_cache_stats")) + + logger = logging.getLogger("test.build_eval_service") + bundle = build_eval_service( + _fake_cfg(), logger, value_factors=["vf1"], store_root=str(tmp_path / "store") + ) + + assert calls == [ + "_build_cache", + "_build_universe", + "_load_panel", + "_maybe_enrich_value", + "_maybe_enrich_covariates", + "_log_run_cache_stats", + ] + assert isinstance(bundle, EvalServiceBundle) + assert bundle.cache is sentinel_cache + assert bundle.symbols == ["AAA", "BBB"] + assert bundle.panel is panel + assert isinstance(bundle.store, FactorValueStore) + assert isinstance(bundle.sources, MaterializeSources) + assert isinstance(bundle.sources.daily, DailyEvalPanelProvider) + assert isinstance(bundle.sources.minute, CacheMinuteProvider) + # the minute provider reads the configured cache root; the store root is the + # caller's (design §3.4 R22 default is artifacts/factor_store). + key = StoreKey(factor_id="f", params_hash="p", code_hash="c", view="decision") + assert str(bundle.store.path(key)).startswith(str(tmp_path / "store")) + + +def test_build_eval_service_default_store_root_is_the_r22_artifacts_root(): + assert DEFAULT_STORE_ROOT == "artifacts/factor_store" + + +def test_build_eval_service_threads_value_factors_and_cache_root(monkeypatch): + seen: dict = {} + + def _enrich(cfg, panel, symbols, factors, logger, cache): + seen["value_factors"] = list(factors) + return panel + + monkeypatch.setattr(fep, "_build_cache", lambda cfg: None) + monkeypatch.setattr(fep, "_build_universe", lambda cfg, logger, cache: (None, ["AAA"])) + monkeypatch.setattr(fep, "_load_panel", lambda cfg, symbols, logger, cache: _panel()) + monkeypatch.setattr(fep, "_maybe_enrich_value", _enrich) + monkeypatch.setattr(fep, "_maybe_enrich_covariates", lambda cfg, panel, s, logger, cache: panel) + monkeypatch.setattr(fep, "_log_run_cache_stats", lambda cache, logger: None) + + bundle = build_eval_service( + _fake_cfg(), logging.getLogger("test.build_eval_service2"), value_factors=["v"] + ) + assert seen["value_factors"] == ["v"] + # cache disabled (None) still wires the cache-only minute provider. + assert bundle.cache is None + assert isinstance(bundle.sources.minute, CacheMinuteProvider) diff --git a/tests/test_factor_eval_reconcile.py b/tests/test_factor_eval_reconcile.py new file mode 100644 index 0000000..66b69d7 --- /dev/null +++ b/tests/test_factor_eval_reconcile.py @@ -0,0 +1,779 @@ +"""Unit tests for qt.factor_eval_reconcile (D5 C4, commit 3). + +All tests are network-free and cache-free: the classification rules are pure +functions driven by synthetic frames/dicts, and the hard gate is exercised +against stub baselines. The real-cache orchestration (run_*_mode) is covered +by the real three-factor reconciliation run, not here. +""" + +from __future__ import annotations + +import types +from pathlib import Path + +import pandas as pd +import pytest + +from qt.factor_eval_reconcile import ( + ReconciliationError, + check_new_pair_consistency, + classify_anchor_row, + classify_panel_differences, + diff_report_json, + diff_report_md, + frozen_panel_path, + require_baseline_verified, +) + +# --------------------------------------------------------------------------- # +# Hard gate +# --------------------------------------------------------------------------- # +def _stub_baseline(ok: int, problems: list[str], file_count: int = 77): + return types.SimpleNamespace( + verify_all=lambda: (ok, problems), file_count=file_count + ) + + +def test_hard_gate_passes_on_full_verification(): + require_baseline_verified(_stub_baseline(77, [])) + + +def test_hard_gate_missing_baseline_bytes_is_a_hard_error_not_a_skip(): + with pytest.raises(ReconciliationError, match="did NOT verify"): + require_baseline_verified(_stub_baseline(76, ["x.json: missing"])) + + +def test_hard_gate_raises_when_count_disagrees_without_problems(): + with pytest.raises(ReconciliationError): + require_baseline_verified(_stub_baseline(76, [])) + + +# --------------------------------------------------------------------------- # +# reports mode — JSON leaf diff +# --------------------------------------------------------------------------- # +def _frozen_like() -> dict: + return { + "schema_version": "0.1", + "criteria_source": "default", + "eval_config": {"universe": "000905.SH", "start": "2021-07-01"}, + "spec": {"factor_id": "volume_peak_count_20", "version": "1.0"}, + "verdict": {"verdict": "Watch"}, + "sections": [{"name": "ic", "payload": {"ic": 0.0177, "icir": 0.21}}], + } + + +def _with_registered_additions(new: dict) -> dict: + new["eval_config"].update( + {"view": "decision", "return_basis": "exec_to_exec", "book_view": None} + ) + new["eval_contract_version"] = "1.1" + new["spec"].update( + { + "requires": "['PanelField(...)']", + "adjustment": "returns_invariant", + "overnight_boundary": "uses_prior_close", + "lookback_depth": 20, + } + ) + return new + + +def test_json_registered_additions_only_passes(): + new = _with_registered_additions(_frozen_like()) + result = diff_report_json( + _frozen_like(), new, name="t", strict=True, correction_expected=False + ) + assert result.ok, result.diffs + assert {d.classification for d in result.diffs} == {"registered_addition"} + + +def test_json_unregistered_addition_fails(): + new = _with_registered_additions(_frozen_like()) + new["surprise"] = 1 + result = diff_report_json( + _frozen_like(), new, name="t", strict=True, correction_expected=False + ) + assert not result.ok + assert result.by_class("unregistered_addition")[0].path == "surprise" + + +def test_json_unregistered_removal_fails(): + new = _with_registered_additions(_frozen_like()) + del new["criteria_source"] + result = diff_report_json( + _frozen_like(), new, name="t", strict=True, correction_expected=False + ) + assert not result.ok + assert result.by_class("unregistered_removal")[0].path == "criteria_source" + + +def test_json_numeric_change_on_aggregate_path_is_the_warmup_class(): + # Aggregate metrics legitimately move: they aggregate a panel whose + # warmup cells moved (the panels leg is the value gate). Registered as + # warmup_aggregate_effect, REPORTED not gated. + new = _with_registered_additions(_frozen_like()) + new["sections"][0]["payload"]["ic"] = 0.0199 + result = diff_report_json( + _frozen_like(), new, name="t", strict=True, correction_expected=False + ) + assert result.ok, result.diffs + assert result.by_class("warmup_aggregate_effect")[0].path.endswith("ic") + + +def test_json_numeric_change_outside_aggregate_paths_fails(): + new = _with_registered_additions(_frozen_like()) + new["n_periods"] = 999 # top-level numeric: not an aggregate path + old = _frozen_like() + old["n_periods"] = 1205 + result = diff_report_json( + old, new, name="t", strict=True, correction_expected=False + ) + assert not result.ok + assert result.by_class("unregistered_change")[0].path == "n_periods" + + +def test_json_verdict_label_flip_carries_no_digit_and_fails(): + new = _with_registered_additions(_frozen_like()) + new["verdict"]["verdict"] = "Adopt" # pure label flip, no digits + result = diff_report_json( + _frozen_like(), new, name="t", strict=True, correction_expected=False + ) + assert not result.ok + assert result.by_class("unregistered_change")[0].path == "verdict.verdict" + + +def test_json_requires_list_leaves_are_registered_by_prefix(): + new = _with_registered_additions(_frozen_like()) + new["spec"]["requires"] = ["PanelField(field='high')", "PanelField(field='low')"] + result = diff_report_json( + _frozen_like(), new, name="t", strict=True, correction_expected=False + ) + assert result.ok, result.diffs + paths = {d.path for d in result.by_class("registered_addition")} + assert "spec.requires[0]" in paths and "spec.requires[1]" in paths + + +def test_json_sanity_report_stem_rename_is_registered(): + old = _frozen_like() + old["sections"][0]["payload"]["sanity_report"] = ( + "artifacts/reports/eval_x_exec_basis_sanity.md" + ) + new = _with_registered_additions(_frozen_like()) + new["sections"][0]["payload"]["sanity_report"] = ( + "artifacts/reports/factor_eval_x_20_exec_basis_sanity.md" + ) + result = diff_report_json( + old, new, name="t", strict=True, correction_expected=False + ) + assert result.ok, result.diffs + assert result.by_class("registered_sanity_stem_rename") + + +def test_json_exec_price_artifact_reused_false_to_true_is_registered(): + old = _frozen_like() + old["sections"][0]["payload"]["exec_price_artifact_reused"] = False + new = _with_registered_additions(_frozen_like()) + new["sections"][0]["payload"]["exec_price_artifact_reused"] = True + result = diff_report_json( + old, new, name="t", strict=True, correction_expected=False + ) + assert result.ok, result.diffs + assert result.by_class("registered_run_order_artifact") + # The REVERSE direction is not a run-order artifact: it must fail. + result_rev = diff_report_json( + new, old, name="t", strict=True, correction_expected=False + ) + assert not result_rev.ok + + +def test_json_jump_description_and_factor_version_are_correction_effects(): + old = _frozen_like() + old["spec"]["factor_id"] = "jump_amount_corr_20" + old["spec"]["description"] = "old (pre-cutoff) description" + old["sections"].append({"name": "provenance", "payload": {"factor_version": "1.0"}}) + new = _with_registered_additions(_frozen_like()) + new["spec"]["factor_id"] = "jump_amount_corr_20" + new["spec"]["description"] = "new description with the 14:50 truncation" + new["sections"].append({"name": "provenance", "payload": {"factor_version": "1.1"}}) + new["corrections"] = [{"defect": "...", "to_version": "1.1"}] + result = diff_report_json( + old, new, name="t", strict=True, correction_expected=True + ) + assert result.ok, result.diffs + changed = {d.path for d in result.by_class("registered_correction_effect")} + assert "spec.description" in changed + assert "sections[1].payload.factor_version" in changed + + +def test_json_numeric_within_tolerance_passes_and_records_max(): + new = _with_registered_additions(_frozen_like()) + new["sections"][0]["payload"]["ic"] = 0.0177 * (1 + 1e-12) + result = diff_report_json( + _frozen_like(), new, name="t", strict=True, correction_expected=False + ) + assert result.ok, result.diffs + assert result.max_numeric_rel_diff > 0.0 + + +def test_json_decision_book_numeric_drift_is_reported_not_gated(): + new = _with_registered_additions(_frozen_like()) + new["sections"][0]["payload"]["ic"] = 0.0201 # the decision-view book moved it + result = diff_report_json( + _frozen_like(), new, name="t", strict=False, correction_expected=False + ) + assert result.ok + assert result.by_class("book_view_effect") + + +def test_json_jump_correction_effect_accepted_only_with_corrections_block(): + old = _frozen_like() + old["spec"]["factor_id"] = "jump_amount_corr_20" + new = _with_registered_additions(_frozen_like()) + new["spec"]["factor_id"] = "jump_amount_corr_20" + new["spec"]["version"] = "1.1" + new["sections"][0]["payload"]["ic"] = -0.030539 # restated by the cutoff fix + with_block = dict(new) + with_block["corrections"] = [{"defect": "...", "to_version": "1.1"}] + result = diff_report_json( + old, with_block, name="t", strict=True, correction_expected=True + ) + assert result.ok, result.diffs + classes = {d.classification for d in result.diffs} + assert "registered_correction_effect" in classes + # WITHOUT the structured carrier the very same drift is unexplained -> fail. + result_no_block = diff_report_json( + old, new, name="t", strict=True, correction_expected=True + ) + assert not result_no_block.ok + + +def test_json_spec_version_change_is_unregistered_without_correction(): + new = _with_registered_additions(_frozen_like()) + new["spec"]["version"] = "1.1" + result = diff_report_json( + _frozen_like(), new, name="t", strict=True, correction_expected=False + ) + assert not result.ok + assert any( + d.path == "spec.version" and d.classification == "unregistered_change" + for d in result.diffs + ) + + +def test_new_pair_eval_config_may_differ_only_in_book_view(): + no_book = {"eval_config": {"universe": "X", "book_view": None}} + with_book = {"eval_config": {"universe": "X", "book_view": "decision"}} + assert check_new_pair_consistency(no_book, with_book) == [] + with_book["eval_config"]["oos_split"] = "2025-01-01" + problems = check_new_pair_consistency(no_book, with_book) + assert len(problems) == 1 + assert problems[0].path == "eval_config.oos_split" + + +# --------------------------------------------------------------------------- # +# reports mode — Markdown line diff +# --------------------------------------------------------------------------- # +_MD_OLD = "# Factor Evaluation — x (v1.0)\n\n## 0. Header & Provenance\n\n- factor_id: x\n- family: microstructure\n" + + +def test_md_registered_provenance_lines_pass(): + new = _MD_OLD + "- evaluation contract: v1.1, ...\n- requires (endpoint inputs): a.b\n" + result = diff_report_md(_MD_OLD, new, name="t", correction_expected=False) + assert result.ok, result.diffs + + +def test_md_unregistered_new_line_fails(): + new = _MD_OLD + "- verdict secretly changed: Adopt\n" + result = diff_report_md(_MD_OLD, new, name="t", correction_expected=False) + assert not result.ok + assert result.by_class("unregistered_addition") + + +def test_md_old_line_vanishing_fails(): + new = _MD_OLD.replace("- family: microstructure\n", "") + result = diff_report_md(_MD_OLD, new, name="t", correction_expected=False) + assert not result.ok + assert result.by_class("unregistered_removal") + + +def test_md_same_key_value_change_pairs_into_a_change_not_a_removal(): + old = _MD_OLD + "- ic_mean: 0.018587\n- settled_rebalances: 1190\n" + new = _MD_OLD + "- ic_mean: 0.018475\n- settled_rebalances: 1209\n" + result = diff_report_md(old, new, name="t", correction_expected=False) + assert result.ok, result.diffs + # No phantom removal/addition pairs: both rows are single CHANGES. + assert len(result.diffs) == 2 + assert {d.classification for d in result.diffs} == {"warmup_aggregate_effect"} + assert all(d.old is not None and d.new is not None for d in result.diffs) + + +def test_md_paired_label_flip_without_digits_fails(): + old = _MD_OLD + "- verdict: Watch\n" + new = _MD_OLD + "- verdict: Adopt\n" + result = diff_report_md(old, new, name="t", correction_expected=False) + assert not result.ok + assert result.by_class("unregistered_change") + + +def test_md_paired_change_in_decision_book_is_book_view_effect(): + old = _MD_OLD + "- incremental_ic_ir: 0.120357\n" + new = _MD_OLD + "- incremental_ic_ir: 0.131000\n" + result = diff_report_md(old, new, name="t", strict=False, correction_expected=False) + assert result.ok + assert result.by_class("book_view_effect") + + +def test_md_correction_expected_turns_paired_changes_into_correction_effects(): + old = _MD_OLD + "- ic_mean: -0.030840\n" + new = _MD_OLD + "- ic_mean: -0.030539\n" + result = diff_report_md(old, new, name="t", correction_expected=True) + assert result.ok + assert result.by_class("registered_correction_effect") + + +def test_md_sanity_report_line_rename_is_registered(): + old = _MD_OLD + "- sanity_report: artifacts/reports/eval_x_exec_basis_sanity.md\n" + new = _MD_OLD + "- sanity_report: artifacts/reports/factor_eval_x_20_exec_basis_sanity.md\n" + result = diff_report_md(old, new, name="t", correction_expected=False) + assert result.ok, result.diffs + assert result.by_class("registered_sanity_stem_rename") + + +def test_md_prose_line_with_number_before_the_colon_pairs_via_normalized_key(): + # The changed number PRECEDES the first colon: the exact head can never + # pair these. The digit-normalized fallback pairs them (real instance: + # the with_book incremental-FAIL bullet in minute_ideal_amp/volume_peak). + old = ( + _MD_OLD + + "- [incremental FAIL] orthogonalized ICIR +0.120 is ~ 0 (redundant with the book):" + " after residualizing on the known-factor book the factor adds no signal.\n" + ) + new = ( + _MD_OLD + + "- [incremental FAIL] orthogonalized ICIR +0.113 is ~ 0 (redundant with the book):" + " after residualizing on the known-factor book the factor adds no signal.\n" + ) + result = diff_report_md(old, new, name="t", correction_expected=False) + assert result.ok, result.diffs + assert len(result.diffs) == 1 + assert result.diffs[0].classification == "warmup_aggregate_effect" + assert result.diffs[0].old is not None and result.diffs[0].new is not None + + +def test_md_long_prose_head_within_cap_still_pairs(): + # The incremental-axis reason line is ~103 chars before its first colon — + # it must remain PAIRABLE (the cap exists only for pathological lines). + old = ( + _MD_OLD + + "- [incremental PASS] orthogonalized ICIR lower CI bound (N_eff-based," + " expected direction) +0.198 > 0.15: the factor convincingly adds a signal.\n" + ) + new = ( + _MD_OLD + + "- [incremental PASS] orthogonalized ICIR lower CI bound (N_eff-based," + " expected direction) +0.195 > 0.15: the factor convincingly adds a signal.\n" + ) + result = diff_report_md(old, new, name="t", correction_expected=False) + assert result.ok, result.diffs + assert len(result.diffs) == 1 + assert result.diffs[0].classification == "warmup_aggregate_effect" + + +def test_md_prose_WORD_change_does_not_pair_and_fails(): + old = ( + _MD_OLD + + "- [incremental FAIL] orthogonalized ICIR +0.120 is ~ 0 (redundant with the book):" + " after residualizing the factor adds no signal.\n" + ) + new = ( + _MD_OLD + + "- [incremental FAIL] orthogonalized ICIR +0.113 is clearly above (redundant with the book):" + " after residualizing the factor adds no signal.\n" + ) + result = diff_report_md(old, new, name="t", correction_expected=False) + assert not result.ok + assert result.by_class("unregistered_addition") + assert result.by_class("unregistered_removal") + + +# --------------------------------------------------------------------------- # +# panels mode — cell classification +# --------------------------------------------------------------------------- # +def _frozen_panel(rows: list[tuple[str, str, float]], fid: str) -> pd.DataFrame: + return pd.DataFrame(rows, columns=["date", "symbol", fid]) + + +def _new_series(rows: list[tuple[str, str, float]]) -> pd.Series: + idx = pd.MultiIndex.from_tuples( + [(pd.Timestamp(d), s) for d, s, _ in rows], names=["date", "symbol"] + ) + return pd.Series([v for _, _, v in rows], index=idx) + + +NAN = float("nan") + + +def test_panels_identical_is_clean(): + frozen = _frozen_panel( + [("2021-07-01", "A", NAN), ("2021-07-02", "A", 1.5), ("2021-07-02", "B", 2.0)], + "f", + ) + new = _new_series( + [("2021-07-01", "A", NAN), ("2021-07-02", "A", 1.5), ("2021-07-02", "B", 2.0)] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=2 + ) + assert result.ok, result.diffs + assert result.equal == 1 and result.within_tolerance == 2 + + +def test_panels_bounded_warmup_class_on_first_lookback_minus_1_rows(): + # lookback_depth=3 -> the first w-1 = 2 trading dates are the warmup + # boundary. A NaN -> finite cell on the 2nd date is the left-extension + # warmup effect (the old runner's partial pool fills in). + frozen = _frozen_panel( + [ + ("2021-07-01", "SPARSE", NAN), ("2021-07-02", "SPARSE", NAN), + ("2021-07-03", "SPARSE", 0.5), + ], + "f", + ) + new = _new_series( + [ + ("2021-07-01", "SPARSE", NAN), ("2021-07-02", "SPARSE", 0.7), + ("2021-07-03", "SPARSE", 0.5), + ] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=3 + ) + assert result.ok, result.diffs + warmup = result.by_class("warmup_left_extension") + assert len(warmup) == 1 + assert result.warmup_by_direction == {"nan_to_finite": 1} + + +def test_panels_bounded_warmup_finite_to_finite_inside_boundary(): + # Partial pool -> full pool: both sides finite, inside the first w-1 + # dates. Registered (this is the 4554/17109-cell form from the first run). + frozen = _frozen_panel( + [("2021-07-01", "A", 0.10), ("2021-07-02", "A", 0.20), ("2021-07-03", "A", 0.30)], + "f", + ) + new = _new_series( + [("2021-07-01", "A", 0.11), ("2021-07-02", "A", 0.25), ("2021-07-03", "A", 0.30)] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=3 + ) + assert result.ok, result.diffs + assert result.warmup_by_direction == {"finite_to_finite": 2} + + +def test_panels_bounded_finite_to_finite_OUTSIDE_warmup_boundary_fails(): + # REVERSE direction: the very same finite->finite change one date later + # is a real regression and must FAIL. + frozen = _frozen_panel( + [("2021-07-01", "A", 0.10), ("2021-07-02", "A", 0.20), ("2021-07-03", "A", 0.30)], + "f", + ) + new = _new_series( + [("2021-07-01", "A", 0.10), ("2021-07-02", "A", 0.20), ("2021-07-03", "A", 0.35)] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=3 + ) + assert not result.ok + assert len(result.by_class("unclassified_finite_vs_finite")) == 1 + + +def test_panels_bounded_nan_to_finite_outside_first_rows_is_unclassified(): + frozen = _frozen_panel( + [("2021-07-01", "A", NAN), ("2021-07-02", "A", NAN), ("2021-07-03", "A", NAN)], + "f", + ) + new = _new_series( + [("2021-07-01", "A", NAN), ("2021-07-02", "A", NAN), ("2021-07-03", "A", 0.7)] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=2 + ) + assert not result.ok + assert len(result.by_class("unclassified_nan_to_finite")) == 1 + + +def test_panels_bounded_new_only_finite_row_inside_warmup_is_registered(): + # B's frozen grid skips 2021-07-02 (a grid gap, as in the real volume_peak + # new-only rows); the served panel fills it. 07-02 sits inside the grid's + # first w-1 = 2 trading dates -> registered warmup, not an unregistered row. + frozen = _frozen_panel( + [ + ("2021-07-01", "A", 1.0), ("2021-07-02", "A", 1.0), ("2021-07-03", "A", 1.0), + ("2021-07-01", "B", 0.5), ("2021-07-03", "B", 0.6), + ], + "f", + ) + new = _new_series( + [ + ("2021-07-01", "A", 1.0), ("2021-07-02", "A", 1.0), ("2021-07-03", "A", 1.0), + ("2021-07-01", "B", 0.5), ("2021-07-02", "B", 0.55), ("2021-07-03", "B", 0.6), + ] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=3 + ) + assert result.ok, result.diffs + assert result.warmup_by_direction == {"new_only_finite": 1} + + +def test_panels_bounded_new_only_finite_row_outside_warmup_fails(): + frozen = _frozen_panel([("2021-07-01", "A", 0.5)], "f") + new = _new_series([("2021-07-01", "A", 0.5), ("2021-09-01", "A", 0.4)]) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=3 + ) + assert not result.ok + assert len(result.by_class("unregistered_new_finite_row")) == 1 + + +def test_panels_pooled_warmup_class_all_directions_with_monthly_decrease(): + frozen = _frozen_panel( + [ + ("2021-07-01", "A", NAN), ("2021-07-02", "A", 1.0), + ("2021-08-02", "A", NAN), + ], + "f", + ) + new = _new_series( + [ + ("2021-07-01", "A", 1.0), ("2021-07-02", "A", 1.5), # nan->finite + finite->finite + ("2021-08-02", "A", 3.0), ("2021-07-05", "A", 2.0), # new-only finite row + ] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=True, lookback_depth=20 + ) + assert result.ok, result.diffs + assert len(result.by_class("warmup_left_extension")) == 4 + assert result.warmup_by_direction == { + "nan_to_finite": 2, "finite_to_finite": 1, "new_only_finite": 1 + } + assert result.warmup_by_month == {"2021-07": 3, "2021-08": 1} + assert result.warmup_monotonic + + +def test_panels_pooled_warmup_after_early_region_is_unclassified(): + frozen = _frozen_panel([("2022-01-04", "A", NAN)], "f") + new = _new_series([("2022-01-04", "A", 1.0)]) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=True, lookback_depth=20 + ) + assert not result.ok + assert len(result.by_class("unclassified_nan_to_finite")) == 1 + + +def test_panels_pooled_warmup_non_monotonic_monthly_counts_fail(): + frozen = _frozen_panel( + [("2021-07-01", "A", NAN), ("2021-08-02", "A", NAN), ("2021-08-03", "A", NAN)], + "f", + ) + new = _new_series( + [("2021-07-01", "A", 1.0), ("2021-08-02", "A", 2.0), ("2021-08-03", "A", 3.0)] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=True, lookback_depth=20 + ) + assert not result.ok # 2021-08 (2) > 2021-07 (1) violates 按月递减至零 + assert not result.warmup_monotonic + + +def test_panels_float_reordering_tail_within_bounds_passes(): + # Scattered cells at rel 3e-12 (the measured jump tail: 1.0e-12..2.9e-12). + # lookback_depth=1 disables the bounded warmup boundary so the tail + # classes are what is being exercised. + frozen = _frozen_panel( + [("2022-11-03", "A", 1.0), ("2023-01-10", "B", 2.0)], "f" + ) + new = _new_series( + [("2022-11-03", "A", 1.0 * (1 + 3e-12)), ("2023-01-10", "B", 2.0 * (1 - 3e-12))] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=1 + ) + assert result.ok, result.diffs + assert len(result.by_class("float_reordering_tail")) == 2 + + +def test_panels_float_tail_beyond_rel_bound_fails(): + frozen = _frozen_panel([("2022-11-03", "A", 1.0)], "f") + new = _new_series([("2022-11-03", "A", 1.0 * (1 + 1e-11))]) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=1 + ) + assert not result.ok + assert len(result.by_class("unclassified_finite_vs_finite")) == 1 + + +def test_panels_float_tail_beyond_cell_cap_fails(): + n = 102 # one above the registered cap (101) + frozen = _frozen_panel([(f"2022-11-{(i % 28) + 1:02d}", f"S{i}", 1.0) for i in range(n)], "f") + new = _new_series( + [(f"2022-11-{(i % 28) + 1:02d}", f"S{i}", 1.0 * (1 + 3e-12)) for i in range(n)] + ) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=1 + ) + assert not result.ok + assert len(result.by_class("float_reordering_tail")) == n + + +def test_panels_threshold_flip_tail_within_bounds_passes(): + # Count flips by EXACTLY +/-1 at small rel (the measured volume_peak + # cluster: 600623.SH, 20 consecutive emits, sigma noise x integer volume). + rows = [(f"2023-06-{d:02d}", "600623.SH", 150.0) for d in (14, 15, 16)] + frozen = _frozen_panel(rows, "f") + new = _new_series([(d, s, v - 1.0) for d, s, v in rows]) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=True, lookback_depth=20 + ) + assert result.ok, result.diffs + assert len(result.by_class("threshold_flip_tail")) == 3 + + +def test_panels_threshold_flip_amplitude_above_one_fails(): + frozen = _frozen_panel([("2023-06-15", "600623.SH", 150.0)], "f") + new = _new_series([("2023-06-15", "600623.SH", 148.0)]) # delta = 2 + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=True, lookback_depth=20 + ) + assert not result.ok + assert len(result.by_class("unclassified_finite_vs_finite")) == 1 + + +def test_panels_threshold_flip_beyond_cell_cap_fails(): + n = 26 # one above the registered cap (25) + frozen = _frozen_panel([(f"2023-06-{(i % 28) + 1:02d}", f"S{i}", 150.0) for i in range(n)], "f") + new = _new_series([(f"2023-06-{(i % 28) + 1:02d}", f"S{i}", 149.0) for i in range(n)]) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=True, lookback_depth=20 + ) + assert not result.ok + assert len(result.by_class("threshold_flip_tail")) == n + + +def test_panels_frozen_finite_new_nan_never_allowed(): + frozen = _frozen_panel([("2021-07-01", "A", 1.0)], "f") + new = _new_series([("2021-07-01", "A", NAN)]) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=2 + ) + assert not result.ok + assert len(result.by_class("unclassified_frozen_finite_new_nan")) == 1 + + +def test_panels_finite_vs_finite_beyond_tolerance_never_allowed(): + # lookback_depth=1 -> EMPTY bounded warmup boundary, so no warmup excuse. + frozen = _frozen_panel([("2021-07-01", "A", 1.0)], "f") + new = _new_series([("2021-07-01", "A", 1.0 + 1e-6)]) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=1 + ) + assert not result.ok + assert len(result.by_class("unclassified_finite_vs_finite")) == 1 + assert result.max_rel_diff > 0 + + +def test_panels_extra_rows_allowed_only_as_nan_footprint(): + frozen = _frozen_panel([("2021-07-01", "A", 1.0)], "f") + new = _new_series([("2021-07-01", "A", 1.0), ("2021-07-01", "NEWSYM", NAN)]) + result = classify_panel_differences( + new, frozen, factor_id="f", is_pooled=False, lookback_depth=2 + ) + assert result.ok, result.diffs + assert result.nan_footprint_rows == 1 + new_bad = _new_series([("2021-07-01", "A", 1.0), ("2021-07-01", "NEWSYM", 3.0)]) + result_bad = classify_panel_differences( + new_bad, frozen, factor_id="f", is_pooled=False, lookback_depth=2 + ) + assert not result_bad.ok + assert len(result_bad.by_class("unregistered_new_finite_row")) == 1 + + +def test_frozen_panel_path_routes_jump_to_the_cutoff_reference(): + root = Path("/repo") + jump = frozen_panel_path("jump_amount_corr_20", root) + assert jump.parent.name == "panels" and jump.parent.parent.name == "pr_c_cutoff_fix" + other = frozen_panel_path("volume_peak_count_20", root) + assert other.parent.parent.name == "refactor_baseline" and "pr_c_cutoff_fix" not in str(other) + + +# --------------------------------------------------------------------------- # +# anchors mode — row classification +# --------------------------------------------------------------------------- # +def test_anchor_jump_mismatch_fails_even_in_early_region(): + row = classify_anchor_row( + factor_id="jump_amount_corr_20", cls="warmup_end", + date=pd.Timestamp("2021-07-02"), symbol="000537.SZ", + hand=0.46, service=0.47, is_pooled=False, tol=1e-12, + ) + assert row.classification == "failed" + + +def test_anchor_jump_match_is_ok(): + row = classify_anchor_row( + factor_id="jump_amount_corr_20", cls="random", + date=pd.Timestamp("2024-05-30"), symbol="002690.SZ", + hand=0.3831735019222186, service=0.3831735019222186, is_pooled=False, tol=1e-12, + ) + assert row.classification == "ok" and row.rel_diff == 0.0 + + +def test_anchor_pooled_early_mismatch_is_the_warmup_class(): + row = classify_anchor_row( + factor_id="volume_peak_count_20", cls="warmup_end", + date=pd.Timestamp("2021-07-28"), symbol="600827.SH", + hand=150.0, service=151.0, is_pooled=True, tol=1e-12, + ) + assert row.classification == "warmup_left_extension" + + +def test_anchor_pooled_late_mismatch_fails(): + row = classify_anchor_row( + factor_id="volume_peak_count_20", cls="random", + date=pd.Timestamp("2023-05-18"), symbol="600867.SH", + hand=1.0, service=1.1, is_pooled=True, tol=1e-12, + ) + assert row.classification == "failed" + + +def test_anchor_bounded_mismatch_inside_warmup_dates_is_the_warmup_class(): + warmup = frozenset(pd.Timestamp(d) for d in ("2021-07-01", "2021-07-02", "2021-07-05")) + row = classify_anchor_row( + factor_id="minute_ideal_amp_10", cls="warmup_end", + date=pd.Timestamp("2021-07-02"), symbol="000537.SZ", + hand=0.46, service=0.47, is_pooled=False, tol=1e-12, warmup_dates=warmup, + ) + assert row.classification == "warmup_left_extension" + + +def test_anchor_bounded_mismatch_outside_warmup_dates_fails(): + warmup = frozenset(pd.Timestamp(d) for d in ("2021-07-01", "2021-07-02", "2021-07-05")) + row = classify_anchor_row( + factor_id="minute_ideal_amp_10", cls="random", + date=pd.Timestamp("2021-07-06"), symbol="000537.SZ", + hand=0.46, service=0.47, is_pooled=False, tol=1e-12, warmup_dates=warmup, + ) + assert row.classification == "failed" + + +def test_anchor_jump_mismatch_inside_warmup_is_warmup_not_definition_failure(): + # The truncation-carried signal lives on jump's NON-warmup rows; a warmup + # row differs by loading geometry (hand side anchored at 2021-07-01). + warmup = frozenset(pd.Timestamp(d) for d in ("2021-07-01", "2021-07-02")) + row = classify_anchor_row( + factor_id="jump_amount_corr_20", cls="warmup_end", + date=pd.Timestamp("2021-07-02"), symbol="000537.SZ", + hand=0.46, service=0.35, is_pooled=False, tol=1e-12, warmup_dates=warmup, + ) + assert row.classification == "warmup_left_extension" diff --git a/tests/test_factor_eval_runner.py b/tests/test_factor_eval_runner.py new file mode 100644 index 0000000..c14a9e6 --- /dev/null +++ b/tests/test_factor_eval_runner.py @@ -0,0 +1,424 @@ +"""The unified exec-only factor-eval runner (D5 C4) — network-free. + +The heavy seams are faked: ``build_eval_service`` returns a bundle wired to +synthetic providers + a tmp value store, and ``run_exec_basis_evaluation`` is a +stub that captures its kwargs (the exec tail is qt.exec_basis_eval's own tested +code). What THESE tests pin is the runner's own logic: the config gates +(catalogue C1/C2 collapse), the BUG 5 config-book closure, the exec identity +on the EvalConfig, the two book modes' ``book_view`` derivation, the +add-Section passthrough, the artifact stem isolation (incl. the ``_bookclose`` +suffix), and the readable refusal of the deferred ``valley_price_quantile``. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from analytics.eval.sections import Section +from data.clean.intraday_schema import normalize_intraday_bars +from factors.materialize import MaterializeSources +from factors.store import FactorValueStore +from qt.config import ( + AlphaCfg, + BacktestCfg, + CacheCfg, + CostCfg, + DataCfg, + FactorCfg, + NeutralizeCfg, + OOSCfg, + OutputCfg, + PortfolioCfg, + ProcessingCfg, + RootConfig, + UniverseCfg, +) +from qt.exec_basis_eval import ExecBasisEvaluation +from qt.factor_eval_disclosures import RidgeCoverage +from qt.factor_eval_providers import EvalServiceBundle +from qt.factor_eval_runner import ( + _build_eval_config, + _check_config_book, + run_factor_eval, +) + +SYMS = ["000001.SZ", "000002.SZ"] +DATES = pd.bdate_range("2021-07-01", periods=30) + +_METRICS = {"deployment": "Watch", "predictive": "PASS", "incremental": "NOT_ASSESSED"} + + +def _min_config(tmp_path, **overrides) -> RootConfig: + kwargs = dict( + data=DataCfg( + source="tushare", + start="2021-07-01", + end="2021-08-11", + external_secret_file="/nonexistent.json", + cache=CacheCfg(enabled=True, root_dir=str(tmp_path / "cache")), + ), + universe=UniverseCfg(type="index", index_code="000905.SH", symbols=[]), + factors=[], + processing=ProcessingCfg(neutralize=NeutralizeCfg(enabled=True)), + alpha=AlphaCfg(), + portfolio=PortfolioCfg(top_n=1), + backtest=BacktestCfg(), + cost=CostCfg(), + output=OutputCfg( + root_dir=str(tmp_path / "artifacts"), + data_dir=str(tmp_path / "artifacts" / "data"), + factor_dir=str(tmp_path / "artifacts" / "factors"), + report_dir=str(tmp_path / "artifacts" / "reports"), + log_dir=str(tmp_path / "artifacts" / "logs"), + ), + oos=OOSCfg(split_date="2021-07-20"), + ) + kwargs.update(overrides) + return RootConfig(**kwargs) + + +# --------------------------------------------------------------------------- # +# Config gates (the collapsed C1/C2 bodies + BUG 5 closure) +# --------------------------------------------------------------------------- # +def test_missing_oos_is_a_readable_error(tmp_path): + cfg = _min_config(tmp_path, oos=None) + with pytest.raises(ValueError, match="requires an 'oos' section"): + _build_eval_config(cfg) + + +def _cfg_demo_source(tmp_path) -> RootConfig: + return _min_config( + tmp_path, + data=DataCfg( + source="demo", + start="2021-07-01", + end="2021-08-11", + cache=CacheCfg(enabled=True, root_dir=str(tmp_path / "cache")), + ), + ) + + +def _cfg_cache_disabled(tmp_path) -> RootConfig: + return _min_config( + tmp_path, + data=DataCfg( + source="tushare", + start="2021-07-01", + end="2021-08-11", + external_secret_file="/nonexistent.json", + cache=CacheCfg(enabled=False, root_dir=str(tmp_path / "cache")), + ), + ) + + +def _cfg_static_universe(tmp_path) -> RootConfig: + return _min_config(tmp_path, universe=UniverseCfg(type="static", symbols=["A.SZ"])) + + +def _cfg_neutralize_disabled(tmp_path) -> RootConfig: + return _min_config( + tmp_path, + processing=ProcessingCfg(neutralize=NeutralizeCfg(enabled=False)), + ) + + +@pytest.mark.parametrize( + "cfg_factory, match", + [ + (_cfg_demo_source, "data.source='tushare'"), + (_cfg_cache_disabled, "data.cache.enabled=true"), + (_cfg_static_universe, "universe.type='index'"), + (_cfg_neutralize_disabled, "neutralize.enabled=true"), + ], +) +def test_preconditions_fail_readably(tmp_path, monkeypatch, cfg_factory, match): + monkeypatch.setattr( + "qt.factor_eval_runner.load_config", lambda path: cfg_factory(tmp_path) + ) + monkeypatch.setattr( + "qt.factor_eval_runner.build_eval_service", + lambda *a, **k: pytest.fail("service must not be built on a failed gate"), + ) + with pytest.raises(ValueError, match=match): + run_factor_eval("ignored.yaml", "jump_amount_corr_20") + + +def test_config_book_empty_or_exact_is_accepted(tmp_path): + _check_config_book(_min_config(tmp_path)) # factors: [] — the honest declaration + exact = _min_config( + tmp_path, + factors=[ + FactorCfg(name="value_ep"), + FactorCfg(name="value_bp"), + FactorCfg(name="volatility_20", params={"window": 20, "price_col": "close"}), + ], + ) + _check_config_book(exact) # the legacy eval configs' declaration still passes + + +def test_config_book_mismatch_is_a_readable_error(tmp_path): + cfg = _min_config(tmp_path, factors=[FactorCfg(name="momentum_20")]) + with pytest.raises(ValueError, match="does not read config 'factors:'"): + _check_config_book(cfg) + wrong_params = _min_config( + tmp_path, + factors=[ + FactorCfg(name="value_ep"), + FactorCfg(name="value_bp"), + FactorCfg(name="volatility_20", params={"window": 10}), + ], + ) + with pytest.raises(ValueError, match="does not read config 'factors:'"): + _check_config_book(wrong_params) + + +def test_build_eval_config_declares_the_exec_identity_and_shared_kwargs(tmp_path): + eval_cfg = _build_eval_config(_min_config(tmp_path)) + # exec-only: the identity is explicit (the EvalConfig default is the close pairing) + assert eval_cfg.view == "decision" + assert eval_cfg.return_basis == "exec_to_exec" + assert eval_cfg.book_view is None # per-run: no-book None / with-book book_mode + # the catalogue C1 shared kwargs, character-for-character with the legacy bodies + assert eval_cfg.universe == "000905.SH" + assert eval_cfg.universe_is_pit is True + assert (eval_cfg.start, eval_cfg.end) == ("2021-07-01", "2021-08-11") + assert eval_cfg.is_exploratory is True + assert eval_cfg.post_hoc_selected is False + assert eval_cfg.rebalance == "daily" + assert eval_cfg.n_quantiles == 5 + assert eval_cfg.cost_scenarios == (1.0, 2.0, 4.0) + assert eval_cfg.oos_split == "2021-07-20" + assert eval_cfg.winsorize is None + assert eval_cfg.standardize == "zscore" + assert eval_cfg.neutralization == ("industry", "size") + assert eval_cfg.industry_level == "L1" + assert eval_cfg.tuned is False + assert eval_cfg.n_factors_screened == 1 + + +def test_invalid_book_mode_is_rejected_before_any_work(tmp_path): + with pytest.raises(ValueError, match="--book-mode"): + run_factor_eval("ignored.yaml", "jump_amount_corr_20", book_mode="bogus") + + +def test_valley_price_quantile_is_a_readable_deferral(tmp_path, monkeypatch): + monkeypatch.setattr( + "qt.factor_eval_runner.load_config", lambda path: _min_config(tmp_path) + ) + monkeypatch.setattr( + "qt.factor_eval_runner.build_eval_service", + lambda *a, **k: pytest.fail("service must not be built for a deferred factor"), + ) + with pytest.raises(ValueError, match="PR-C4b"): + run_factor_eval("ignored.yaml", "valley_price_quantile_20") + + +# --------------------------------------------------------------------------- # +# End-to-end wiring with fake providers + a stubbed exec tail +# --------------------------------------------------------------------------- # +def _daily_panel() -> pd.DataFrame: + # The bundle's panel is the ENRICHED close-view panel: value_ep / value_bp + # columns are what _maybe_enrich_value would have added from daily_basic + # pe/pb, and industry / market_cap are the neutralization covariates. + rows = [] + for si, s in enumerate(SYMS): + px = 100.0 + si * 20 + np.cumsum( + np.random.RandomState(si).normal(0, 1.0, len(DATES)) + ) + for d, p in zip(DATES, px): + rows.append( + (d, s, p - 0.3, p + 0.5, p - 0.5, p, 1e5, p * 1e5, + 0.05 + 0.01 * si, 0.5 + 0.1 * si, 1e9 * (si + 1), f"industry_{si}") + ) + return ( + pd.DataFrame( + rows, + columns=[ + "date", "symbol", "open", "high", "low", "close", "volume", + "amount", "value_ep", "value_bp", "market_cap", "industry", + ], + ) + .set_index(["date", "symbol"]) + .sort_index() + ) + + +def _minute_bars() -> pd.DataFrame: + rng = np.random.RandomState(7) + rows = [] + for si, s in enumerate(SYMS): + for d in DATES: + base = pd.Timestamp(d) + pd.Timedelta("09:31:00") + price = 100.0 + si * 5 + rng.normal(0, 2) + for i in range(100): + t = base + pd.Timedelta(minutes=i) + price += rng.normal(0, 0.05) + vol = 1e4 * (1.0 + rng.rand()) + rows.append((t, s, price, price + 0.1, price - 0.1, price, vol, price * vol)) + frame = pd.DataFrame( + rows, + columns=["time", "symbol", "open", "high", "low", "close", "volume", "amount"], + ) + return normalize_intraday_bars(frame, freq="1min") + + +class _DailyProv: + def __init__(self, panel): + self._panel = panel + + def daily_panel(self, symbols, start, end): + m = self._panel.index.get_level_values("date") + return self._panel[(m >= pd.Timestamp(start)) & (m <= pd.Timestamp(end))] + + +class _MinuteProv: + live_calls = 0 + + def __init__(self, bars): + self._bars = bars + + def minute_bars(self, symbols, start, end): + t = self._bars.index.get_level_values("time") + keep = self._bars[(t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end))] + syms = keep.index.get_level_values("symbol") + return keep[syms.isin(list(symbols))] + + def earliest_available(self, symbols): + return DATES[0] + + +def _stub_exec_basis(report_dir, stem): + report_dir.mkdir(parents=True, exist_ok=True) + paths = {} + for kind in ("no_book", "with_book"): + for suffix, key in ((".md", "md"), (".json", "json"), ("_dashboard.png", "dashboard")): + p = report_dir / f"{stem}_exec_{kind}{suffix}" + p.write_text("stub", encoding="utf-8") + paths[f"{kind}_{key}"] = p + return ExecBasisEvaluation( + spec=None, params=None, artifact_path=report_dir / "a.parquet", + artifact_key="k", artifact_reused=False, minute_live_calls=0, + coverage={}, sanity=None, sanity_report_path=report_dir / "sanity.md", + no_book=None, with_book=None, + no_book_md=paths["no_book_md"], no_book_json=paths["no_book_json"], + with_book_md=paths["with_book_md"], with_book_json=paths["with_book_json"], + no_book_dashboard=paths["no_book_dashboard"], + with_book_dashboard=paths["with_book_dashboard"], + no_book_metrics=dict(_METRICS), with_book_metrics=dict(_METRICS), elapsed=0.0, + ) + + +def _wire(monkeypatch, tmp_path, captured): + cfg = _min_config(tmp_path) + monkeypatch.setattr("qt.factor_eval_runner.load_config", lambda path: cfg) + daily = _daily_panel() + bundle = EvalServiceBundle( + store=FactorValueStore(str(tmp_path / "store")), + sources=MaterializeSources( + daily=_DailyProv(daily), minute=_MinuteProv(_minute_bars()) + ), + panel=daily, + symbols=list(SYMS), + cache=None, + ) + monkeypatch.setattr( + "qt.factor_eval_runner.build_eval_service", lambda *a, **k: bundle + ) + + def fake_exec_eval(factor_panel, spec, eval_cfg, book, **kwargs): + captured.update( + factor_panel=factor_panel, spec=spec, eval_cfg=eval_cfg, book=book, **kwargs + ) + return _stub_exec_basis(kwargs["report_dir"], kwargs["stem"]) + + monkeypatch.setattr( + "qt.factor_eval_runner.run_exec_basis_evaluation", fake_exec_eval + ) + return bundle + + +@pytest.mark.parametrize("book_mode", ["decision", "close"]) +def test_end_to_end_wiring_both_book_modes(monkeypatch, tmp_path, book_mode): + captured: dict = {} + _wire(monkeypatch, tmp_path, captured) + result = run_factor_eval("ignored.yaml", "jump_amount_corr_20", book_mode=book_mode) + + # the service produced the subject values; processing ran; the exec tail got them + assert result.requested_symbols == 2 + assert captured["eval_cfg"].view == "decision" + assert captured["eval_cfg"].return_basis == "exec_to_exec" + # book_view IS the book mode (no-book's None is derived inside exec_basis_eval) + assert captured["book_view"] == book_mode + # the book carries exactly the frozen trio + assert sorted(captured["book"].columns) == ["value_bp", "value_ep", "volatility_20"] + # artifact stem isolation: never the legacy eval_{name} stem + assert captured["stem"] == "factor_eval_jump_amount_corr_20" + assert not captured["stem"].startswith("eval_") + # jump publishes no per-day disclosure -> no extra sections, no coverage + assert captured["extra_sections"] is None + assert result.coverage is None + assert result.minute_live_calls == 0 + nb = result.exec_basis.no_book_md + assert nb.name == "factor_eval_jump_amount_corr_20_exec_no_book.md" + wb = result.exec_basis.with_book_md + if book_mode == "close": + # the with-book artifacts are suffix-isolated from the decision-mode ones + assert wb.name == "factor_eval_jump_amount_corr_20_exec_with_book_bookclose.md" + assert wb.exists() + assert result.exec_basis.with_book_json.exists() + assert result.exec_basis.with_book_dashboard.exists() + else: + assert wb.name == "factor_eval_jump_amount_corr_20_exec_with_book.md" + + +def test_disclosure_rides_the_diagnostics_sink_into_an_extra_section( + monkeypatch, tmp_path +): + """valley_ridge_vwap_ratio: sink -> summarizer -> add-Section passthrough.""" + captured: dict = {} + _wire(monkeypatch, tmp_path, captured) + + diag = pd.DataFrame( + { + "classifiable_bars": [240, 240, 240, 240], + "valley_bars": [200, 200, 200, 200], + "ridge_bars": [4, 12, 25, 30], + "valid": [False, True, True, True], + }, + index=pd.DatetimeIndex( + pd.bdate_range("2022-01-03", periods=4), name="trade_date" + ), + ) + def fake_panel(factor_ids, universe, decisions, **kwargs): + diagnostics = kwargs.get("diagnostics") + if diagnostics is not None: + diagnostics.append(diag) + idx = pd.MultiIndex.from_product( + [pd.DatetimeIndex(DATES), list(universe)], names=["date", "symbol"] + ) + return pd.DataFrame( + {fid: np.arange(len(idx), dtype=float) for fid in factor_ids}, index=idx + ) + + import factors.service as service_mod + + monkeypatch.setattr(service_mod, "panel", fake_panel) + result = run_factor_eval("ignored.yaml", "valley_ridge_vwap_ratio_20") + + # the coverage was summarized from the sink and rendered into the run record + assert isinstance(result.coverage, RidgeCoverage) + assert result.coverage.symbol_days == 4 + assert result.coverage.valid_days == 3 + # ...and it reached the exec tail as exactly one add-Section + extras = captured["extra_sections"] + assert extras is not None and len(extras) == 1 + section = extras[0] + assert isinstance(section, Section) + assert section.name == "ridge_scarcity_coverage" + assert section.payload["valid_days"] == 3 + assert section.note == result.coverage.render() + # every requested symbol had a finite value in the fake panel + assert result.covered_symbols == 2 + assert result.empty_symbols == 0 diff --git a/tests/test_factor_store_incremental.py b/tests/test_factor_store_incremental.py index 191144c..39b9395 100644 --- a/tests/test_factor_store_incremental.py +++ b/tests/test_factor_store_incremental.py @@ -332,3 +332,107 @@ def test_schema_version_change_triggers_full_recompute_not_a_revision(tmp_path, # the store now carries the new schema and the full column assert store.stored_fingerprint(_key())["schema_version"] == "SCHEMA_NEW" assert len(store.read(_key())) == len(full) + + +# --------------------------------------------------------------------------- # +# D4c footprint fills: stored NaN -> finite is NOT an upstream revision (D5 C4) +# --------------------------------------------------------------------------- # +def _seed_with_footprint_nans(tmp_path, nan_symbol="AAA", nan_tail_days=3): + """Seed a store whose overlap carries explicit NaN FOOTPRINT rows. + + Mirrors ``factors.service._record_fill_footprint``: a fill covered these + cells and the factor produced nothing, so the store holds NaN. After the + upstream cache warms, the same recompute returns FINITE values there. + """ + dates = _business_dates(30) + raw = _raw_panel(dates) + clean = _make_recompute(raw) + today = dates[-1] + full = clean(None, today, _TRANSITIVE_DEPTH) + cutoff = dates[-6] + seeded = full[full.index.get_level_values("date") <= cutoff].copy() + d = seeded.index.get_level_values("date") + s = seeded.index.get_level_values("symbol") + footprint = (s == nan_symbol) & (d > cutoff - pd.Timedelta(days=nan_tail_days)) + assert footprint.any() + seeded[footprint] = np.nan + store = FactorValueStore(tmp_path) + store.write(_key(), seeded, fingerprint=_fp()) + return store, clean, full, today + + +def test_stored_nan_refilled_is_a_footprint_fill_not_a_revision(tmp_path, caplog): + store, clean, full, today = _seed_with_footprint_nans(tmp_path) + factor = _make_factor(lookback_depth=_TRANSITIVE_DEPTH, adjustment="returns_invariant") + logger = logging.getLogger("test.tailrecompute") + with caplog.at_level(logging.WARNING): + result = tail_recompute( + store, _key(), factor, recompute=clean, today=today, + horizon_cfg=CacheHorizonConfig(refresh_recent_days=1), fingerprint=_fp(), + logger=logger, + ) + # a stored NaN asserts no value, so refilling it is NOT a revision ... + assert result.revision_detected is False + assert result.mismatched_symbols == () + assert result.recolumned_symbols == () + # ... it is counted as a footprint fill ... + assert any("filled_after_footprint: 1 symbol(s)" in n for n in result.notes) + assert not any("REVISION detected" in r.message for r in caplog.records) + # ... and the recomputed values are absorbed (the store now matches batch). + assert _nan_aware_equal(store.read(_key()), full) + + +def test_finite_to_nan_is_still_a_loud_revision(tmp_path, caplog): + dates = _business_dates(30) + raw = _raw_panel(dates) + clean = _make_recompute(raw) + today = dates[-1] + full = clean(None, today, _TRANSITIVE_DEPTH) + store = FactorValueStore(tmp_path) + cutoff = dates[-6] + store.write(_key(), full[full.index.get_level_values("date") <= cutoff], fingerprint=_fp()) + + def nan_out(emit_start, end, warmup): + s = clean(emit_start, end, warmup).copy() + s[s.index.get_level_values("symbol") == "AAA"] = np.nan + return s + + factor = _make_factor(lookback_depth=_TRANSITIVE_DEPTH, adjustment="returns_invariant") + with caplog.at_level(logging.WARNING): + result = tail_recompute( + store, _key(), factor, recompute=nan_out, today=today, + horizon_cfg=CacheHorizonConfig(refresh_recent_days=1), fingerprint=_fp(), + logger=logging.getLogger("test.tailrecompute"), + ) + # a stored FINITE value contradicted (here: erased to NaN) IS a revision. + assert "AAA" in result.mismatched_symbols + assert result.revision_detected is True + assert any("REVISION detected" in r.message for r in caplog.records) + + +# --------------------------------------------------------------------------- # +# cross-sectional intermediate payload: explicitly NOT WIRED (D5 C4 deferral) +# --------------------------------------------------------------------------- # +def test_intermediate_payload_raises_the_named_not_wired_error(tmp_path): + from factors.store.incremental import IntermediatePayloadNotWiredError + + dates = _business_dates(10) + idx = pd.MultiIndex.from_product([dates, ["AAA", "BBB"]], names=["date", "symbol"]) + frame = pd.DataFrame( + {"stat_a": np.arange(len(idx), dtype=float), "stat_b": 1.0}, index=idx + ) + store = FactorValueStore(tmp_path) + store.write_frame(_key(), frame, fingerprint=_fp()) + + factor = _make_factor(lookback_depth=_TRANSITIVE_DEPTH) + recompute = _make_recompute(_raw_panel(dates)) + with pytest.raises(IntermediatePayloadNotWiredError, match="NOT") as excinfo: + tail_recompute( + store, _key(), factor, recompute=recompute, today=dates[-1], + horizon_cfg=CacheHorizonConfig(refresh_recent_days=1), fingerprint=_fp(), + ) + msg = str(excinfo.value) + assert "per-symbol INTERMEDIATE" in msg + assert "deliberate deferral" in msg + # a ValueError subclass, so any legacy ``except ValueError`` still catches it. + assert isinstance(excinfo.value, ValueError)