From fbb63c747e1d4ba5ab77fb579ba03845d063aeaf Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 01:12:05 -0700 Subject: [PATCH 1/2] feat(qt): add the D1 end-of-stage raw factor-panel freeze tool qt/panel_freeze.py freezes the closing 14 factors' RAW daily panels (11 minute factors + the 3 book factors) on the eleven-factor evaluation data plane (CSI500 000905.SH, 2021-07-01..2026-06-30, cache-only) as the pre-D2 baseline for the D5 cell-by-cell reconciliation. No factor math is reimplemented: every panel is produced by calling the qt/eval_*.py runners' own loader chains with the runners' own constants, imported from the runner modules themselves; book factors go through _build_book_factors + factor.compute on the enriched panel. The frozen value is raw (pre-_process_factors). The authoritative fingerprint is a canonical content hash (sha256 over the sorted (date, symbol) index serialization + float64 raw bytes, NaN payloads collapsed, +/-0.0 kept distinct) because parquet file bytes depend on writer metadata; the file sha256 is recorded as a convenience only. Panels are written atomically (tmp + os.replace). Built-in verification: per-factor reconciliation of the processed boundary against the shipped eval_*_no_book.json data_coverage payloads (mismatch raises, the panel is never frozen), a determinism double-run (2 minute + 1 book subjects), a zero-live-call assertion on every minute loader, and a --resume mode that re-verifies existing panel files through the same processing + reconciliation instead of trusting them. Runs via python -m qt.panel_freeze (deliberately not registered in qt/cli). tests/test_panel_freeze.py (23 network-free tests) pins the hash semantics (value/index sensitivity paired with order/NaN-payload invariance), the atomic-write failure paths, manifest field completeness, and the reconciliation mismatch discipline. --- qt/panel_freeze.py | 770 +++++++++++++++++++++++++++++++++++++ tests/test_panel_freeze.py | 315 +++++++++++++++ 2 files changed, 1085 insertions(+) create mode 100644 qt/panel_freeze.py create mode 100644 tests/test_panel_freeze.py diff --git a/qt/panel_freeze.py b/qt/panel_freeze.py new file mode 100644 index 0000000..2099c78 --- /dev/null +++ b/qt/panel_freeze.py @@ -0,0 +1,770 @@ +"""D1 end-of-stage RAW factor-value panel freeze (the pre-D2 baseline for D5). + +Freezes the closing 14 factors' RAW daily factor-value panels (11 minute-derived +factors + the 3 confirmed book factors) on the eleven-factor evaluation data +plane (CSI500 ``000905.SH``, 2021-07-01 .. 2026-06-30, CACHE-ONLY). ``main`` at +the producing SHA still carries the pre-refactor factor math bit-for-bit (the D1 +registry PR was dispatch-only), so these panels are the ONLY baseline the D5 +cell-by-cell reconciliation can compare against once D2 rewrites the math. + +NO FACTOR MATH IS REIMPLEMENTED HERE. Every panel is produced by calling the +SAME loader chain the ``qt/eval_*.py`` runners call, with the SAME constants, +imported FROM the runner modules themselves (identity with the runner call +sites, not a transcription of them): + +* minute factors — each runner's private ``_load_*_panel`` (per-symbol + cache-only 1min read -> ``data.clean`` ``compute_*`` -> daily aggregation), + then the runner's own restriction to the daily panel dates + (``load.factor[dates.isin(panel_dates)]``). The frozen value is the RAW + restricted series, BEFORE ``_process_factors`` (z-score / neutralization are + shared machinery, not factor math). +* book factors — the runners' ``_build_book_factors()`` + + ``factor.compute(panel)`` on the enriched daily panel, frozen RAW. + +PROVENANCE RULE (design v3.2 §5 leg 4): regenerating this baseline is only +legitimate from a checkout of the pinned pre-D2 producing SHA recorded in the +manifest — NEVER from current code. Rebuilding the baseline from the same tree +that is being validated would make the D5 reconciliation structurally unable to +fail (the ``compare_postmerge.py`` empty-reconciliation failure mode). + +Canonical content hash +---------------------- +A parquet file's byte hash depends on writer metadata (library versions, page +layout), so the AUTHORITATIVE fingerprint is a canonical CONTENT hash, defined +as sha256 over, in order: + +1. the version tag ``"qt.panel_freeze canonical v1"`` + ``\\n``; +2. the row count (ascii) + ``\\n``; +3. the ``date`` level as little-endian int64 epoch-nanoseconds, in canonical + row order, + ``\\n``; +4. the ``symbol`` level joined with ``\\x1f`` (utf-8), same order, + ``\\n``; +5. the values as little-endian float64 raw bytes, same order, with every NaN + rewritten to the single canonical ``np.float64("nan")`` bit pattern. + +Canonical row order = sort by (date, symbol); duplicate (date, symbol) keys are +an error (the hash would otherwise be order-ambiguous). The column NAME is not +hashed (content equality, not labelling). NaN payload bits are collapsed on +purpose; +0.0 and -0.0 stay distinct (a real IEEE value difference). + +Run: ``python -m qt.panel_freeze`` (deliberately NOT registered in qt/cli). +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import logging +import os +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +import numpy as np +import pandas as pd + +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL + +CANONICAL_HASH_VERSION = "qt.panel_freeze canonical v1" +DEFAULT_CONFIG = "config/phase_c_jump_amount_corr.yaml" +DEFAULT_OUTPUT_ROOT = "artifacts/refactor_baseline" +#: PanelStore artifact name for THIS run — content-identical to the runners' +#: per-run panel (same cache, same window), under a freeze-specific name so the +#: freeze never clobbers an accepted eval run's ``artifacts/data`` artifact. +PANEL_STORE_NAME = "d1_panel_freeze_daily" +#: Determinism double-run subjects (task §2): two minute factors — including +#: valley_price_quantile, the one loader that also consumes the daily panel — +#: plus one book factor. +DETERMINISM_FACTORS = ("jump_amount_corr_20", "valley_price_quantile_20", "value_ep") +#: data_coverage payload fields reconciled against the shipped eval JSONs. +#: They describe the PROCESSED series at the evaluator boundary (see +#: ``analytics/eval/standard.py::data_coverage``), so the reconciliation +#: re-derives that boundary from the frozen RAW panel via the runners' own +#: ``_process_factors`` — the frozen artifact itself stays raw. +RECONCILE_INT_FIELDS = ( + "panel_rows", + "evaluation_periods", + "symbols_evaluated", + "universe_symbols_declared", + "dropped_symbols_count", +) +RECONCILE_FLOAT_FIELDS = ("factor_nan_rate",) + + +# --------------------------------------------------------------------------- # +# Canonical content hash + atomic write (pure, network-free, unit-tested) +# --------------------------------------------------------------------------- # +def _as_single_series(panel: pd.Series | pd.DataFrame) -> pd.Series: + """Validate and return the one factor series of ``panel`` (no coercion). + + Requires a 2-level MultiIndex named exactly (``date``, ``symbol``) with a + ``datetime64[ns]`` date level, unique (date, symbol) keys, and numeric + values. Anything else raises — the freeze never silently reshapes. + """ + if isinstance(panel, pd.DataFrame): + if panel.shape[1] != 1: + raise ValueError( + f"expected a single-column factor panel; got {panel.shape[1]} columns " + f"({list(panel.columns)!r})." + ) + series = panel.iloc[:, 0] + elif isinstance(panel, pd.Series): + series = panel + else: + raise TypeError( + f"expected a pandas Series or single-column DataFrame; got " + f"{type(panel).__name__}." + ) + index = series.index + if not isinstance(index, pd.MultiIndex) or index.nlevels != 2: + raise ValueError("factor panel index must be a 2-level MultiIndex(date, symbol).") + if tuple(index.names) != (DATE_LEVEL, SYMBOL_LEVEL): + raise ValueError( + f"factor panel index levels must be named ({DATE_LEVEL!r}, {SYMBOL_LEVEL!r}); " + f"got {tuple(index.names)!r}." + ) + dates = index.get_level_values(DATE_LEVEL) + if str(dates.dtype) != "datetime64[ns]": + raise ValueError( + f"date level must be datetime64[ns]; got {dates.dtype}. The freeze does " + "not coerce dtypes — fix the producer." + ) + if index.has_duplicates: + dupes = index[index.duplicated()].tolist()[:3] + raise ValueError( + f"factor panel has duplicate (date, symbol) keys (e.g. {dupes!r}); the " + "canonical hash would be order-ambiguous." + ) + if not pd.api.types.is_numeric_dtype(series): + raise ValueError(f"factor values must be numeric; got dtype {series.dtype}.") + return series + + +def canonical_content_hash(panel: pd.Series | pd.DataFrame) -> str: + """The authoritative content fingerprint (module docstring definition). + + Row-order independent (canonical sort applied first), sensitive to any + value or (date, symbol) index change, NaN-payload independent. + """ + series = _as_single_series(panel).sort_index(kind="mergesort") + dates = series.index.get_level_values(DATE_LEVEL) + symbols = series.index.get_level_values(SYMBOL_LEVEL) + values = np.array(series.to_numpy(), dtype=" str: + """Atomically write the panel (canonical row order) as parquet; return file sha256. + + tmp-file + ``os.replace`` in the target directory: readers never observe a + partial file, and a failed write leaves no tmp residue (and never touches an + existing target). + """ + series = _as_single_series(panel).sort_index(kind="mergesort") + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + tmp = target.with_name(target.name + ".tmp") + try: + series.to_frame().reset_index().to_parquet(tmp, engine="pyarrow", index=False) + os.replace(tmp, target) + finally: + tmp.unlink(missing_ok=True) # failure path only; os.replace consumed it on success + return file_sha256(target) + + +def file_sha256(path: Path | str) -> str: + """sha256 of the file bytes (convenience fingerprint; NOT the authority).""" + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def read_frozen_panel(path: Path | str, factor_id: str) -> pd.Series: + """Read one frozen panel back into the canonical MultiIndex(date, symbol) series.""" + frame = pd.read_parquet(path) + if factor_id not in frame.columns: + raise ValueError( + f"frozen panel {Path(path).name} carries no column {factor_id!r} " + f"(columns: {list(frame.columns)!r})." + ) + series = frame.set_index([DATE_LEVEL, SYMBOL_LEVEL])[factor_id] + return _as_single_series(series).sort_index(kind="mergesort") + + +# --------------------------------------------------------------------------- # +# Manifest rows + rendering (pure, unit-tested) +# --------------------------------------------------------------------------- # +MANIFEST_ROW_FIELDS = ( + "factor_id", + "kind", + "rows", + "date_min", + "date_max", + "n_symbols", + "n_nan", + "mean", + "std", + "canonical_sha256", + "file_sha256", + "file", +) + + +def manifest_row( + factor_id: str, + kind: str, + panel: pd.Series | pd.DataFrame, + canonical_sha256: str, + file_sha256_hex: str, + file_name: str, +) -> dict: + """One manifest record. mean/std are float64 ``Series.mean()``/``std(ddof=1)`` + with NaN skipped (pandas defaults), reported at full precision.""" + series = _as_single_series(panel) + dates = series.index.get_level_values(DATE_LEVEL) + return { + "factor_id": str(factor_id), + "kind": str(kind), + "rows": int(len(series)), + "date_min": dates.min().strftime("%Y-%m-%d") if len(series) else None, + "date_max": dates.max().strftime("%Y-%m-%d") if len(series) else None, + "n_symbols": int(series.index.get_level_values(SYMBOL_LEVEL).nunique()), + "n_nan": int(series.isna().sum()), + "mean": float(series.mean()), + "std": float(series.std()), + "canonical_sha256": canonical_sha256, + "file_sha256": file_sha256_hex, + "file": str(file_name), + } + + +def render_manifest_markdown(header: dict, rows: list[dict]) -> str: + """Deterministic Markdown manifest: a header block + one row per factor.""" + lines = ["# D1 panel freeze manifest", ""] + for key in sorted(header): + lines.append(f"- **{key}**: {header[key]}") + lines.append("") + cols = list(MANIFEST_ROW_FIELDS) + lines.append("| " + " | ".join(cols) + " |") + lines.append("|" + "|".join("---" for _ in cols) + "|") + for row in rows: + cells = [] + for col in cols: + value = row.get(col) + cells.append(repr(value) if isinstance(value, float) else str(value)) + lines.append("| " + " | ".join(cells) + " |") + lines.append("") + return "\n".join(lines) + + +def atomic_write_text(text: str, path: Path | str) -> None: + """tmp + ``os.replace`` text write (same atomicity contract as the parquet).""" + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + tmp = target.with_name(target.name + ".tmp") + try: + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, target) + finally: + tmp.unlink(missing_ok=True) + + +# --------------------------------------------------------------------------- # +# Recipes: the runner call sites, referenced (not transcribed) +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class MinuteRecipe: + """One minute factor's runner-owned build chain. + + ``build`` closes over the runner module's loader AND its constants, so the + call is identical to the runner's own call site by construction. + """ + + factor_id: str + stem: str # the runner's _REPORT_STEM (names the eval JSON artifacts) + spec: object + build: Callable[[object, list[str], pd.DataFrame, logging.Logger], object] + + +def minute_recipes() -> list[MinuteRecipe]: + """The 11 minute factors, each bound to its runner's loader + constants. + + Imports live inside the function so importing ``qt.panel_freeze`` for the + network-free unit tests stays light. + """ + import qt.eval_amp_marginal_anomaly_vol as e + import qt.eval_intraday_amp_cut as g + import qt.eval_jump_amount_corr as c + import qt.eval_minute_ideal_amplitude as d + import qt.eval_peak_interval_kurtosis as h + import qt.eval_peak_ridge_amount_ratio as m + import qt.eval_ridge_minute_return as k + import qt.eval_valley_price_quantile as ell + import qt.eval_valley_relative_vwap as i + import qt.eval_valley_ridge_vwap_ratio as j + import qt.eval_volume_peak_count as f + + recipes: list[MinuteRecipe] = [] + + def add(module, factor, build) -> None: + spec = factor.spec + recipes.append( + MinuteRecipe( + factor_id=spec.factor_id, + stem=module._REPORT_STEM, + spec=spec, + build=build, + ) + ) + + jac = c.JumpAmountCorrFactor(lookback_days=c.JUMP_LOOKBACK_DAYS) + add(c, jac, lambda cfg, symbols, panel, logger, _s=jac.spec: c._load_jump_factor_panel( + cfg, symbols, _s, logger, + lookback_days=c.JUMP_LOOKBACK_DAYS, min_pairs=c.JUMP_MIN_PAIRS, + )) + + mia = d.MinuteIdealAmplitudeFactor(lookback_days=d.IDEAL_AMP_LOOKBACK_DAYS) + add(d, mia, lambda cfg, symbols, panel, logger, _s=mia.spec: d._load_minute_ideal_amp_panel( + cfg, symbols, _s, logger, + lookback_days=d.IDEAL_AMP_LOOKBACK_DAYS, lam=d.IDEAL_AMP_LAMBDA, + min_minutes=d.IDEAL_AMP_MIN_MINUTES, + )) + + amav = e.AmpMarginalAnomalyVolFactor(lookback_days=e.AMP_ANOMALY_LOOKBACK_DAYS) + add(e, amav, lambda cfg, symbols, panel, logger, _s=amav.spec: e._load_amp_anomaly_vol_panel( + cfg, symbols, _s, logger, + lookback_days=e.AMP_ANOMALY_LOOKBACK_DAYS, min_pool=e.AMP_ANOMALY_MIN_POOL, + min_selected=e.AMP_ANOMALY_MIN_SELECTED, sigma_k=e.AMP_ANOMALY_SIGMA_K, + )) + + vpc = f.VolumePeakCountFactor(lookback_days=f.VOLUME_PRV_LOOKBACK_DAYS) + add(f, vpc, lambda cfg, symbols, panel, logger, _s=vpc.spec: f._load_volume_peak_count_panel( + cfg, symbols, _s, logger, + lookback_days=f.VOLUME_PRV_LOOKBACK_DAYS, baseline_days=f.VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs=f.VOLUME_PRV_BASELINE_MIN_OBS, sigma_k=f.VOLUME_PRV_SIGMA_K, + min_valid_days=f.VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable=f.VOLUME_PRV_MIN_CLASSIFIABLE, + )) + + iac = g.IntradayAmpCutFactor(lookback_days=g.AMP_CUT_LOOKBACK_DAYS) + add(g, iac, lambda cfg, symbols, panel, logger, _s=iac.spec: g._load_amp_cut_panel( + cfg, symbols, _s, logger, + lookback_days=g.AMP_CUT_LOOKBACK_DAYS, lam=g.AMP_CUT_LAMBDA, + min_day_minutes=g.AMP_CUT_MIN_DAY_MINUTES, min_valid_days=g.AMP_CUT_MIN_VALID_DAYS, + min_cross_section=g.AMP_CUT_MIN_CROSS_SECTION, + )) + + pik = h.PeakIntervalKurtosisFactor(lookback_days=h.PEAK_INTERVAL_LOOKBACK_DAYS) + add(h, pik, lambda cfg, symbols, panel, logger, _s=pik.spec: h._load_peak_interval_kurtosis_panel( + cfg, symbols, _s, logger, + lookback_days=h.PEAK_INTERVAL_LOOKBACK_DAYS, baseline_days=h.VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs=h.VOLUME_PRV_BASELINE_MIN_OBS, sigma_k=h.VOLUME_PRV_SIGMA_K, + min_valid_days=h.VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable=h.VOLUME_PRV_MIN_CLASSIFIABLE, + min_intervals=h.PEAK_INTERVAL_MIN_INTERVALS, + )) + + vrv = i.ValleyRelativeVwapFactor(lookback_days=i.VALLEY_VWAP_LOOKBACK_DAYS) + add(i, vrv, lambda cfg, symbols, panel, logger, _s=vrv.spec: i._load_valley_relative_vwap_panel( + cfg, symbols, _s, logger, + lookback_days=i.VALLEY_VWAP_LOOKBACK_DAYS, baseline_days=i.VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs=i.VOLUME_PRV_BASELINE_MIN_OBS, sigma_k=i.VOLUME_PRV_SIGMA_K, + min_valid_days=i.VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable=i.VOLUME_PRV_MIN_CLASSIFIABLE, + min_valley_bars=i.VALLEY_VWAP_MIN_VALLEY_BARS, + )) + + vrr = j.ValleyRidgeVwapRatioFactor(lookback_days=j.VALLEY_RIDGE_LOOKBACK_DAYS) + add(j, vrr, lambda cfg, symbols, panel, logger, _s=vrr.spec: j._load_valley_ridge_vwap_ratio_panel( + cfg, symbols, _s, logger, + lookback_days=j.VALLEY_RIDGE_LOOKBACK_DAYS, baseline_days=j.VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs=j.VOLUME_PRV_BASELINE_MIN_OBS, sigma_k=j.VOLUME_PRV_SIGMA_K, + min_valid_days=j.VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable=j.VOLUME_PRV_MIN_CLASSIFIABLE, + min_valley_bars=j.VALLEY_RIDGE_MIN_VALLEY_BARS, + min_ridge_bars=j.VALLEY_RIDGE_MIN_RIDGE_BARS, + )) + + rmr = k.RidgeMinuteReturnFactor(lookback_days=k.RIDGE_RETURN_LOOKBACK_DAYS) + add(k, rmr, lambda cfg, symbols, panel, logger, _s=rmr.spec: k._load_ridge_minute_return_panel( + cfg, symbols, _s, logger, + lookback_days=k.RIDGE_RETURN_LOOKBACK_DAYS, baseline_days=k.VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs=k.VOLUME_PRV_BASELINE_MIN_OBS, sigma_k=k.VOLUME_PRV_SIGMA_K, + min_valid_days=k.VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable=k.VOLUME_PRV_MIN_CLASSIFIABLE, + min_ridge_bars=k.RIDGE_RETURN_MIN_RIDGE_BARS, + )) + + vpq = ell.ValleyPriceQuantileFactor(lookback_days=ell.VALLEY_QUANTILE_LOOKBACK_DAYS) + # The ONE loader that also consumes the daily panel (its reversal + # neutralization needs daily closes) — the runner passes it positionally. + add(ell, vpq, lambda cfg, symbols, panel, logger, _s=vpq.spec: ell._load_valley_price_quantile_panel( + cfg, symbols, _s, panel, logger, + lookback_days=ell.VALLEY_QUANTILE_LOOKBACK_DAYS, + baseline_days=ell.VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs=ell.VOLUME_PRV_BASELINE_MIN_OBS, sigma_k=ell.VOLUME_PRV_SIGMA_K, + min_valid_days=ell.VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable=ell.VOLUME_PRV_MIN_CLASSIFIABLE, + min_valley_bars=ell.VALLEY_QUANTILE_MIN_VALLEY_BARS, + min_cross_section=ell.VALLEY_QUANTILE_MIN_CROSS_SECTION, + reversal_days=ell.VALLEY_QUANTILE_REVERSAL_DAYS, + )) + + pra = m.PeakRidgeAmountRatioFactor(lookback_days=m.PEAK_RIDGE_LOOKBACK_DAYS) + add(m, pra, lambda cfg, symbols, panel, logger, _s=pra.spec: m._load_peak_ridge_amount_ratio_panel( + cfg, symbols, _s, logger, + lookback_days=m.PEAK_RIDGE_LOOKBACK_DAYS, baseline_days=m.VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs=m.VOLUME_PRV_BASELINE_MIN_OBS, sigma_k=m.VOLUME_PRV_SIGMA_K, + min_valid_days=m.VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable=m.VOLUME_PRV_MIN_CLASSIFIABLE, + min_peak_bars=m.PEAK_RIDGE_MIN_PEAK_BARS, + min_ridge_bars=m.PEAK_RIDGE_MIN_RIDGE_BARS, + )) + + return recipes + + +# --------------------------------------------------------------------------- # +# Artifact reconciliation (against the shipped eval JSONs) +# --------------------------------------------------------------------------- # +def reconcile_with_eval_artifact( + stem: str, + processed: pd.Series, + declared_symbols: list[str], + reports_dir: Path, +) -> dict: + """Check the frozen panel's coverage against ``{stem}_no_book.json``. + + The JSON's ``data_coverage`` fields describe the PROCESSED factor series at + the evaluator boundary; ``processed`` here was re-derived from the frozen + RAW panel through the runners' own ``_process_factors``, so equality proves + the frozen panel is the same series the shipped evaluation consumed. Any + mismatch raises — a divergent panel must NOT be recorded (task: stop and + investigate, never freeze a discrepancy). + """ + json_path = reports_dir / f"{stem}_no_book.json" + if not json_path.exists(): + raise FileNotFoundError( + f"eval artifact {json_path.name} not found under {reports_dir} — the " + "reconciliation target is missing; refusing to freeze unreconciled." + ) + document = json.loads(json_path.read_text(encoding="utf-8")) + coverage = None + for section in document.get("sections", []): + if section.get("name") == "data_coverage": + coverage = section.get("payload", {}) + if not coverage: + raise ValueError(f"{json_path.name} carries no data_coverage payload.") + + values = np.asarray(processed.to_numpy(), dtype=float) + finite = np.isfinite(values) + total = int(len(processed)) + evaluated = pd.unique(processed.index.get_level_values(SYMBOL_LEVEL)) + ours: dict[str, object] = { + "panel_rows": total, + "evaluation_periods": int( + pd.unique(processed.index.get_level_values(DATE_LEVEL)).size + ), + "symbols_evaluated": int(len(evaluated)), + "universe_symbols_declared": int(len(declared_symbols)), + "dropped_symbols_count": len( + set(map(str, declared_symbols)) - set(map(str, evaluated)) + ), + # the eval JSON writer rounds floats to 6 decimals (data.quality clean_value) + "factor_nan_rate": round(1.0 - finite.sum() / total, 6) if total else None, + } + checks: dict[str, dict] = {} + mismatches: list[str] = [] + for field in RECONCILE_INT_FIELDS + RECONCILE_FLOAT_FIELDS: + expected = coverage.get(field) + actual = ours[field] + ok = expected == actual + checks[field] = {"artifact": expected, "frozen": actual, "ok": bool(ok)} + if not ok: + mismatches.append(f"{field}: artifact={expected!r} frozen={actual!r}") + if mismatches: + raise ValueError( + f"frozen panel disagrees with {json_path.name} — NOT freezing: " + + "; ".join(mismatches) + ) + return {"artifact": json_path.name, "checks": checks} + + +# --------------------------------------------------------------------------- # +# Freeze orchestration +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class FreezeResult: + """Immutable summary of one panel-freeze run (no secrets).""" + + output_root: Path + manifest_json: Path + manifest_md: Path + rows: tuple[dict, ...] + header: dict + elapsed: float + + +def _git_head_sha() -> str: + """Best-effort producing SHA (the manifest doc records the authoritative one).""" + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, text=True, check=True, timeout=10, + ) + return out.stdout.strip() + except Exception: # noqa: BLE001 - provenance is best-effort here + return "UNKNOWN" + + +def run_panel_freeze( + config_path: str = DEFAULT_CONFIG, + output_root: str | Path = DEFAULT_OUTPUT_ROOT, + *, + resume: bool = False, +) -> FreezeResult: + """Freeze all 14 RAW factor panels + verify determinism + reconcile artifacts. + + ``resume=True`` completes an interrupted freeze: a minute factor whose panel + file already exists is NOT rebuilt from the minute cache — its RAW series is + read back FROM the frozen file and then pushed through the SAME processing + + eval-artifact reconciliation before being accepted into the manifest (a + stale or corrupted file fails loudly; nothing is reused unverified). Its + canonical hash is therefore the hash of the file CONTENT. Book factors are + always recomputed (cheap). The determinism double-run still rebuilds its + subjects end-to-end via the runner loaders, so a resumed subject is compared + fresh-build-vs-frozen-file — a strictly stronger check than two in-process + builds. Resumed factor ids are disclosed in the manifest header. + """ + from qt.config import load_config + from qt.eval_jump_amount_corr import _build_book_factors, _check_preconditions + from qt.pipeline import ( + _build_cache, + _build_universe, + _load_panel, + _log_run_cache_stats, + _make_logger, + _maybe_enrich_covariates, + _maybe_enrich_value, + _process_factors, + ) + + started = time.monotonic() + cfg = load_config(config_path) + _check_preconditions(cfg) # tushare + cache-only + PIT index + neutralize, as the runners + # Freeze-specific PanelStore name: identical content, never clobbers an + # accepted eval run's per-run panel artifact. + cfg = cfg.model_copy( + update={"data": cfg.data.model_copy(update={"output_name": PANEL_STORE_NAME})} + ) + + out_root = Path(output_root) + panels_dir = out_root / "panels" + log_path = Path(cfg.output.log_dir) / "panel_freeze.log" + logger = _make_logger(log_path, name="qt.panel_freeze") + logger.info("panel freeze: config=%s output_root=%s", config_path, out_root) + + # -- shared data plane, the runners' exact preamble --------------------- # + cache = _build_cache(cfg) + universe, symbols = _build_universe(cfg, logger, cache) + panel = _load_panel(cfg, symbols, logger, cache) + book_factors = _build_book_factors() + panel = _maybe_enrich_value(cfg, panel, symbols, book_factors, logger, cache) + panel = _maybe_enrich_covariates(cfg, panel, symbols, logger, cache) + _log_run_cache_stats(cache, logger) + panel_dates = pd.Index( + pd.unique(panel.index.get_level_values(DATE_LEVEL)), name=DATE_LEVEL + ) + + rows: list[dict] = [] + reconciliations: dict[str, dict] = {} + canonical_by_id: dict[str, str] = {} + live_calls_total = 0 + + # -- book factors (raw, pre-processing) --------------------------------- # + book_raw = pd.concat( + [factor.compute(panel).rename(factor.name) for factor in book_factors], axis=1 + ) + for factor in book_factors: + raw = book_raw[factor.name].rename(factor.name) + canonical = canonical_content_hash(raw) + target = panels_dir / f"{factor.name}.parquet" + sha = atomic_write_parquet(raw, target) + rows.append(manifest_row(factor.name, "book", raw, canonical, sha, target.name)) + canonical_by_id[factor.name] = canonical + logger.info("frozen book %s: rows=%d canonical=%s", factor.name, len(raw), canonical) + + # -- minute factors (raw, the runners' own loader chains) ---------------- # + resumed: list[str] = [] + for recipe in minute_recipes(): + target = panels_dir / f"{recipe.factor_id}.parquet" + if resume and target.exists(): + raw = read_frozen_panel(target, recipe.factor_id) + resumed.append(recipe.factor_id) + covered_note = "resumed-from-file" + else: + load = recipe.build(cfg, symbols, panel, logger) + live_calls = int(load.live_calls) + live_calls_total += live_calls + if live_calls != 0: + raise RuntimeError( + f"{recipe.factor_id}: stk_mins_live_calls={live_calls} != 0 — the " + "freeze is cache-only by contract." + ) + raw = load.factor[ + load.factor.index.get_level_values(DATE_LEVEL).isin(panel_dates) + ].rename(recipe.factor_id) + covered_note = f"covered={len(load.covered)}/{load.requested}" + # reconcile BEFORE accepting (a divergent panel is never frozen/kept) + processed = _process_factors( + cfg, raw.to_frame(recipe.factor_id), panel + )[recipe.factor_id] + reconciliations[recipe.factor_id] = reconcile_with_eval_artifact( + recipe.stem, processed, symbols, Path(cfg.output.report_dir) + ) + canonical = canonical_content_hash(raw) + if recipe.factor_id in resumed: + sha = file_sha256(target) + else: + sha = atomic_write_parquet(raw, target) + rows.append( + manifest_row(recipe.factor_id, "minute", raw, canonical, sha, target.name) + ) + canonical_by_id[recipe.factor_id] = canonical + logger.info( + "frozen minute %s: rows=%d %s canonical=%s (reconciled vs %s)", + recipe.factor_id, len(raw), covered_note, canonical, + reconciliations[recipe.factor_id]["artifact"], + ) + + # -- determinism double-run (2 minute + 1 book) -------------------------- # + determinism: dict[str, dict] = {} + for factor_id in DETERMINISM_FACTORS: + rebuilt = _rebuild_for_determinism( + factor_id, cfg, symbols, panel, panel_dates, logger + ) + second = canonical_content_hash(rebuilt) + first = canonical_by_id[factor_id] + determinism[factor_id] = {"first": first, "second": second, "ok": second == first} + if second != first: + raise RuntimeError( + f"determinism double-run FAILED for {factor_id}: {first} != {second}" + ) + logger.info("determinism double-run ok: %s %s", factor_id, first) + + header = { + "producing_git_sha": _git_head_sha(), + "config": str(config_path), + "universe": f"{cfg.universe.type}:{cfg.universe.index_code}", + "window": f"{cfg.data.start}..{cfg.data.end}", + "cache_root": str(cfg.data.cache.root_dir), + "panel_store_name": PANEL_STORE_NAME, + "panel_rows": int(len(panel)), + "panel_dates": int(len(panel_dates)), + "universe_symbols": int(len(symbols)), + "stk_mins_live_calls": int(live_calls_total), + "resumed_factors": ",".join(resumed) if resumed else "none", + "elapsed_seconds": round(time.monotonic() - started, 1), + "generated_utc": pd.Timestamp.now(tz="UTC").strftime("%Y-%m-%d %H:%M:%S"), + "canonical_hash_version": CANONICAL_HASH_VERSION, + "determinism_double_run": "; ".join( + f"{fid}:{'ok' if determinism[fid]['ok'] else 'FAIL'}" + for fid in DETERMINISM_FACTORS + ), + "artifact_reconciliation": ( + f"{len(reconciliations)}/11 minute factors reconciled against " + "eval_*_no_book.json data_coverage (book factors carry no coverage " + "fields in the eval artifacts — disclosed, not skipped silently)" + ), + } + + manifest = { + "header": header, + "rows": rows, + "reconciliation": reconciliations, + "determinism": determinism, + } + manifest_json = out_root / "manifest.json" + manifest_md = out_root / "manifest.md" + atomic_write_text(json.dumps(manifest, indent=2, sort_keys=True), manifest_json) + atomic_write_text(render_manifest_markdown(header, rows), manifest_md) + logger.info( + "panel freeze complete: %d panels, %ss, manifest=%s", + len(rows), header["elapsed_seconds"], manifest_json, + ) + return FreezeResult( + output_root=out_root, + manifest_json=manifest_json, + manifest_md=manifest_md, + rows=tuple(rows), + header=header, + elapsed=time.monotonic() - started, + ) + + +def _rebuild_for_determinism( + factor_id: str, + cfg, + symbols: list[str], + panel: pd.DataFrame, + panel_dates: pd.Index, + logger: logging.Logger, +) -> pd.Series: + """Second, independent build of one factor's RAW panel (same chains). + + Minute factors re-run the runner loader end-to-end (a fresh cache-store + read); book factors re-instantiate ``_build_book_factors()`` and recompute + from the shared enriched panel (the panel load itself is once-per-process, + exactly as in the runners; its own determinism is covered by the cache + equivalence suites). + """ + from qt.eval_jump_amount_corr import _build_book_factors + + for factor in _build_book_factors(): + if factor.name == factor_id: + return factor.compute(panel).rename(factor_id) + for recipe in minute_recipes(): + if recipe.factor_id == factor_id: + load = recipe.build(cfg, symbols, panel, logger) + return load.factor[ + load.factor.index.get_level_values(DATE_LEVEL).isin(panel_dates) + ].rename(factor_id) + raise KeyError(f"unknown determinism factor_id {factor_id!r}") + + +def main(argv: list[str] | None = None) -> int: + """CLI: ``python -m qt.panel_freeze [--config ...] [--output-root ...]``.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--config", default=DEFAULT_CONFIG) + parser.add_argument("--output-root", default=DEFAULT_OUTPUT_ROOT) + parser.add_argument( + "--resume", + action="store_true", + help=( + "complete an interrupted freeze: existing minute panels are read back " + "from their frozen files and re-verified (processing + eval-artifact " + "reconciliation) instead of being rebuilt from the minute cache" + ), + ) + args = parser.parse_args(argv) + result = run_panel_freeze(args.config, args.output_root, resume=args.resume) + print(f"panels frozen: {len(result.rows)} -> {result.output_root / 'panels'}") + print(f"manifest: {result.manifest_json}") + print(f"stk_mins_live_calls: {result.header['stk_mins_live_calls']}") + print(f"elapsed_seconds: {result.header['elapsed_seconds']}") + return 0 + + +if __name__ == "__main__": # pragma: no cover - thin CLI shim + raise SystemExit(main()) diff --git a/tests/test_panel_freeze.py b/tests/test_panel_freeze.py new file mode 100644 index 0000000..124f853 --- /dev/null +++ b/tests/test_panel_freeze.py @@ -0,0 +1,315 @@ +"""Network-free tests for qt.panel_freeze (the D1 raw-panel freeze tool). + +Synthetic panels only — the real-data obligations (determinism double-run, +artifact reconciliation, live-call zero) are RUN-time verifications recorded in +the freeze manifest, not unit tests. Here we pin the pure machinery: + +* canonical content hash: sensitive to any value / index change, row-order + independent, NaN-payload independent, +0/-0 distinct, loud on malformed input; +* atomic parquet write: readers never see a partial file, failed writes leave + no tmp residue and never touch an existing target; +* manifest row / renderer: field-complete, full-precision, deterministic; +* eval-artifact reconciliation: exact-match discipline — a mismatch raises. + +Every invariance claim here is paired with a sensitivity assertion on the same +axis (the shuffle test alone could be satisfied by a constant hash; the +value/index sensitivity tests kill that degenerate implementation). +""" + +from __future__ import annotations + +import json +import struct +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL +from qt.panel_freeze import ( + DETERMINISM_FACTORS, + MANIFEST_ROW_FIELDS, + atomic_write_parquet, + canonical_content_hash, + file_sha256, + manifest_row, + read_frozen_panel, + reconcile_with_eval_artifact, + render_manifest_markdown, +) + + +def _panel(values, keys, name="factor_x") -> pd.Series: + index = pd.MultiIndex.from_tuples( + [(pd.Timestamp(d), s) for d, s in keys], names=[DATE_LEVEL, SYMBOL_LEVEL] + ) + return pd.Series(np.asarray(values, dtype="float64"), index=index, name=name) + + +KEYS = [ + ("2024-01-02", "000001.SZ"), + ("2024-01-02", "600000.SH"), + ("2024-01-03", "000001.SZ"), + ("2024-01-03", "600000.SH"), +] + + +# --------------------------------------------------------------------------- # +# canonical_content_hash +# --------------------------------------------------------------------------- # +def test_hash_row_order_independent(): + base = _panel([1.0, 2.0, 3.0, 4.0], KEYS) + shuffled = base.iloc[[2, 0, 3, 1]] + assert canonical_content_hash(shuffled) == canonical_content_hash(base) + + +def test_hash_value_sensitive(): + base = _panel([1.0, 2.0, 3.0, 4.0], KEYS) + changed = _panel([1.0, 2.0, 3.0, 4.0 + 1e-12], KEYS) + assert canonical_content_hash(changed) != canonical_content_hash(base) + + +def test_hash_symbol_sensitive(): + base = _panel([1.0, 2.0, 3.0, 4.0], KEYS) + keys = list(KEYS) + keys[3] = ("2024-01-03", "600001.SH") + assert canonical_content_hash(_panel([1.0, 2.0, 3.0, 4.0], keys)) != ( + canonical_content_hash(base) + ) + + +def test_hash_date_sensitive(): + base = _panel([1.0, 2.0, 3.0, 4.0], KEYS) + keys = list(KEYS) + keys[3] = ("2024-01-04", "600000.SH") + assert canonical_content_hash(_panel([1.0, 2.0, 3.0, 4.0], keys)) != ( + canonical_content_hash(base) + ) + + +def test_hash_nan_payload_bits_collapse(): + # Two DIFFERENT IEEE NaN bit patterns must hash identically (the canonical + # hash rewrites every NaN to one bit pattern). Build the arrays in numpy so + # no pandas construction path can normalize the payload behind our back. + alt_nan = struct.unpack(" full precision survives the markdown + assert repr(row["mean"]) in text + + +def test_determinism_subjects_cover_two_minute_and_one_book(): + assert "value_ep" in DETERMINISM_FACTORS # the book subject + minute = [f for f in DETERMINISM_FACTORS if f.endswith("_20")] + assert len(minute) >= 2 # two minute subjects (incl. the panel-consuming loader) + + +# --------------------------------------------------------------------------- # +# eval-artifact reconciliation +# --------------------------------------------------------------------------- # +def _write_eval_json(reports_dir: Path, stem: str, payload: dict) -> None: + document = {"sections": [{"name": "data_coverage", "payload": payload}]} + reports_dir.mkdir(parents=True, exist_ok=True) + (reports_dir / f"{stem}_no_book.json").write_text( + json.dumps(document), encoding="utf-8" + ) + + +def _matching_payload(processed: pd.Series, declared: list[str]) -> dict: + values = processed.to_numpy(dtype=float) + evaluated = set(processed.index.get_level_values(SYMBOL_LEVEL)) + return { + "panel_rows": int(len(processed)), + "evaluation_periods": int( + pd.unique(processed.index.get_level_values(DATE_LEVEL)).size + ), + "symbols_evaluated": len(evaluated), + "universe_symbols_declared": len(declared), + "dropped_symbols_count": len(set(declared) - evaluated), + "factor_nan_rate": round(1.0 - np.isfinite(values).sum() / len(values), 6), + } + + +def test_reconcile_passes_on_exact_match(tmp_path: Path): + processed = _panel([1.0, np.nan, 3.0, 4.0], KEYS) + declared = ["000001.SZ", "600000.SH", "999999.SZ"] + _write_eval_json(tmp_path, "eval_x", _matching_payload(processed, declared)) + out = reconcile_with_eval_artifact("eval_x", processed, declared, tmp_path) + assert out["artifact"] == "eval_x_no_book.json" + assert all(check["ok"] for check in out["checks"].values()) + + +def test_reconcile_raises_on_mismatch(tmp_path: Path): + processed = _panel([1.0, np.nan, 3.0, 4.0], KEYS) + declared = ["000001.SZ", "600000.SH"] + payload = _matching_payload(processed, declared) + payload["panel_rows"] += 1 # the artifact claims one more row than we froze + _write_eval_json(tmp_path, "eval_x", payload) + with pytest.raises(ValueError, match="panel_rows"): + reconcile_with_eval_artifact("eval_x", processed, declared, tmp_path) + + +def test_reconcile_raises_on_nan_rate_mismatch(tmp_path: Path): + processed = _panel([1.0, np.nan, 3.0, 4.0], KEYS) + declared = ["000001.SZ", "600000.SH"] + payload = _matching_payload(processed, declared) + payload["factor_nan_rate"] = round(payload["factor_nan_rate"] + 1e-6, 6) + _write_eval_json(tmp_path, "eval_x", payload) + with pytest.raises(ValueError, match="factor_nan_rate"): + reconcile_with_eval_artifact("eval_x", processed, declared, tmp_path) + + +def test_reconcile_missing_artifact_is_loud(tmp_path: Path): + processed = _panel([1.0, 2.0, 3.0, 4.0], KEYS) + with pytest.raises(FileNotFoundError, match="no_book.json"): + reconcile_with_eval_artifact("eval_x", processed, ["000001.SZ"], tmp_path) + + +def test_reconcile_missing_payload_is_loud(tmp_path: Path): + (tmp_path / "eval_x_no_book.json").write_text( + json.dumps({"sections": [{"name": "other", "payload": {"a": 1}}]}), + encoding="utf-8", + ) + processed = _panel([1.0, 2.0, 3.0, 4.0], KEYS) + with pytest.raises(ValueError, match="data_coverage"): + reconcile_with_eval_artifact("eval_x", processed, ["000001.SZ"], tmp_path) From e3299ee19c4cde4067b3a5a0f8eb728074a6a2a5 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 01:12:24 -0700 Subject: [PATCH 2/2] docs(factors): record the D1 panel-freeze manifest for the 14-factor baseline docs/factors/d1_panel_freeze_manifest.md is the git-tracked authority for the frozen baseline: producing SHA 3669c90 (HEAD during both run segments; the freeze tool ran as untracked files, so the tracked tree was bit-identical to main), the data plane (CSI500 000905.SH, 2021-07-01..2026-06-30, cache-only, 10 daily endpoints at zero gap-fetches, stk_mins_live_calls=0), the provenance rule (baseline regeneration only from the pinned pre-D2 SHA, never from current code), the canonical-hash definition pointer, the two-segment run record (first segment killed by the session harness at ~60min after 12/14 panels; the resume segment re-verified those panels from file content through processing + reconciliation), the verification results (determinism double-run 3/3 across processes, eval-artifact reconciliation 6 fields x 11 factors all equal, book factors disclosed as having no coverage fields to reconcile, secret scan 0 hits), and the full 14-row table with per-factor stats and both hashes. --- docs/factors/d1_panel_freeze_manifest.md | 122 +++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 docs/factors/d1_panel_freeze_manifest.md diff --git a/docs/factors/d1_panel_freeze_manifest.md b/docs/factors/d1_panel_freeze_manifest.md new file mode 100644 index 0000000..04f9c66 --- /dev/null +++ b/docs/factors/d1_panel_freeze_manifest.md @@ -0,0 +1,122 @@ +# 因子层重构 D1 末面板冻结 manifest：收官 14 因子 raw 因子值基线 + +> **状态**：D1 末交付物（设计 v3.2 §五第 4 腿 + §九 D1 行；背景 `tmp/design/review_round3_v32_revision_list.md` R11）。 +> 本文是 **入 git 的权威 manifest**；bulk 面板与机器可读 manifest 在 +> `artifacts/refactor_baseline/`（gitignored，不入 git）。 +> **产物意义**：`main` @ producing SHA 的因子数学与重构前逐位一致（D1 registry 是 dispatch-only）。 +> 此后 D2 改写数学，本基线是 **D5 逐格对账的唯一比较对象**。 + +--- + +## 一、Provenance（先读这段再动任何再生成） + +- **producing SHA**：`3669c9068cd53dc684b75e0edb0f94346384808f`（= `main` @ D1 registry 合并后）。 + 两段 run（见 §四）期间 worktree `HEAD` 均为该 SHA 且 tracked 树零修改——冻结工具 + `qt/panel_freeze.py` 当时以 **untracked 新文件**存在，因子/runner/pipeline 模块全部 + 逐位等于 `main@3669c90`。 +- **provenance 规则（v3.2 §五第 4 腿，原文）**：**基线再生只许从钉住的 pre-D2 SHA checkout， + 绝不从当前代码**——这是 `compare_postmerge.py` 空对账（拿新结果和自己比，构造性恒真） + 的结构性预防。具体操作：checkout `3669c90`，把本分支的 `qt/panel_freeze.py` 原样放入 + （它只 import、不改任何因子数学模块），再跑下方命令。**任何在 D2 之后的树上直接重跑 + 本工具得到的"基线"都不是基线**，与冻结哈希不一致时以本文记录的哈希为准。 +- **重跑命令**（cwd = 仓库根，缓存根就位）： + + ``` + /home/shaofl/Development/env_tools/envs/quant_mf/bin/python -m qt.panel_freeze + ``` + + 输出根默认 `artifacts/refactor_baseline`（`--output-root` 可改；`--resume` 语义见 + 模块 docstring——已存在的面板从冻结文件读回并**重新走 process + 对账**后才被接受， + 绝不盲信旧文件）。 + +## 二、数据面（与十一因子评估循环同面） + +| 项 | 值 | +|---|---| +| universe | CSI500 `000905.SH`，PIT 成分（73 快照，996 distinct 成分） | +| 窗口 | 2021-07-01 .. 2026-06-30（1210 个交易日） | +| config | `config/phase_c_jump_amount_corr.yaml`（已程序化验证：11 个 eval config 除 `project.name`/`data.output_name` 外**逐字段一致**，任取其一同面） | +| 缓存根 | `artifacts/cache/tushare/v1`，cache-only | +| 日频端点 | 10 端点 gap-fetch **全 0**（run log `data cache:` 行：market_daily/adj_factor/index_weight/suspend_d/namechange/stk_limit/stock_basic/daily_basic/fina_indicator/index_member_all） | +| `stk_mins_live_calls` | **0**（分钟读走 `IntradayParquetStore.read_range`，无 fetch 闭包，构造性为零；每因子 loader 的 `live_calls` 计数逐一断言为 0） | +| PanelStore | 写为冻结专名 `d1_panel_freeze_daily`（内容与 runner 面板同源同参，不覆盖任何已验收 eval run 的 artifact） | + +## 三、冻结口径（什么被冻结、什么不被冻结） + +- 冻结 **raw 因子值**（处理前）。分钟因子 = 各 runner 私有 `_load_*_panel`（逐 symbol + cache-only 1min 读 → `data/clean` `compute_*` → 日频聚合）后按 runner 原样 + `factor[dates.isin(panel_dates)]` 限制到日频交易网格；书因子 = runner 的 + `_build_book_factors()` + `factor.compute(enriched_panel)`。**零公式重抄**：loader、 + compute、常数全部从 runner 模块 import（同一对象，非转录）。 +- `_process_factors`（zscore + 行业/市值中性化）**不在冻结值内**——它是共享机器不是因子 + 数学；但在对账中被调用（见 §五）。 +- **canonical content hash**（权威指纹）：sha256 over 版本标签 + 行数 + 排序后 + (date,symbol) 的 int64-ns 日期字节 + `\x1f` 连接的 symbol utf-8 + float64 原始字节 + （NaN 归一为单一位型；±0.0 保持可区分）。定义与实现在 `qt/panel_freeze.py` + （`canonical_content_hash`）。文件 sha256 仅为便利指纹（受 parquet writer 元数据影响）。 + +## 四、运行记录（两段，诚实披露） + +| 段 | 时间（本地） | 内容 | +|---|---|---| +| 首段 | 2026-07-23 18:05:57 → ~19:05:54（被会话 harness 在 ~60min 处杀死） | 面板加载 + 3 书因子 + 9 分钟因子（jump…ridge_minute_return）全部冻结并对账通过 | +| resume 段 | 2026-07-23 19:09:08 → 19:35:24（wall **1576.1s**） | 书因子重算（哈希与首段**逐位一致**）；9 个已冻结面板**从文件读回重新 process+对账**（哈希与首段日志逐一一致）；补齐 valley_price_quantile_20 / peak_ridge_amount_ratio_20；确定性双跑；写 manifest | + +⚠️ run log 留存披露：`_make_logger` 以截断模式打开日志（`qt/pipeline.py`，`mode="w"`）， +故 `artifacts/logs/panel_freeze.log` 现只含 resume 段——首段的逐因子日志行未留存于盘。 +这**不削弱**验收：① 工具对每个新建分钟因子的 `live_calls` 是 **raise-on-nonzero** 断言 +（非零即中止，面板不会落盘），首段 12 个面板全部落盘本身就是 live_calls=0 的证据； +② resume 段把首段全部 9 个分钟面板**从文件内容**重新 process + 对账 + 重算 canonical +hash（哈希与首段实时监控记录逐一一致），3 个双跑对象另行端到端重建。resume 语义使 +首段面板的验收从「写文件前对账过」升级为「**文件内容本身**重新对账过」。 + +## 五、验证义务结果 + +- **确定性双跑 3/3 OK**（`jump_amount_corr_20` / `valley_price_quantile_20` / `value_ep`）： + 重建走完整 loader/compute 链，canonical hash 与冻结值逐字节相同。其中 jump 与 + valley_price_quantile 为**跨进程**对比（resume 进程全新构建 vs 首段进程写出的文件内容）， + 强于同进程双跑；value_ep 另有跨 run 佐证（首段与 resume 段独立重算同哈希 + `1404a68f…`）。 +- **与既有 eval artifact 对账 11/11 全过**：对每个分钟因子，把冻结 raw 面板经 runner 自己的 + `_process_factors` 推回 evaluator 边界，与 `artifacts/reports/eval_*_no_book.json` 的 + `data_coverage` payload 逐字段比对——`panel_rows` / `evaluation_periods` / + `symbols_evaluated` / `universe_symbols_declared` / `dropped_symbols_count`（整数精确相等） + + `factor_nan_rate`（JSON writer 的 6 位舍入口径）。**6 字段 × 11 因子全部一致**；不一致 + 即 raise、不入库（机制见 `qt/panel_freeze.py::reconcile_with_eval_artifact`）。 +- **书因子无 JSON 覆盖字段可对账**（eval JSON 的 purity 段只有 anchor IC，无 book 行数/覆盖 + 字段）——照实披露，不假装对过；其正确性证据 = 双跑 + 跨 run 同哈希 + 与面板网格的行数 + 恒等（3 × 1,158,912 = 面板全网格）。 +- **secret scan 0 命中**：17 文件（14 面板 parquet + manifest.json/md + run log）扫真实 + token 值与 secret 配置文件路径/键名标记，全无。 + +## 六、14 因子 manifest 表 + +（mean/std 为 float64 全精度 `Series.mean()` / `std(ddof=1)`，NaN 跳过；canonical hash 为权威。） + +| factor_id | kind | rows | date_min | date_max | n_symbols | n_nan | mean | std | canonical_sha256 | file_sha256 | +|---|---|---|---|---|---|---|---|---|---|---| +| value_ep | book | 1158912 | 2021-07-01 | 2026-06-30 | 996 | 141362 | 0.04807913635037822 | 0.050405247296850246 | 1404a68fc88778e78d47da1ed6375c39abec36244a29961c3316a74f0c042e76 | 3b3a8970fcfe51d9b221da79a8f001a03d622d37592fdd76d19275a5159164d4 | +| value_bp | book | 1158912 | 2021-07-01 | 2026-06-30 | 996 | 4028 | 0.5761035855649241 | 0.4768837399414735 | c2f4d0536f6adcee5475562a6aa4f2036073a3b07a1dae690aa22e7032e01c75 | ce582623bf7846e849dc12605cbdb0b1dc9a39ef1f229522309c57e720e4d9ff | +| volatility_20 | book | 1158912 | 2021-07-01 | 2026-06-30 | 996 | 19920 | 0.02480289343756926 | 0.012617262687296474 | 8d46a34c69852352f6aa063016f19b7a6f12b226c43dd880d58431f409f86d6b | 5f81b0a47a3672778a828160d5db4ab8de6e98d5aaea5f6d25ac3f030af5733c | +| jump_amount_corr_20 | minute | 1159263 | 2021-07-01 | 2026-06-30 | 995 | 880 | 0.5912439770493371 | 0.15727054230823656 | b6359f128c3f645672a7bb62e9f1903760fbf5c70e40842f02f097f9f43ccfb2 | ffea5d028ac2d8c28d3fd2befd5e7cf5d918bf6f68169030502437f4be8c2e5c | +| minute_ideal_amp_10 | minute | 1159263 | 2021-07-01 | 2026-06-30 | 995 | 3980 | 0.00015591462929114136 | 0.0006014530933985754 | b03c721eafe728fe9e68c4db55e1bbabbca386397312045b969b76078e092233 | 44c62ec64629bf3099dda6ad87eb45409db560b3f29a4ee602c6d0046c228bee | +| amp_marginal_anomaly_vol_20 | minute | 1159263 | 2021-07-01 | 2026-06-30 | 995 | 10931 | 0.008288440013596557 | 0.004227652811127039 | b189ecf6471e30e973a5cae4ba9c2f5410ca08caa74169cab1552aedb9c1fe1b | b293a9de4bf5063825e3aef55ebfc4bea6dc8addcbf1abcffe9d426bbc8fa198 | +| volume_peak_count_20 | minute | 1149313 | 2021-07-15 | 2026-06-30 | 995 | 8955 | 263.7242804452637 | 72.10585825271954 | b5e94aedc7c62332b3df399c4caa66ef7fc6e8cc2841c21903066e40ec11f2f0 | 6b5b9b7941b6f3456ae83bac3e707249617c7f382a1cef3228a4089f2186ee04 | +| intraday_amp_cut_10 | minute | 1154288 | 2021-07-08 | 2026-06-30 | 995 | 0 | 2.806990001685937e-18 | 0.8527302575863168 | 33d69c9a0cd3080b6489685211a8ebb3b4bbd2c1d27b487d4cd2627d06632ba0 | a48152e0e9ab79b294b2932183d7b373bf44c8c2116209f6f7e17aee0fbfe44d | +| peak_interval_kurtosis_20 | minute | 1149313 | 2021-07-15 | 2026-06-30 | 995 | 9567 | 8.522552350694497 | 5.613139688378547 | e8e337f21c56a727e2747f31e7f09a85e4c8c725e1d7f9134d90cb262232b624 | 71ad513f9a1af9cddc7fb75f428449963ae1946bf0e4a91e859517f2bf3d3daa | +| valley_relative_vwap_20 | minute | 1146878 | 2021-07-15 | 2026-06-30 | 995 | 8955 | 0.9993996386025606 | 0.0010562130843499646 | 0e3599ce151f099fb9278fb0c1ba5527f1383a906642e2cca01d1b32e8ea8e38 | 4780ad06b8053c1315256e0b44c43141a7877f19d80ce34d613a78c47867d9ea | +| valley_ridge_vwap_ratio_20 | minute | 591524 | 2021-07-15 | 2026-06-30 | 995 | 8955 | 0.9976201213193444 | 0.0028711378709016205 | e56b70f1bc95b02ffcbceabe827a47ab6c1e28a32535f07f6333c9378eb5a496 | c1194bdb0f5606d1bcf5b6da52cb31d940524015e88c79602cb2cc6608a738be | +| ridge_minute_return_20 | minute | 588536 | 2021-07-15 | 2026-06-30 | 995 | 8955 | 0.1622597430688057 | 0.14784924675685046 | b1f476f3e37b6bbf4ec0b5e563bb61cf520690e28d8cbcbe24deb61625cd32f6 | ea082f6e8a00e8c8a0e0ae46da433cd55d73431e83d9ac0d2a7eee3307cffc4d | +| valley_price_quantile_20 | minute | 1146878 | 2021-07-15 | 2026-06-30 | 995 | 10952 | 6.358786574311415e-19 | 0.027529790230469833 | 79f1219768ef95daf48484545658a4e307b0125a11e5fee28eb6c886ed5e422f | cee7ed6686ac4a44c72b7d3020beb418eb99e4194b2de75a58387cd3c1997481 | +| peak_ridge_amount_ratio_20 | minute | 579030 | 2021-07-15 | 2026-06-30 | 995 | 8955 | 0.3818472950505535 | 0.18604246679633485 | 8368be5d720184d4fa606cdfa5cf68bfa078179c4e27a196a933e42c26d0725c | 08ee6e3dc705bea03c66e87b566372f2966c3026d2fdb464e5e2d49bdded179b | + +读数注（不是异常）：分钟因子共享同一分钟覆盖网格时 rows 相同（1,159,263）；带自身 +classifiable/valley/ridge 门的因子 rows 少（peak/ridge 家族 ~57–59 万行是因子定义的 +稀疏性，与 eval JSON 逐字段对账一致）；书因子 rows = 面板全网格 1,158,912。 + +## 七、D5 使用方式 + +D2 重写因子数学后，D5 在**同一数据面**重算 14 因子 raw 面板，与 +`artifacts/refactor_baseline/panels/{factor_id}.parquet` 逐格对账；canonical hash 相同 ⇔ +逐位一致，不同则用面板逐格 diff 定位。比较工具必须**独立读取两份产物**（不经共享中间 +产物），并先核对本文 §一 的 producing SHA——防的是同一类空对账。