diff --git a/analytics/eval/__init__.py b/analytics/eval/__init__.py new file mode 100644 index 0000000..65e3b3c --- /dev/null +++ b/analytics/eval/__init__.py @@ -0,0 +1,88 @@ +"""Factor-evaluation contract: the frozen acceptance target (PR-A). + +Standardizes AND enforces how a factor is evaluated, so "write a new factor -> +evaluate it" cannot route around a set of mandatory sections. Three enforcement +layers (design ``tmp/design/factor_eval_contract_v0.1.md`` §7): + + construction FactorSpec / EvalConfig __post_init__ + Factor.__init_subclass__ + -> no hypothesis, no PIT contract, no spec = no object. + assembly every section is a Section or an explicit Skipped(reason); + validate_all_mandatory_present() raises on a silent hole. + CI tests assert the sections and the verdict rule table. + +Layering: ``analytics`` is downstream of ``factors``, so importing +``factors.spec`` here is fine; ``FactorSpec`` itself lives in ``factors/`` +because the ``Factor`` class must hold it (invariant #3, 分层解耦). + +``StandardFactorEvaluator`` (``standard.py``) fills the 8 sections; the +vectorized eval-IR it reduces lives in ``ir.py`` (design §8). +""" + +from analytics.eval.config import EvalConfig +from analytics.eval.evaluator import EvalIR, FactorEvaluator +from analytics.eval.ir import EvalContext, StandardEvalIR, build_eval_ir +from analytics.eval.report import ( + SCHEMA_VERSION, + FactorEvalReport, + extract_verdict_inputs, +) +from analytics.eval.standard import StandardFactorEvaluator +from analytics.eval.sections import ( + MANDATORY_SECTIONS, + VERDICT_KEYS, + Section, + SectionLike, + Skipped, +) +from analytics.eval.verdict import ( + ADOPT, + AXIS_FAIL, + AXIS_INSUFFICIENT_DATA, + AXIS_NAMES, + AXIS_NOT_ASSESSED, + AXIS_PASS, + AXIS_VERDICTS, + INSUFFICIENT_DATA, + REJECT, + VERDICTS, + WATCH, + AxisVerdict, + VerdictInputs, + VerdictResult, + VerdictThresholds, + decide_verdict, +) + +__all__ = [ + "ADOPT", + "AXIS_FAIL", + "AXIS_INSUFFICIENT_DATA", + "AXIS_NAMES", + "AXIS_NOT_ASSESSED", + "AXIS_PASS", + "AXIS_VERDICTS", + "INSUFFICIENT_DATA", + "MANDATORY_SECTIONS", + "REJECT", + "SCHEMA_VERSION", + "VERDICTS", + "VERDICT_KEYS", + "WATCH", + "AxisVerdict", + "EvalConfig", + "EvalContext", + "EvalIR", + "FactorEvalReport", + "FactorEvaluator", + "Section", + "SectionLike", + "Skipped", + "StandardEvalIR", + "StandardFactorEvaluator", + "VerdictInputs", + "VerdictResult", + "VerdictThresholds", + "build_eval_ir", + "decide_verdict", + "extract_verdict_inputs", +] diff --git a/analytics/eval/config.py b/analytics/eval/config.py new file mode 100644 index 0000000..c9171c4 --- /dev/null +++ b/analytics/eval/config.py @@ -0,0 +1,288 @@ +"""``EvalConfig``: the PER-RUN half of the factor-evaluation contract. + +Where :class:`~factors.spec.FactorSpec` describes the factor (and never +changes), ``EvalConfig`` describes ONE evaluation of it: the window, the +universe, what was neutralized away — and the honesty flags the project has +learned to demand (``is_exploratory`` / ``post_hoc_selected`` / ``tuned`` / +``n_factors_screened``). + +Together they are the complete provenance an evaluator requires (design doc +``tmp/design/factor_eval_contract_v0.1.md`` §3). The validators here are +enforcement layer #1: a run that cannot state its honesty flags coherently +cannot be configured at all. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +from analytics.eval.verdict import VerdictThresholds + +# Tolerance for spotting the base (multiplier 1.0) cost scenario among floats. +_BASE_COST = 1.0 +_COST_TOL = 1e-9 + + +@dataclass(frozen=True) +class EvalConfig: + """Parameters + honesty declarations of ONE factor evaluation (immutable). + + Attributes + ---------- + universe : the universe identifier (e.g. an index code) being evaluated. + universe_is_pit : whether the constituents are point-in-time. Declaring + False is allowed but must be EXPLICIT — it is a survivorship caveat. + start, end : the evaluation window. + is_exploratory : an exploratory study, not a return claim. + post_hoc_selected : was this factor picked AFTER seeing results on (some of) + this data? If so it cannot be a confirmation — see the validator. + rebalance : rebalance frequency label. Defaults to "daily" (user ruling: the + project's default rebalance frequency). NOTE this label is not + decoration — an evaluator resolves the forward-return horizon and the + annualization on the panel's OWN grid and checks the supplied spacing + against this label, and the verdict's sample gate reads a CALENDAR span + and an EFFECTIVE sample size precisely because a raw count means + something different at each frequency (design §6, v0.3). + n_quantiles : number of factor buckets (>= 2). + long_short : (top, bottom) bucket labels. Defaults to (n_quantiles, 1). NOTE + this is a SYNTHETIC long-only leg difference, not a dollar-neutral + executed portfolio — the report must say so (P-I5d/P-I5e lesson). + cost_scenarios : fee multipliers. MUST contain 1.0, the base anchor every + other scenario is read against ("the same trades at k x the fee"). + limit_feasibility : apply the raw stk_limit up/down execution gate (I5b). + capacity_notional, max_participation_rate : the I5f capacity diagnostic. + oos_split : train/test split date; None means the report must state + explicitly that NO out-of-sample split was done. + independent_cells : cells declared genuinely independent of factor + screening. Independence is a HUMAN declaration — the machine cannot know + which data took part in screening. Overlap with the screening window + cannot be checked here (this object does not know it), so an evaluator + that can check it should; otherwise the report carries the caveat. + winsorize, standardize, neutralization, industry_level : what style + exposure was stripped before evaluating. + tuned : whether any parameter was tuned. Default False = "not tuned". + n_factors_screened : multiple-testing background (how many factors were + looked at to find this one). + data_snapshot_id : data/cache version, for reproducibility. + success_criteria : the PRE-REGISTERED verdict bar (design §6, v0.6). A frozen + :class:`~analytics.eval.verdict.VerdictThresholds` declared HERE, before + ``evaluate`` runs, IS pre-registered by construction: you cannot tune the + bar after seeing the result without producing a NEW report that visibly + declares different criteria (the report stamps ``criteria_source``). None + means the documented global default is used. A supplied object must pass + exactly the same ``VerdictThresholds.__post_init__`` validation — it is not + weakened for being per-run. + """ + + universe: str + universe_is_pit: bool + start: str + end: str + is_exploratory: bool + post_hoc_selected: bool + rebalance: str = "daily" + n_quantiles: int = 5 + long_short: tuple[int, int] | None = None + cost_scenarios: tuple[float, ...] = (1.0, 2.0, 4.0) + limit_feasibility: bool = True + capacity_notional: float | None = None + max_participation_rate: float = 0.05 + oos_split: str | None = None + independent_cells: tuple[object, ...] = () + winsorize: str | None = "mad" + standardize: str | None = "zscore" + neutralization: tuple[str, ...] = ("industry", "size") + industry_level: str = "L1" + tuned: bool = False + n_factors_screened: int | None = None + data_snapshot_id: str | None = None + success_criteria: VerdictThresholds | None = None + + def __post_init__(self) -> None: + self._check_window() + self._check_honesty() + self._check_quantiles() + self._check_costs() + self._check_capacity() + self._check_declared_sequences() + self._check_success_criteria() + + # -- validators (enforcement layer #1) -------------------------------- + + def _check_window(self) -> None: + for field_name in ("universe", "start", "end", "rebalance"): + value = getattr(self, field_name) + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"EvalConfig.{field_name} must be a non-empty string; got " + f"{value!r}." + ) + if not isinstance(self.universe_is_pit, bool): + raise ValueError( + f"EvalConfig.universe_is_pit must be a bool (a non-PIT universe " + f"must be admitted explicitly — survivorship); got " + f"{self.universe_is_pit!r}." + ) + + def _check_honesty(self) -> None: + for field_name in ("is_exploratory", "post_hoc_selected", "tuned"): + value = getattr(self, field_name) + if not isinstance(value, bool): + raise ValueError( + f"EvalConfig.{field_name} must be a bool; got {value!r}." + ) + if self.post_hoc_selected and not self.is_exploratory: + raise ValueError( + "EvalConfig: post_hoc_selected=True requires is_exploratory=True. " + "A factor chosen after seeing results on this data cannot be " + "reported as a confirmation (P3-6 lesson)." + ) + screened = self.n_factors_screened + if screened is not None and ( + isinstance(screened, bool) or not isinstance(screened, int) or screened < 1 + ): + raise ValueError( + f"EvalConfig.n_factors_screened must be None or a positive int " + f"(the multiple-testing background); got {screened!r}." + ) + + def _check_quantiles(self) -> None: + n = self.n_quantiles + if isinstance(n, bool) or not isinstance(n, int) or n < 2: + raise ValueError( + f"EvalConfig.n_quantiles must be an int >= 2; got {n!r}." + ) + if self.long_short is None: + # default: top bucket vs bottom bucket + object.__setattr__(self, "long_short", (n, 1)) + return + pair = tuple(self.long_short) + if len(pair) != 2 or any( + isinstance(x, bool) or not isinstance(x, int) for x in pair + ): + raise ValueError( + f"EvalConfig.long_short must be a (top, bottom) pair of ints; got " + f"{self.long_short!r}." + ) + top, bottom = pair + if not (1 <= top <= n) or not (1 <= bottom <= n): + raise ValueError( + f"EvalConfig.long_short=({top}, {bottom}) must name buckets within " + f"1..{n}." + ) + if top <= bottom: + # The legs stay in their natural (higher bucket, lower bucket) + # orientation so the hypothesis is applied in exactly ONE place: the + # verdict, via expected_ic_sign. A pre-flipped pair here would + # double-flip a -1 factor (e.g. low-vol) into a false Reject. + raise ValueError( + f"EvalConfig.long_short=({top}, {bottom}) must have top > bottom: " + f"the leg difference is always (higher bucket - lower bucket) and " + f"the DIRECTION comes from FactorSpec.expected_ic_sign, never from " + f"flipping the legs here." + ) + object.__setattr__(self, "long_short", pair) + + def _check_costs(self) -> None: + scenarios = self.cost_scenarios + if isinstance(scenarios, str) or not isinstance(scenarios, Sequence): + raise ValueError( + f"EvalConfig.cost_scenarios must be a sequence of fee multipliers; " + f"got {scenarios!r}." + ) + # Guarded BEFORE the float() coercion, which is what makes this the worst + # of the bool holes: float(True) is 1.0, so a stray True would not merely + # sneak in — it would silently satisfy the mandatory 1.0 base anchor below + # and the run would report "base cost" scenarios that nobody declared. + bad_bools = [c for c in scenarios if isinstance(c, bool)] + if bad_bools: + raise ValueError( + f"EvalConfig.cost_scenarios entries must be numeric fee " + f"multipliers, never bool: float(True) is 1.0, so a stray True " + f"would masquerade as the mandatory 1.0 base anchor. Got " + f"{tuple(scenarios)!r}." + ) + normalized = tuple(float(c) for c in scenarios) + bad = [c for c in normalized if c <= 0] + if bad: + raise ValueError( + f"EvalConfig.cost_scenarios multipliers must be positive; got {bad}." + ) + if not any(abs(c - _BASE_COST) < _COST_TOL for c in normalized): + raise ValueError( + f"EvalConfig.cost_scenarios must contain the base anchor 1.0 — every " + f"other scenario is only readable as 'the same trades at k x the " + f"fee' (project rule); got {normalized}." + ) + object.__setattr__(self, "cost_scenarios", normalized) + + def _check_capacity(self) -> None: + # ``bool`` is an int subclass, so True sails through a bare numeric check + # and lands in a declared-float field (and in the exported record) — the + # same gotcha as ``expected_ic_sign=1.0``. Guarded explicitly, as in every + # other numeric validator here and in FactorSpec. + notional = self.capacity_notional + if notional is not None and ( + isinstance(notional, bool) + or not isinstance(notional, (int, float)) + or notional <= 0 + ): + raise ValueError( + f"EvalConfig.capacity_notional must be None or a positive number; " + f"got {notional!r}." + ) + rate = self.max_participation_rate + if ( + isinstance(rate, bool) + or not isinstance(rate, (int, float)) + or not (0 < rate <= 1) + ): + raise ValueError( + f"EvalConfig.max_participation_rate must be in (0, 1]; got {rate!r}." + ) + if not isinstance(self.limit_feasibility, bool): + raise ValueError( + f"EvalConfig.limit_feasibility must be a bool; got " + f"{self.limit_feasibility!r}." + ) + + def _check_success_criteria(self) -> None: + """The pre-registered bar, if supplied, must be a real VerdictThresholds. + + Its own ``__post_init__`` has already validated type/range/NaN/inf/the + 1.0-exclusive bounds — a per-run object is NOT weakened for being per-run. + This only rejects the wrong TYPE (e.g. a bare dict) rather than silently + ignoring it and falling back to the default bar, which would let a caller + THINK they pre-registered a bar that never took effect. + """ + criteria = self.success_criteria + if criteria is not None and not isinstance(criteria, VerdictThresholds): + raise ValueError( + f"EvalConfig.success_criteria must be None or a VerdictThresholds " + f"(the pre-registered verdict bar, already self-validating); got " + f"{type(criteria).__name__}. A dict/other would be silently ignored " + f"and the run would fall back to the default bar — a pre-registration " + f"that never took effect." + ) + + def _check_declared_sequences(self) -> None: + """Coerce the remaining declared sequences to tuples, like the others. + + ``cost_scenarios`` / ``long_short`` are already normalized above. Without + the same treatment here a caller handing in a LIST keeps a live reference + into a ``frozen=True`` config (mutate the list -> mutate the config) and + ``hash(cfg)`` raises — an immutable provenance record that is neither. + """ + for field_name in ("independent_cells", "neutralization"): + value = getattr(self, field_name) + # A bare string would silently become a tuple of single characters. + if isinstance(value, str) or not isinstance(value, Sequence): + raise ValueError( + f"EvalConfig.{field_name} must be a sequence (not a bare " + f"string); got {value!r}." + ) + object.__setattr__(self, field_name, tuple(value)) + + +__all__ = ["EvalConfig"] diff --git a/analytics/eval/evaluator.py b/analytics/eval/evaluator.py new file mode 100644 index 0000000..2a57b6b --- /dev/null +++ b/analytics/eval/evaluator.py @@ -0,0 +1,131 @@ +"""``FactorEvaluator``: the standard interface every factor evaluation walks. + +The mandatory report sections are ABSTRACT METHODS, and the flow is a template +method — so a custom evaluator may ADD sections but can never drop a mandatory +one (design ``tmp/design/factor_eval_contract_v0.1.md`` §5). Together with +:meth:`FactorEvalReport.validate_all_mandatory_present` that is the double lock: +the ABC stops you at class definition, the report validation stops you at +assembly. + +SCOPE (PR-A = the contract only): this module defines the interface, the fixed +section order and the eval-IR seam. ``StandardFactorEvaluator`` and the +vectorized ``build_ir`` are **PR-B's job** and are deliberately absent here. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Protocol + +import pandas as pd + +from analytics.eval.config import EvalConfig +from analytics.eval.report import FactorEvalReport +from analytics.eval.sections import MANDATORY_SECTIONS, Section, Skipped +from analytics.eval.verdict import VerdictThresholds +from factors.spec import FactorSpec + + +class EvalIR(Protocol): + """The four vectorized intermediates every mandatory metric derives from. + + Design §8: computing these ONCE and reducing them is what turns the report + from a per-period python loop into two ``groupby`` passes. **PR-B implements + the builder**; PR-A only fixes the shape so the reduction code and the + speed-up have a stable target. + + Attributes + ---------- + factor : (1) the processed factor panel ``F``, MultiIndex(date, symbol). + forward_returns : (2) ``R_h`` at the spec's horizon, MultiIndex(date, symbol), + computed at the analytics boundary — the factor layer never sees it. + ic : (3) the per-date IC series, indexed by date. + quantile_returns : (4) the per-(date, quantile) mean-return matrix. + """ + + factor: pd.Series + forward_returns: pd.Series + ic: pd.Series + quantile_returns: pd.DataFrame + + +class FactorEvaluator(ABC): + """Standard factor evaluation: 8 mandatory sections + a fixed flow. + + Each section returns a :class:`Section` or an explicit + :class:`Skipped` (with a reason) — never None, never a silent omission. + """ + + #: the fixed section order the template method calls (design §5). Aliased to + #: the report's mandatory set on purpose: ONE source of truth, so the caller + #: order and the presence check can never drift apart. + SECTION_ORDER: tuple[str, ...] = MANDATORY_SECTIONS + + @abstractmethod + def build_ir( + self, + factor_panel: pd.Series | pd.DataFrame, + spec: FactorSpec, + cfg: EvalConfig, + ctx: object | None = None, + ) -> EvalIR: + """Build the vectorized eval-IR once (design §8). **PR-B implements this.** + + ``ctx`` carries whatever a section needs beyond the factor panel (price + panel, universe, minute bars, ...); PR-B defines its shape. + """ + + @abstractmethod + def predictive_power(self, ir: EvalIR) -> Section | Skipped: + """Rank IC / ICIR / IC win rate / Newey-West t / IC decay.""" + + @abstractmethod + def return_risk(self, ir: EvalIR) -> Section | Skipped: + """Quantile NAV / long-short spread / monotonicity / Sharpe / maxDD / vol.""" + + @abstractmethod + def stability_cost(self, ir: EvalIR) -> Section | Skipped: + """Turnover / autocorrelation (half-life) / cost-sensitivity gradient.""" + + @abstractmethod + def purity(self, ir: EvalIR) -> Section | Skipped: + """Correlation with known factors / orthogonalized IC / post-neutralization / VIF.""" + + @abstractmethod + def oos_generalization(self, ir: EvalIR) -> Section | Skipped: + """Train/test sign consistency / independent-universe verdict (the project's core lesson).""" + + @abstractmethod + def execution_capacity(self, ir: EvalIR) -> Section | Skipped: + """Fill feasibility (price-limit gating, I5b) / capacity ratio (I5f).""" + + @abstractmethod + def data_coverage(self, ir: EvalIR) -> Section | Skipped: + """Coverage / dropped symbols / survivorship / NaN — and the sample size.""" + + @abstractmethod + def caveats(self, ir: EvalIR) -> Section | Skipped: + """Post-hoc / exploratory / sample size / multiple testing.""" + + def evaluate( + self, + factor_panel: pd.Series | pd.DataFrame, + spec: FactorSpec, + cfg: EvalConfig, + ctx: object | None = None, + thresholds: VerdictThresholds | None = None, + ) -> FactorEvalReport: + """Build the IR once, run the 8 sections IN ORDER, assemble, validate, verdict. + + The three contract steps stay visible on purpose: assemble -> + validate_all_mandatory_present (raises on a silently missing section) -> + with_verdict (a report may not be published without one). + """ + ir = self.build_ir(factor_panel, spec, cfg, ctx) + sections = [getattr(self, name)(ir) for name in self.SECTION_ORDER] + report = FactorEvalReport.assemble(spec, cfg, sections, thresholds=thresholds) + report.validate_all_mandatory_present() + return report.with_verdict(thresholds) + + +__all__ = ["EvalIR", "FactorEvaluator"] diff --git a/analytics/eval/figures.py b/analytics/eval/figures.py new file mode 100644 index 0000000..9aee10f --- /dev/null +++ b/analytics/eval/figures.py @@ -0,0 +1,482 @@ +"""Sell-side-research-style visual dashboard for a factor evaluation report. + +ONE PNG that renders a :class:`~analytics.eval.report.FactorEvalReport` (the frozen +section payloads + three-axis verdict) together with the two time series only the +:class:`~analytics.eval.ir.StandardEvalIR` carries (per-rebalance RankIC and the +quantile return matrix). Pure plotting: :func:`render_factor_dashboard` takes an +already-evaluated report + IR and writes the figure, so it is decoupled from the +run logic and unit-testable with a toy :class:`DashboardData` (mirroring +``qt.intraday_group_figures``). + +Design language references the Kaiyuan (开源证券) layered-backtest charts: a white +canvas, thin dark spines, no heavy gridlines, the SIGNATURE thin per-quantile NAV +curves plus a THICK dark-maroon long-short curve on a secondary axis, a dense +YYYYMMDD date axis, and a top multi-column legend. + +MANDATORY factor-definition panel (contract constraint): every dashboard renders a +"Factor definition" band sourced from the report's :class:`~factors.spec.FactorSpec` +— what the factor measures, its inputs, hypothesis sign, horizon / return basis, +family, and (when intraday) the minute execution block. A factor may not be shown +without stating how it is computed. + +matplotlib is imported lazily here and this module is deliberately NOT re-exported +from ``analytics.eval.__init__`` so importing the eval package never pulls a +plotting backend. +""" +from __future__ import annotations + +import textwrap +from dataclasses import dataclass +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") # headless: render to PNG, never open a display. +import matplotlib.pyplot as plt # noqa: E402 (must follow use("Agg")) +import pandas as pd # noqa: E402 +from matplotlib.gridspec import GridSpec # noqa: E402 + +from analytics.eval.report import FactorEvalReport # noqa: E402 +from analytics.eval.verdict import VerdictResult # noqa: E402 +from factors.spec import FactorSpec # noqa: E402 + +# -- palette (Kaiyuan-report inspired) ------------------------------------- +_INK = "#222222" +_GRID = "#E9E9E9" +_BAR_BLUE = "#4472C4" +_BAR_RED = "#C0392B" +_GOLD = "#E1A730" +_LS_MAROON = "#8B1A1A" # the signature long-short curve +#: ordered quantile palette Q1(low) -> QN(high): deep-blue -> gray -> red. +_Q_COLORS = ("#2E5B9C", "#6FA8DC", "#9AA0A6", "#E69138", "#CC0000") +_DEP_COLORS = {"Adopt": "#2E8B57", "Watch": "#D98C00", "Reject": "#C0392B"} +_AXIS_COLORS = { + "PASS": "#2E8B57", "FAIL": "#C0392B", + "NOT_ASSESSED": "#8A8A8A", "INSUFFICIENT_DATA": "#D98C00", +} +_DEF_BG = "#F4F6F9" # factor-definition band background + + +# --------------------------------------------------------------------------- # +# Normalized input (decoupled from report/IR internals -> unit-testable) +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class DashboardData: + """Everything the dashboard renders, pulled out of a report + its IR. + + Building this explicitly (rather than reaching into a report/IR inside every + panel) keeps :func:`_render` testable with toy data. + """ + + spec: FactorSpec + verdict: VerdictResult + payloads: dict[str, dict] # section name -> payload dict (empty for Skipped) + ic: pd.Series # per-rebalance RankIC (may be empty) + quantile_returns: pd.DataFrame # date x 1..n quantile returns (may be empty) + + @classmethod + def from_report(cls, report: FactorEvalReport, ir: object) -> "DashboardData": + payloads = { + s.name: dict(getattr(s, "payload", None) or {}) for s in report.sections + } + ic = getattr(ir, "ic", pd.Series(dtype=float)) + qr = getattr(ir, "quantile_returns", pd.DataFrame()) + return cls( + spec=report.spec, + verdict=report.require_verdict(), + payloads=payloads, + ic=ic if isinstance(ic, pd.Series) else pd.Series(dtype=float), + quantile_returns=qr if isinstance(qr, pd.DataFrame) else pd.DataFrame(), + ) + + +# --------------------------------------------------------------------------- # +# styling helpers +# --------------------------------------------------------------------------- # +def _apply_rc() -> None: + plt.rcParams.update({ + "font.family": "DejaVu Sans", + "axes.edgecolor": _INK, + "axes.linewidth": 0.8, + "figure.facecolor": "white", + "axes.facecolor": "white", + "savefig.facecolor": "white", + }) + + +def _style(ax, title: str | None = None) -> None: + ax.spines["top"].set_visible(False) + ax.set_axisbelow(True) + ax.grid(axis="y", color=_GRID, linewidth=0.7, zorder=0) + ax.tick_params(colors=_INK, labelsize=8, length=3) + if title: + ax.set_title(title, loc="left", fontsize=11.5, fontweight="bold", + color="#1a1a1a", pad=8) + + +def _date_axis(ax, index, n_ticks: int = 16) -> None: + idx = pd.DatetimeIndex(index) + if len(idx) == 0: + return + step = max(1, len(idx) // n_ticks) + pos = list(range(0, len(idx), step)) + ax.set_xticks([idx[p] for p in pos]) + ax.set_xticklabels([idx[p].strftime("%Y%m%d") for p in pos], + rotation=90, fontsize=6.8) + + +def _empty(ax, title: str, msg: str = "not available") -> None: + _style(ax, title) + ax.text(0.5, 0.5, msg, ha="center", va="center", fontsize=10, + color="#999999", transform=ax.transAxes) + ax.set_xticks([]) + ax.set_yticks([]) + + +def _qcols(qr: pd.DataFrame) -> list: + return sorted(qr.columns, key=lambda c: int(str(c))) + + +# --------------------------------------------------------------------------- # +# header + verdict chips + MANDATORY factor-definition band +# --------------------------------------------------------------------------- # +def _header_text(ax, data: DashboardData) -> None: + ax.axis("off") + cov = data.payloads.get("data_coverage", {}) + pp = data.payloads.get("predictive_power", {}) + cav = data.payloads.get("caveats", {}) + spec = data.spec + win = cav.get("window", "") + ax.text(0.0, 0.72, f"Factor Evaluation — {spec.factor_id}", + fontsize=17, fontweight="bold", color="#111111", transform=ax.transAxes) + n = cov.get("settled_rebalances") + neff = cov.get("effective_samples") + sub = f"{cav.get('universe', '')} · {win} · {cav.get('rebalance', '')} rebalance" + if n is not None: + sub += f" · N={int(n)} settled" + if isinstance(neff, (int, float)): + sub += f" (N_eff={neff:.0f})" + if cav.get("is_exploratory"): + sub += " · EXPLORATORY (cap: Watch)" + ax.text(0.0, 0.16, sub, fontsize=10, color="#444444", transform=ax.transAxes) + if pp: + ax.text(1.0, 0.72, + f"RankIC {pp.get('ic_mean', float('nan')):+.4f} " + f"ICIR {pp.get('ic_ir', float('nan')):+.3f} " + f"NW-t {pp.get('ic_nw_t', float('nan')):+.1f}", + fontsize=11, color="#111111", ha="right", transform=ax.transAxes) + lo, hi = pp.get("ic_ir_ci_low"), pp.get("ic_ir_ci_high") + if isinstance(lo, (int, float)) and isinstance(hi, (int, float)): + ax.text(1.0, 0.16, + f"ICIR 95% CI [{lo:+.3f}, {hi:+.3f}] " + f"win rate {pp.get('ic_win_rate', float('nan')):.1%}", + fontsize=10, color="#444444", ha="right", transform=ax.transAxes) + + +def _chips_row(ax, data: DashboardData) -> None: + ax.axis("off") + ax.axhline(0.97, color="#DDDDDD", lw=1.0) + v = data.verdict + axes_v = v.axes() + chips = [ + ("DEPLOYMENT", v.verdict, _DEP_COLORS.get(v.verdict, "#8A8A8A"), True), + ("Predictive", axes_v["predictive"].verdict.replace("_", " "), + _AXIS_COLORS.get(axes_v["predictive"].verdict, "#8A8A8A"), False), + ("Incremental", axes_v["incremental"].verdict.replace("_", " "), + _AXIS_COLORS.get(axes_v["incremental"].verdict, "#8A8A8A"), False), + ("Tradable", axes_v["tradable"].verdict.replace("_", " "), + _AXIS_COLORS.get(axes_v["tradable"].verdict, "#8A8A8A"), False), + ] + for (label, value, color, big), x in zip(chips, (0.02, 0.28, 0.52, 0.76)): + ax.text(x, 0.62, label, fontsize=9.5, color="#666666", transform=ax.transAxes) + ax.text(x, 0.12, f" {value} ", fontsize=15 if big else 12, fontweight="bold", + color="white", transform=ax.transAxes, + bbox=dict(boxstyle="round,pad=0.34", fc=color, ec="none")) + + +def _definition_meta_line(spec: FactorSpec) -> str: + """The structured one-line factor-definition metadata (pure -> unit-testable).""" + return ( + f"inputs: {', '.join(spec.input_fields)} · " + f"expected sign: {spec.expected_ic_sign:+d} · " + f"horizon: {spec.forward_return_horizon} ({spec.return_basis}) · " + f"family: {spec.family or '—'} · " + f"min-history: {spec.min_history_bars} bars · " + f"price: {spec.price_adjust} · " + f"intraday: {spec.is_intraday}" + ) + + +def _definition_block_line(spec: FactorSpec) -> str | None: + """The intraday minute-execution block line, or None for a daily factor.""" + if not spec.is_intraday: + return None + return ( + f"minute block — cutoff: {spec.decision_cutoff} · lag: {spec.data_lag}" + f" · session-open: {spec.session_open} · exec: {spec.execution_model}" + f" · window: {spec.execution_window}" + ) + + +def _definition_band(ax, data: DashboardData) -> None: + """MANDATORY panel: how the factor is computed, from the FactorSpec. + + Stacked vertically so nothing overlaps: header + id on one line, the full + computation description wrapped full-width below it, then the structured + metadata line (and the minute block for an intraday factor). + """ + ax.axis("off") + ax.add_patch(plt.Rectangle((0, 0), 1, 1, transform=ax.transAxes, + facecolor=_DEF_BG, edgecolor="#D8DEE7", lw=1.0, + zorder=0)) + spec = data.spec + ax.text(0.012, 0.90, "FACTOR DEFINITION", fontsize=10.5, fontweight="bold", + color="#33475B", transform=ax.transAxes, va="top") + ax.text(0.185, 0.90, f"{spec.factor_id} (v{spec.version})", fontsize=9.5, + fontweight="bold", color="#111111", transform=ax.transAxes, va="top", + family="DejaVu Sans Mono") + ax.text(0.012, 0.62, textwrap.fill(spec.description, width=150), fontsize=9.2, + color="#222222", transform=ax.transAxes, va="top") + block = _definition_block_line(spec) + meta_y = 0.20 if block is not None else 0.10 + ax.text(0.012, meta_y, _definition_meta_line(spec), fontsize=8.6, + color="#41505F", transform=ax.transAxes, va="bottom", + family="DejaVu Sans Mono") + if block is not None: + ax.text(0.012, 0.04, block, fontsize=8.4, color="#7A5C00", + transform=ax.transAxes, va="bottom", family="DejaVu Sans Mono") + + +# --------------------------------------------------------------------------- # +# data panels +# --------------------------------------------------------------------------- # +def _panel_hero_nav(ax, data: DashboardData) -> None: + qr = data.quantile_returns + if qr.empty: + _empty(ax, "Layered backtest — quantile NAV & long-short") + return + qcols = _qcols(qr) + nav = (1.0 + qr[qcols].fillna(0.0)).cumprod() + for i, q in enumerate(qcols): + tag = (" (low)" if i == 0 else " (high)" if i == len(qcols) - 1 else "") + ax.plot(nav.index, nav[q].values, color=_Q_COLORS[i % len(_Q_COLORS)], + lw=1.15, label=f"Q{q}{tag}") + ax.set_ylabel("Quantile NAV (long-only, equal-weight)", fontsize=9, color=_INK) + sign = data.spec.expected_ic_sign + _style(ax, f"Layered backtest — quantile NAV & long-short " + f"(Q1 lowest factor · aligned sign {sign:+d})") + # long Q1 / short Q_top when sign is -1 (else flip) -> GROSS aligned spread. + lo, hi = (qcols[0], qcols[-1]) if sign < 0 else (qcols[-1], qcols[0]) + ls = qr[lo].fillna(0.0) - qr[hi].fillna(0.0) + ls_nav = (1.0 + ls).cumprod() + ax2 = ax.twinx() + ax2.spines["top"].set_visible(False) + ax2.plot(ls_nav.index, ls_nav.values, color=_LS_MAROON, lw=2.3, + label=f"Long-Short Q{lo}−Q{hi} (gross, right)") + ax2.set_ylabel("Long-Short NAV (gross)", fontsize=9, color=_LS_MAROON) + ax2.tick_params(colors=_LS_MAROON, labelsize=8, length=3) + _date_axis(ax, nav.index) + h1, l1 = ax.get_legend_handles_labels() + h2, l2 = ax2.get_legend_handles_labels() + ax.legend(h1 + h2, l1 + l2, loc="upper left", ncol=3, fontsize=8.5, + frameon=False, handlelength=1.6, columnspacing=1.4) + + +def _panel_ic(ax, data: DashboardData) -> None: + ic = data.ic.dropna() + if ic.empty: + _empty(ax, "Information coefficient") + return + ic.index = pd.to_datetime(ic.index) + ax.bar(ic.index, ic.values, width=2.0, color="#B7C7E2", zorder=1, + label="RankIC per rebalance") + ax.axhline(0, color=_INK, lw=0.7) + ax.set_ylabel("RankIC", fontsize=9, color=_INK) + _style(ax, "Information coefficient — per-period & cumulative") + ax2 = ax.twinx() + ax2.spines["top"].set_visible(False) + ax2.plot(ic.index, ic.cumsum().values, color=_LS_MAROON, lw=2.0, + label="Cumulative RankIC (right)") + ax2.set_ylabel("Cumulative RankIC", fontsize=9, color=_LS_MAROON) + ax2.tick_params(colors=_LS_MAROON, labelsize=8, length=3) + _date_axis(ax, ic.index) + h1, l1 = ax.get_legend_handles_labels() + h2, l2 = ax2.get_legend_handles_labels() + ax.legend(h1 + h2, l1 + l2, loc="lower left", ncol=2, fontsize=8.5, frameon=False) + + +def _panel_quantile_bar(ax, data: DashboardData) -> None: + fnav = data.payloads.get("return_risk", {}).get("quantile_final_nav") + if not fnav: + _empty(ax, "Quantile final NAV (monotonicity)") + return + qs = sorted(fnav, key=lambda c: int(str(c))) + vals = [fnav[q] for q in qs] + colors = [_Q_COLORS[i % len(_Q_COLORS)] for i in range(len(qs))] + ax.bar([f"Q{q}" for q in qs], vals, color=colors, zorder=2, width=0.66) + ax.axhline(1.0, color=_INK, lw=0.7, ls="--") + for i, val in enumerate(vals): + ax.text(i, val + 0.01, f"{val:.2f}", ha="center", va="bottom", fontsize=8, + color=_INK) + ax.set_ylabel("Final NAV", fontsize=9) + _style(ax, "Quantile final NAV (monotonicity)") + + +def _panel_oos(ax, data: DashboardData) -> None: + oos = data.payloads.get("oos_generalization", {}) + keys = [("train_ic_mean", "Train IC"), ("test_ic_mean", "Test IC"), + ("holdout_subperiod_1_ic_mean", "Holdout-1"), + ("holdout_subperiod_2_ic_mean", "Holdout-2")] + labels = [lbl for k, lbl in keys if oos.get(k) is not None] + vals = [oos[k] for k, _ in keys if oos.get(k) is not None] + if not vals: + _empty(ax, "Out-of-sample sign consistency") + return + colors = [_BAR_BLUE if (v or 0) < 0 else _BAR_RED for v in vals] + ax.bar(labels, vals, color=colors, zorder=2, width=0.62) + ax.axhline(0, color=_INK, lw=0.7) + for i, val in enumerate(vals): + ax.text(i, val, f"{val:+.4f}", ha="center", + va="top" if val < 0 else "bottom", fontsize=7.5, color=_INK) + ax.set_ylabel("mean RankIC", fontsize=9) + ax.tick_params(axis="x", labelsize=8) + _style(ax, "Out-of-sample sign consistency") + + +def _panel_decay(ax, data: DashboardData) -> None: + pp = data.payloads.get("predictive_power", {}) + lags = [1, 2, 3, 5] + pairs = [(k, pp.get(f"ic_decay_stale_lag_{k}_mean")) for k in lags] + pairs = [(k, v) for k, v in pairs if v is not None] + if not pairs: + _empty(ax, "IC decay (signal persistence)") + return + xs, vals = [k for k, _ in pairs], [v for _, v in pairs] + ax.plot(xs, vals, marker="o", color=_BAR_BLUE, lw=1.8, ms=6, zorder=2) + ax.axhline(0, color=_INK, lw=0.7) + ax.set_xlabel("stale lag (periods)", fontsize=9) + ax.set_ylabel("mean RankIC", fontsize=9) + ax.set_xticks(xs) + _style(ax, "IC decay (signal persistence)") + + +def _panel_purity(ax, data: DashboardData) -> None: + purity = data.payloads.get("purity", {}) + corr = purity.get("mean_rank_corr_with_anchor") + if not corr: + _empty(ax, "Purity — corr with book", + "no known-factor book\n(Incremental NOT_ASSESSED)") + return + names = list(corr) + vals = [corr[n] for n in names] + colors = [_BAR_RED if v > 0 else _BAR_BLUE for v in vals] + ax.barh(names, vals, color=colors, zorder=2, height=0.55) + ax.axvline(0, color=_INK, lw=0.7) + lo, hi = min(vals + [0.0]), max(vals + [0.0]) + pad = 0.10 * max(hi - lo, 0.1) + ax.set_xlim(lo - pad - 0.12, hi + pad + 0.12) + for i, val in enumerate(vals): + ax.text(val + (pad * 0.3 if val >= 0 else -pad * 0.3), i, f"{val:+.3f}", + va="center", ha="left" if val >= 0 else "right", fontsize=8, + color=_INK) + incr = purity.get("incremental_ic_ir") + ax.set_xlabel("mean rank-corr with book factor", fontsize=9) + ax.tick_params(axis="y", labelsize=8.5) + title = "Purity — corr w/ book" + if isinstance(incr, (int, float)): + title += f" (incr ICIR {incr:+.3f})" + _style(ax, title) + + +def _panel_cost(ax, data: DashboardData) -> None: + sc = data.payloads.get("stability_cost", {}) + drag = sc.get("annualized_cost_drag_by_scenario") + if not drag: + _empty(ax, "Cost sensitivity") + return + ks = sorted(drag, key=lambda c: float(str(c))) + vals = [drag[k] for k in ks] + labels = [f"{float(k):.0f}x fee" for k in ks] + ax.bar(labels, vals, color=[_GOLD, "#D98C00", _BAR_RED][: len(ks)], zorder=2, + width=0.6) + for i, val in enumerate(vals): + ax.text(i, val, f"{val:.1%}", ha="center", va="bottom", fontsize=8.5, + color=_INK) + ax.set_ylabel("annualized cost drag", fontsize=9) + ax.set_ylim(0, max(vals) * 1.18 if max(vals) > 0 else 1.0) + _style(ax, "Cost sensitivity (turnover × fee)") + + +def _panel_coverage(ax, data: DashboardData) -> None: + ax.axis("off") + cov = data.payloads.get("data_coverage", {}) + sc = data.payloads.get("stability_cost", {}) + ac = sc.get("factor_rank_autocorr_by_lag", {}) or {} + # payload dict keys may be int (live report) or str (round-tripped JSON). + autocorr_1 = ac.get(1, ac.get("1")) + ax.text(0.0, 0.95, "Coverage & turnover", fontsize=11.5, fontweight="bold", + color="#1a1a1a", transform=ax.transAxes, va="top") + rows = [ + ("symbols evaluated", cov.get("symbols_evaluated")), + ("cross-section med", cov.get("cross_section_size_median")), + ("factor NaN rate", cov.get("factor_nan_rate")), + ("rank autocorr(1)", autocorr_1), + ("half-life (periods)", sc.get("half_life_periods")), + ("PIT universe", cov.get("universe_is_pit")), + ] + + def _fmt(k, v): + if v is None: + return f"{k:<18}: —" + if k == "factor NaN rate" and isinstance(v, (int, float)): + return f"{k:<18}: {v:.1%}" + if isinstance(v, float): + return f"{k:<18}: {v:.3f}" if abs(v) < 100 else f"{k:<18}: {v:.0f}" + return f"{k:<18}: {v}" + + ax.text(0.0, 0.72, "\n".join(_fmt(k, v) for k, v in rows), fontsize=9.5, + color="#333333", family="DejaVu Sans Mono", transform=ax.transAxes, + va="top") + + +# --------------------------------------------------------------------------- # +# public API +# --------------------------------------------------------------------------- # +def _render(data: DashboardData, out_path: str | Path) -> Path: + _apply_rc() + out = Path(out_path) + out.parent.mkdir(parents=True, exist_ok=True) + fig = plt.figure(figsize=(15, 21.5)) + gs = GridSpec(7, 3, figure=fig, + height_ratios=[0.30, 0.38, 0.80, 1.5, 1.1, 1.0, 1.0], + hspace=0.62, wspace=0.28, + left=0.06, right=0.945, top=0.975, bottom=0.04) + _header_text(fig.add_subplot(gs[0, :]), data) + _chips_row(fig.add_subplot(gs[1, :]), data) + _definition_band(fig.add_subplot(gs[2, :]), data) + _panel_hero_nav(fig.add_subplot(gs[3, :]), data) + _panel_ic(fig.add_subplot(gs[4, :]), data) + _panel_quantile_bar(fig.add_subplot(gs[5, 0]), data) + _panel_oos(fig.add_subplot(gs[5, 1]), data) + _panel_decay(fig.add_subplot(gs[5, 2]), data) + _panel_purity(fig.add_subplot(gs[6, 0]), data) + _panel_cost(fig.add_subplot(gs[6, 1]), data) + _panel_coverage(fig.add_subplot(gs[6, 2]), data) + fig.savefig(out, dpi=130) + plt.close(fig) + return out + + +def render_factor_dashboard( + report: FactorEvalReport, ir: object, out_path: str | Path +) -> Path: + """Render ``report`` (+ its ``ir`` series) to a one-PNG dashboard at ``out_path``. + + ``ir`` is the :class:`~analytics.eval.ir.StandardEvalIR` produced alongside the + report (see :meth:`StandardFactorEvaluator.evaluate_with_ir`); only its ``ic`` + and ``quantile_returns`` are read. + """ + return _render(DashboardData.from_report(report, ir), out_path) + + +__all__ = ["DashboardData", "render_factor_dashboard"] diff --git a/analytics/eval/ir.py b/analytics/eval/ir.py new file mode 100644 index 0000000..f758f88 --- /dev/null +++ b/analytics/eval/ir.py @@ -0,0 +1,756 @@ +"""``build_eval_ir``: the four VECTORIZED intermediates every metric reduces from. + +Design ``tmp/design/factor_eval_contract_v0.1.md`` §8. The user's actual pain is +"分钟信号生成的因子的 ICIR 分层回测太慢", and the root cause is a per-rebalance +Python loop: for every date, build a frame, dropna, correlate, bucket, average. +This module computes the same four objects with **groupby/bincount passes over +the whole panel** — the number of Python-level iterations no longer grows with +the number of dates: + + 1. ``factor`` F, MultiIndex(date, symbol) — the processed input + 2. ``forward_returns`` R_h at the spec's horizon, same index + 3. ``ic`` Series[date] — ONE grouped reduction + 4. ``quantile_returns`` DataFrame[date, quantile] — ONE grouped reduction + +Everything a mandatory section needs is then a cheap reduction of these four +(plus the metadata carried alongside), which is what makes the report interface +stable while the engine underneath is free to change (design §10: 先钉接口,后换 +引擎). + +PIT BOUNDARY (invariant #1). Forward returns are computed HERE, at the analytics +boundary — ``analytics/factor.py`` is the only place allowed to look into the +future, and this module reuses it rather than re-deriving it. ``factors/`` never +sees an ``R_h``, and this module is downstream of ``factors`` (it imports +``factors.spec``, never the reverse). + +TWO CONVENTIONS WORTH KNOWING BEFORE READING A REPORT + * **One row of F = one evaluation period.** The IR does NOT resample: it + evaluates the grid it is handed. But ``EvalConfig.rebalance`` is not merely + a label either — a daily panel declared "monthly" would report ~250 settled + rebalances instead of ~12 and walk straight through + ``VerdictThresholds.min_rebalances``, the gate whose entire job is deciding + INSUFFICIENT-DATA. So :func:`_check_rebalance_grid` compares the observed + spacing against the declared frequency and **raises** on a mismatch (it does + not resample — see that function for why). + One row = one period is also why ``FactorSpec.forward_return_horizon`` + (documented as "the horizon IN EVALUATION PERIODS") is resolved on F's OWN + grid: a monthly factor over a daily price panel is scored on its + next-PERIOD return, not its next-TRADING-DAY return. See + :func:`_resolve_forward_returns`. + * **Buckets are formed from the FACTOR cross-section alone**, never from the + (factor, forward-return) intersection. Dropping a name because its future + return is missing would let tomorrow's data decide today's bucket. The + per-bucket missing-return rate is disclosed instead. This is a deliberate + difference from ``analytics.factor.quantile_returns`` (which buckets the + intersection); see :func:`assign_quantile_buckets`. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd + +from analytics.eval.config import EvalConfig +from analytics.factor import forward_returns as _forward_returns +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL +from factors.spec import FactorSpec + +#: Staleness lags (in evaluation periods) the IC-decay / factor-autocorrelation +#: curves are evaluated at. A loop over THESE is a loop over 4 constants — each +#: iteration is still one vectorized pass over the whole panel — which is not the +#: per-period loop design §8 is about. +DECAY_LAGS: tuple[int, ...] = (1, 2, 3, 5) + +#: Fallback annualization when ``EvalConfig.rebalance`` is not a label we know. +#: Disclosed in the report rather than silently assumed. +PERIODS_PER_YEAR: dict[str, int] = { + "daily": 252, + "weekly": 52, + "biweekly": 26, + "monthly": 12, + "quarterly": 4, + "annual": 1, + "yearly": 1, +} +DEFAULT_PERIODS_PER_YEAR = 252 + +#: Expected spacing, in CALENDAR days, between consecutive evaluation periods for +#: each recognized rebalance label. A trading-day grid has a median gap of 1.0 +#: (weekends sit in the tail, not the median); a month averages 365.25/12. +REBALANCE_SPACING_DAYS: dict[str, float] = { + "daily": 1.0, + "weekly": 7.0, + "biweekly": 14.0, + "monthly": 365.25 / 12.0, + "quarterly": 365.25 / 4.0, + "annual": 365.25, + "yearly": 365.25, +} + +#: A declared grid is accepted when the observed median spacing lands inside +#: ``[expected / TOL, expected * TOL]``. Deliberately COARSE: this exists to catch +#: the ORDER-OF-MAGNITUDE lie (a daily panel declared "monthly" is off by ~30x), +#: not a fine distinction (weekly vs biweekly is 2x and passes). The gate it +#: protects — ``min_rebalances`` — only cares about order of magnitude. +REBALANCE_SPACING_TOLERANCE = 2.0 + + +@dataclass(frozen=True) +class EvalContext: + """Everything :func:`build_eval_ir` needs BEYOND the factor panel. + + The evaluator contract leaves ``ctx`` to PR-B (``evaluator.py``: "``ctx`` + carries whatever a section needs beyond the factor panel; PR-B defines its + shape"). This is that shape. Every field is optional, and an absent field + makes the section that needed it say so — it never makes a section invent a + number. + + Attributes + ---------- + price_panel : the CANONICAL market panel (MultiIndex(date, symbol), + front-adjusted per ``FactorSpec.price_adjust``), used to derive ``R_h`` for + a ``close_to_close`` factor. Only ``close`` is read, but the whole core + column set is required: ``R_h`` comes from + ``analytics.factor.forward_returns``, which validates the panel through + the shared schema. Reusing the project's ONE forward-return boundary is + worth carrying its contract. + forward_returns : a PRE-COMPUTED ``R_h``, for bases this module cannot derive + itself (``exec_to_exec``: the holding period runs exec(T) -> exec(T_next) + and only the minute tail machinery knows the execution anchors). + known_factors : MultiIndex(date, symbol) frame of ALREADY-PROCESSED anchor + factors for the purity section. Absent -> purity is Skipped, never faked. + universe_symbols : the universe the factor was supposed to cover, so + data_coverage can report what was DROPPED instead of guessing. + fee_rate : the base one-way fee rate the cost scenarios multiply. + ``EvalConfig.cost_scenarios`` are MULTIPLIERS and the contract carries no + base rate, so it lives here. 0.001 is the project's standing base (I5d). + execution_capacity : execution facts MEASURED ELSEWHERE (I5b fill feasibility + / I5f capacity) — this evaluator does not run the backtest engine, so it + reports what it is handed and Skips when handed nothing. Recognized keys: + ``tradable`` (bool), ``capacity_sufficient`` (bool), plus any diagnostics + to render. The provenance is stamped into the section: these numbers are + never produced here. + """ + + price_panel: pd.DataFrame | None = None + forward_returns: pd.Series | None = None + known_factors: pd.DataFrame | None = None + universe_symbols: tuple[str, ...] = () + fee_rate: float = 0.001 + execution_capacity: Mapping[str, object] | None = None + + def __post_init__(self) -> None: + rate = self.fee_rate + if isinstance(rate, bool) or not isinstance(rate, (int, float)) or rate < 0: + raise ValueError( + f"EvalContext.fee_rate must be a non-negative number (the base " + f"one-way fee the EvalConfig.cost_scenarios multiply); got {rate!r}." + ) + object.__setattr__(self, "universe_symbols", tuple(self.universe_symbols)) + + +@dataclass(frozen=True) +class StandardEvalIR: + """The design §8 four objects + the metadata the sections and verdict need. + + Satisfies the frozen ``analytics.eval.evaluator.EvalIR`` Protocol (``factor`` + / ``forward_returns`` / ``ic`` / ``quantile_returns``) and carries the rest as + plain, already-reduced facts so no section has to touch the panel again. + """ + + # -- the Protocol's four ------------------------------------------------ + factor: pd.Series + forward_returns: pd.Series + ic: pd.Series + quantile_returns: pd.DataFrame + + # -- provenance --------------------------------------------------------- + spec: FactorSpec + cfg: EvalConfig + ctx: EvalContext + + # -- derived, computed once -------------------------------------------- + #: linear (Pearson) IC — secondary to the rank IC (design §A). + ic_pearson: pd.Series = field(default_factory=lambda: pd.Series(dtype=float)) + #: 1..n_quantiles bucket label per (date, symbol) with a valid factor value. + quantile_labels: pd.Series = field(default_factory=lambda: pd.Series(dtype="int64")) + #: per-(date, bucket) turnover, sum_i |w_i(t) - w_i(t-1)|. Computed with the + #: rest of the IR so return_risk and stability_cost reduce it instead of each + #: rebuilding it (turnover x fee_rate = the period's cost). + quantile_turnover: pd.DataFrame = field(default_factory=pd.DataFrame) + #: per-date count of (factor, forward-return) pairs the IC actually used. + cross_section_size: pd.Series = field(default_factory=lambda: pd.Series(dtype=float)) + #: per-date count of non-NaN factor values (the bucketed cross-section). + factor_count: pd.Series = field(default_factory=lambda: pd.Series(dtype=float)) + #: date -> the date the period's forward return is REALIZED on (t+h), NaT at + #: the tail. The OOS split reads THIS, never the signal date (P3-3's fix). + realized_date: pd.Series = field(default_factory=lambda: pd.Series(dtype=object)) + #: sorted unique dates of F (one per evaluation period). + dates: pd.Index = field(default_factory=lambda: pd.Index([], name=DATE_LEVEL)) + #: how R_h was obtained — disclosed, never guessed. + forward_return_source: str = "" + #: observed median spacing between evaluation periods, in calendar days. + median_period_gap_days: float = float("nan") + #: outcome of checking the supplied grid against ``cfg.rebalance`` (a mismatch + #: raises in the builder; this records OK / NOT CHECKED and why). + rebalance_grid_check: str = "" + #: rows within a symbol's declared ``min_history_bars`` warm-up. + warmup_mask: pd.Series = field(default_factory=lambda: pd.Series(dtype=bool)) + + @property + def settled_rebalances(self) -> int: + """Periods that produced usable evidence = a FINITE IC. + + A period with no realized forward return (the last h rows) or with a + degenerate cross-section yields NaN and is NOT counted: the verdict's + sample-size gate must not be inflated by periods that never settled. + """ + return int(self.ic.notna().sum()) + + @property + def n_rebalances(self) -> int: + """Evaluation periods on the supplied grid (settled or not).""" + return int(len(self.dates)) + + @property + def periods_per_year(self) -> int: + return PERIODS_PER_YEAR.get(self.cfg.rebalance.strip().lower(), DEFAULT_PERIODS_PER_YEAR) + + @property + def periods_per_year_is_default(self) -> bool: + """True when ``rebalance`` was not a recognized label (-> disclose it).""" + return self.cfg.rebalance.strip().lower() not in PERIODS_PER_YEAR + + +# -- vectorized primitives ------------------------------------------------- + + +def _unique_dates(index: pd.Index) -> pd.Index: + return pd.Index( + pd.unique(index.get_level_values(DATE_LEVEL)), name=DATE_LEVEL + ).sort_values() + + +def _finite(frame: pd.DataFrame) -> pd.DataFrame: + """Drop rows with a NaN or an infinity, like ``analytics.factor`` does. + + +/-inf is treated as missing rather than as an extreme value: a polluted + return would otherwise silently dominate a correlation or a bucket mean. + """ + return frame.replace([np.inf, -np.inf], np.nan).dropna() + + +def cross_section_corr( + left: pd.Series, + right: pd.Series, + *, + rank: bool = True, + dates: pd.Index | None = None, +) -> pd.Series: + """Per-date cross-sectional correlation of two aligned panels — NO date loop. + + Semantically identical to calling ``analytics.factor.compute_ic`` (drop + non-finite pairs within the date; NaN when fewer than 2 pairs survive or + either side has zero variance; Spearman = Pearson of the average-tie ranks), + but computed as a handful of whole-panel passes: ONE grouped rank, then + ``np.bincount`` reductions over the date codes. + + Numerically it demeans first (two-pass) instead of using the + sum-of-squares shortcut, so a large-mean cross-section cannot lose the + correlation to catastrophic cancellation. + + Parameters + ---------- + left, right : aligned MultiIndex(date, symbol) Series. + rank : True -> rank IC (Spearman, the primary; design §A); False -> Pearson. + dates : the full date index to report on. Dates with no usable pair still + appear, as NaN — an absent period must not silently vanish from the + series the ICIR and the Newey-West t are computed over. + + Returns + ------- + Series indexed by ``dates`` (or the union index's dates), one value per date. + """ + frame = pd.DataFrame({"l": left, "r": right}) + all_dates = _unique_dates(frame.index) if dates is None else dates + empty = pd.Series(np.nan, index=all_dates, dtype=float, name="ic") + valid = _finite(frame) + if valid.empty: + return empty + + if rank: + by_date = valid.groupby(level=DATE_LEVEL, sort=False) + x = by_date["l"].rank().to_numpy(dtype=float) + y = by_date["r"].rank().to_numpy(dtype=float) + else: + x = valid["l"].to_numpy(dtype=float) + y = valid["r"].to_numpy(dtype=float) + + codes, uniques = pd.factorize(valid.index.get_level_values(DATE_LEVEL), sort=True) + k = len(uniques) + n = np.bincount(codes, minlength=k).astype(float) + with np.errstate(invalid="ignore", divide="ignore"): + mx = np.bincount(codes, weights=x, minlength=k) / n + my = np.bincount(codes, weights=y, minlength=k) / n + xc = x - mx[codes] + yc = y - my[codes] + sxy = np.bincount(codes, weights=xc * yc, minlength=k) + sxx = np.bincount(codes, weights=xc * xc, minlength=k) + syy = np.bincount(codes, weights=yc * yc, minlength=k) + den = np.sqrt(sxx * syy) + # den > 0 already implies n >= 2 (a single point demeans to zero), but the + # count check is kept explicit: it is the documented contract, not a + # side effect of the algebra. + usable = (den > 0) & (n >= 2) + corr = np.where(usable, sxy / np.where(usable, den, 1.0), np.nan) + out = pd.Series( + np.clip(corr, -1.0, 1.0), index=pd.Index(uniques, name=DATE_LEVEL), name="ic" + ) + return out.reindex(all_dates) + + +def assign_quantile_buckets(factor: pd.Series, n_quantiles: int) -> pd.Series: + """EQUAL-COUNT rank buckets (1 = lowest factor) per date — NO date loop. + + Reproduces the project's canonical rule + (:func:`qt.intraday_groups.assign_quantile_buckets`, the I5d/I5e grouped + backtest) exactly, with the per-date ``sorted() + np.array_split`` replaced by + integer arithmetic on grouped ranks: + + * names are ordered by ``(factor ascending, symbol ascending)`` — the + ``rank(method="first")`` tie-break IS the symbol order because + :func:`build_eval_ir` sorts the panel first — and split BY POSITION, so + ties and degenerate cross-sections still assign deterministically where a + value cut could not; + * chunk sizes differ by at most one and the extra names go to the LOW + buckets (the ``np.array_split`` convention); + * fewer names than buckets simply leaves the high buckets empty. + + ⚠️ This deliberately differs from ``analytics.factor.quantile_returns``, which + ``pd.qcut``s the ranks (equivalent when ``n_quantiles`` divides the + cross-section, off by a name or two otherwise) AND buckets the (factor, + forward-return) intersection. Buckets here are formed from the FACTOR + cross-section ALONE: a name whose future return is missing must not be able to + change which bucket its neighbours land in. + """ + valid = _finite(factor.to_frame("f"))["f"] + if valid.empty: + return pd.Series([], index=factor.index[:0], dtype="int64", name="quantile") + + by_date = valid.groupby(level=DATE_LEVEL, sort=False) + ranks = by_date.rank(method="first").to_numpy(dtype=np.int64) # 1..n within date + sizes = by_date.transform("size").to_numpy(dtype=np.int64) + + # np.array_split(n, q): the first (n % q) chunks hold (n // q + 1) names, the + # rest hold (n // q). Invert that to rank -> chunk without materializing it. + base = sizes // n_quantiles + remainder = sizes % n_quantiles + head = remainder * (base + 1) # ranks 1..head live in the bigger chunks + in_head = ranks <= head + safe_base = np.where(base > 0, base, 1) # base == 0 <=> every rank is in head + labels = np.where( + in_head, + (ranks - 1) // (base + 1) + 1, + remainder + (ranks - 1 - head) // safe_base + 1, + ) + return pd.Series( + np.clip(labels, 1, n_quantiles), index=valid.index, name="quantile" + ) + + +def quantile_return_matrix( + fwd: pd.Series, labels: pd.Series, n_quantiles: int, dates: pd.Index +) -> pd.DataFrame: + """Mean forward return per (date, bucket) — ONE ``groupby([date, q]).mean()``. + + Returns a ``dates x 1..n_quantiles`` frame. A bucket with no realized return + on a date is NaN (an empty bucket, or every member's future missing) — never + a silent zero. + """ + columns = pd.Index(range(1, n_quantiles + 1), name="quantile") + pair = _finite(pd.DataFrame({"r": fwd.reindex(labels.index), "q": labels})) + if pair.empty: + return pd.DataFrame(np.nan, index=dates, columns=columns) + grouped = pair.groupby( + [pair.index.get_level_values(DATE_LEVEL), pair["q"].astype("int64")], + sort=True, + )["r"].mean() + out = grouped.unstack(level=-1) + out.index.name = DATE_LEVEL + return out.reindex(index=dates, columns=columns) + + +def bucket_membership_weights(labels_wide: pd.DataFrame, bucket: int) -> pd.DataFrame: + """Equal weights of ONE bucket as a date x symbol frame (0.0 outside it). + + ``labels_wide`` is the bucket-label frame already unstacked ONCE (unstacking + per bucket would rebuild a 6M-cell frame five times over at all-A scale). A + missing label compares False, which is what we want: not a member. + """ + member = (labels_wide == bucket).astype(float) + counts = member.sum(axis=1) + return member.div(counts.where(counts > 0), axis=0).fillna(0.0) + + +def quantile_turnover( + labels: pd.Series, n_quantiles: int, dates: pd.Index +) -> pd.DataFrame: + """Per-(date, bucket) turnover ``sum_i |w_i(t) - w_i(t-1)|`` — NO date loop. + + Matches the project's turnover convention, where ``turnover x fee_rate`` is + the period's cost (the phase-2 baseline reports turnover 1.0818 -> cost + 0.108%/period at fee_rate 0.001), i.e. it counts BOTH sides of the trade. + + The FIRST period's turnover is the cost of establishing the book + (``sum_i |w_i(0)|`` = 1.0 for a non-empty bucket), not NaN: entering the + position is a real trade and must be charged for. + + The loop is over the ``n_quantiles`` buckets — a handful of whole-panel + vectorized passes, not one pass per date. + """ + columns = pd.Index(range(1, n_quantiles + 1), name="quantile") + if labels.empty: + return pd.DataFrame(0.0, index=dates, columns=columns) + labels_wide = labels.unstack(level=SYMBOL_LEVEL).reindex(index=dates) + out: dict[int, pd.Series] = {} + for bucket in columns: + weights = bucket_membership_weights(labels_wide, bucket) + delta = weights.diff() + if len(delta): + delta.iloc[0] = weights.iloc[0] + out[bucket] = delta.abs().sum(axis=1) + frame = pd.DataFrame(out, index=dates) + frame.columns = columns + frame.index.name = DATE_LEVEL + return frame + + +# -- the builder ----------------------------------------------------------- + + +def _as_factor_series( + factor_panel: pd.Series | pd.DataFrame, spec: FactorSpec +) -> pd.Series: + """Coerce the supplied panel to ONE float factor Series, or say why not.""" + if isinstance(factor_panel, pd.DataFrame): + if spec.factor_id in factor_panel.columns: + series = factor_panel[spec.factor_id] + elif factor_panel.shape[1] == 1: + series = factor_panel.iloc[:, 0] + else: + raise ValueError( + f"build_eval_ir: the factor panel has columns " + f"{list(factor_panel.columns)!r} and none is named " + f"{spec.factor_id!r}. Pass the factor's own column — guessing which " + f"column IS the factor would silently evaluate the wrong one." + ) + elif isinstance(factor_panel, pd.Series): + series = factor_panel + else: + raise TypeError( + f"build_eval_ir: factor_panel must be a Series or a DataFrame; got " + f"{type(factor_panel).__name__}." + ) + if not isinstance(series.index, pd.MultiIndex) or series.index.nlevels != 2: + raise ValueError( + f"build_eval_ir: the factor panel must carry a MultiIndex" + f"({DATE_LEVEL}, {SYMBOL_LEVEL}); got index names " + f"{list(series.index.names)!r}." + ) + if list(series.index.names) != [DATE_LEVEL, SYMBOL_LEVEL]: + raise ValueError( + f"build_eval_ir: the factor panel index levels must be named " + f"({DATE_LEVEL!r}, {SYMBOL_LEVEL!r}); got {list(series.index.names)!r}." + ) + if series.index.has_duplicates: + raise ValueError( + "build_eval_ir: the factor panel has duplicate (date, symbol) rows; " + "one of them would silently win every alignment." + ) + # Sorting is load-bearing, not cosmetic: it makes the bucket tie-break + # (rank method='first') mean "symbol ascending", which is what the project's + # canonical assign_quantile_buckets does. + return series.astype(float).sort_index().rename(spec.factor_id) + + +def _resolve_forward_returns( + spec: FactorSpec, ctx: EvalContext, dates: pd.Index +) -> tuple[pd.Series, str]: + """``R_h`` + how it was obtained (never guessed). + + ``h`` IS A HORIZON IN EVALUATION PERIODS — that is what the frozen + ``FactorSpec.forward_return_horizon`` says it is ("the horizon (in evaluation + periods) the factor claims to predict"), and one row of F is one evaluation + period. So the shift must happen on the FACTOR's grid. Handing the whole price + panel to ``forward_returns(periods=(h,))`` would shift h rows of the PRICE + panel instead: identical when the two grids coincide (the common case), but a + monthly factor over a daily price panel would silently be scored on its + next-TRADING-DAY return rather than its next-PERIOD return. The panel is + therefore restricted to F's own dates first. + """ + horizon = spec.forward_return_horizon + if ctx.forward_returns is not None: + fwd = pd.Series(ctx.forward_returns, dtype=float) + if not isinstance(fwd.index, pd.MultiIndex) or fwd.index.nlevels != 2: + raise ValueError( + "build_eval_ir: EvalContext.forward_returns must carry a " + f"MultiIndex({DATE_LEVEL}, {SYMBOL_LEVEL})." + ) + source = ( + f"supplied via EvalContext.forward_returns (basis={spec.return_basis!r}, " + f"h={horizon} evaluation periods); computed OUTSIDE this evaluator" + ) + return fwd, source + if spec.return_basis != "close_to_close": + raise ValueError( + f"build_eval_ir: FactorSpec.return_basis={spec.return_basis!r} cannot be " + f"derived from a close panel — its holding period is execution-anchored " + f"(exec(T) -> exec(T_next)) and only the minute-tail machinery knows the " + f"execution anchors. Supply EvalContext.forward_returns computed there." + ) + if ctx.price_panel is None: + raise ValueError( + "build_eval_ir: no forward returns available. Supply either " + "EvalContext.price_panel (a close panel, for a close_to_close factor) " + "or EvalContext.forward_returns. Forward returns are computed at the " + "analytics boundary and never handed to the factor layer." + ) + panel_dates = _unique_dates(ctx.price_panel.index) + absent = dates.difference(panel_dates) + if len(absent): + raise ValueError( + f"build_eval_ir: {len(absent)} factor date(s) are absent from the price " + f"panel (e.g. {[str(d) for d in absent[:3]]}), so their forward return " + f"cannot be measured at all. Align the factor panel to the price panel " + f"before evaluating." + ) + if dates.equals(panel_dates): + on_grid = ctx.price_panel # the common case: no copy of a 6M-row panel + restriction = "" + else: + on_grid = ctx.price_panel[ + ctx.price_panel.index.get_level_values(DATE_LEVEL).isin(dates) + ] + restriction = ( + f" (price panel restricted from {len(panel_dates)} to the factor's own " + f"{len(dates)} evaluation dates FIRST, so h counts evaluation periods " + f"and not price-panel rows)" + ) + column = f"forward_return_{horizon}d" + fwd = _forward_returns(on_grid, periods=(horizon,))[column] + source = ( + f"analytics.factor.forward_returns(price_panel, periods=({horizon},)) — " + f"close[t+{horizon} evaluation periods]/close[t] - 1{restriction}" + ) + return fwd.astype(float), source + + +def _realized_dates(dates: pd.Index, horizon: int) -> pd.Series: + """date -> the date its forward return is realized on: ``h`` PERIODS later. + + Positional on F's own grid, because h is a horizon in evaluation periods and + one row of F is one period. + + Every OOS split in this project slices by the REALIZED date, never the signal + date: a period signalled at t is only out-of-sample once t+h has happened + (P3-3 fixed exactly this bug, and the fix moved the numbers materially). + """ + ahead = np.arange(len(dates)) + horizon + within = ahead < len(dates) + realized = np.full(len(dates), pd.NaT, dtype=object) + realized[within] = dates[ahead[within]] + return pd.Series(realized, index=dates, name="realized_date") + + +def _check_rebalance_grid(cfg: EvalConfig, dates: pd.Index, median_gap: float) -> str: + """Reject a factor grid that contradicts the DECLARED rebalance frequency. + + Returns a disclosure string when the grid is consistent (or unverifiable); + raises when it is provably not. + + WHY THIS EXISTS. One row of F is one evaluation period and the IR does not + resample, so a DAILY panel declared ``rebalance: monthly`` reports ~250 settled + rebalances instead of ~12 — and sails past ``VerdictThresholds.min_rebalances`` + (default 24), the gate whose ENTIRE JOB is deciding INSUFFICIENT-DATA. Merely + disclosing that is not enough: a sample-adequacy gate that can be satisfied by a + sample which does not exist at the declared frequency is not a gate, and every + annualized number in the report is scaled by that same declared frequency. + + WHY REJECT RATHER THAN RESAMPLE — the alternative was to snap F onto the + declared grid ourselves. Rejected, deliberately: + + 1. The project already has exactly ONE monthly convention (last trading day of + the month, ``runtime.backtest.events.monthly_rebalance_dates``, which calls + itself "the single source of truth for which dates"). Resampling here would + either duplicate it — a second, silently divergent calendar, the very + problem the two coexisting bucketing rules already cause — or extend it to + "weekly"/"quarterly", for which the project has NO convention at all and the + eval layer would simply be inventing one. + 2. It would make the EVALUATOR choose which ~12 of ~250 dates constitute the + sample, and that choice moves every number in the report. The caller knows + their intent; a heuristic here would silently overrule it (month-end vs + month-start alone flips the answer). + 3. It would silently discard ~95% of the supplied rows. + 4. A rejection cannot quietly produce a wrong number. A wrong resample can. + + So: the caller resamples to the grid they mean, or declares the frequency they + actually supplied. The contract ("one row = one period") is ENFORCED here rather + than papered over. + """ + label = cfg.rebalance.strip().lower() + expected = REBALANCE_SPACING_DAYS.get(label) + if expected is None: + # Not a recognized label (e.g. a minute-frequency study): there is no + # expected spacing to compare against. Say so — an unverified declaration + # must not read as a verified one. + return ( + f"NOT CHECKED — rebalance={cfg.rebalance!r} is not one of " + f"{tuple(sorted(REBALANCE_SPACING_DAYS))}, so the evaluator has no " + f"expected spacing to check it against. The declared frequency is " + f"UNVERIFIED: settled_rebalances counts the rows supplied, and " + f"min_rebalances gates on that count." + ) + if not math.isfinite(median_gap): + return ( + f"NOT CHECKED — fewer than 2 evaluation periods, so no spacing is " + f"observable (declared {label!r})." + ) + low = expected / REBALANCE_SPACING_TOLERANCE + high = expected * REBALANCE_SPACING_TOLERANCE + if not low <= median_gap <= high: + raise ValueError( + f"build_eval_ir: EvalConfig.rebalance={cfg.rebalance!r} declares " + f"evaluation periods ~{expected:.2f} calendar days apart, but the " + f"supplied factor panel's {len(dates)} periods are a median " + f"{median_gap:.2f} days apart (accepted band {low:.2f}..{high:.2f} days).\n" + f"\n" + f"ONE ROW OF THE FACTOR PANEL IS ONE EVALUATION PERIOD — this evaluator " + f"does not resample. It would therefore report {len(dates)} settled " + f"rebalances for a grid you called {label!r}, and " + f"VerdictThresholds.min_rebalances (the gate that decides " + f"INSUFFICIENT-DATA) would pass on a sample size that does not exist at " + f"the declared frequency. Every annualized number would be scaled by the " + f"declared frequency too.\n" + f"\n" + f"Fix: resample the factor panel to the grid you actually mean to " + f"evaluate BEFORE calling the evaluator (the project's monthly " + f"convention is the last trading day of each month — " + f"runtime.backtest.events.monthly_rebalance_dates), or declare the " + f"frequency you actually supplied." + ) + return ( + f"OK — declared {label!r} (~{expected:.2f} calendar days between periods); " + f"observed median spacing {median_gap:.2f} days over {len(dates)} periods." + ) + + +def _median_gap_days(dates: pd.Index) -> float: + if len(dates) < 2 or not isinstance(dates, pd.DatetimeIndex): + return float("nan") + gaps = pd.Series(dates).diff().dropna() + if gaps.empty: + return float("nan") + return float(gaps.dt.total_seconds().median() / 86400.0) + + +def build_eval_ir( + factor_panel: pd.Series | pd.DataFrame, + spec: FactorSpec, + cfg: EvalConfig, + ctx: EvalContext | None = None, +) -> StandardEvalIR: + """Compute the design §8 four objects ONCE, vectorized (no per-period loop). + + Parameters + ---------- + factor_panel : the PROCESSED factor, MultiIndex(date, symbol). A DataFrame is + accepted when the factor's own column can be identified unambiguously. + spec, cfg : the frozen provenance contract (horizon, hypothesis, quantiles). + ctx : :class:`EvalContext` — the price panel / pre-computed returns / anchors. + + Returns + ------- + :class:`StandardEvalIR`, satisfying the frozen ``EvalIR`` Protocol. + """ + if not isinstance(spec, FactorSpec): + raise TypeError(f"build_eval_ir needs a FactorSpec; got {type(spec).__name__}.") + if not isinstance(cfg, EvalConfig): + raise TypeError(f"build_eval_ir needs an EvalConfig; got {type(cfg).__name__}.") + context = ctx if ctx is not None else EvalContext() + if not isinstance(context, EvalContext): + raise TypeError( + f"build_eval_ir: ctx must be an EvalContext; got {type(context).__name__}." + ) + + factor = _as_factor_series(factor_panel, spec) + dates = _unique_dates(factor.index) + # Fail fast, BEFORE any expensive work: a grid that contradicts the declared + # rebalance frequency makes the sample-adequacy gate meaningless downstream. + median_gap = _median_gap_days(dates) + grid_check = _check_rebalance_grid(cfg, dates, median_gap) + fwd_full, fwd_source = _resolve_forward_returns(spec, context, dates) + # R_h is aligned ONTO F: the IR's two panels always share one index, so every + # downstream reduction is a plain column-wise operation. + fwd = fwd_full.reindex(factor.index).astype(float).rename("forward_return") + + ic = cross_section_corr(factor, fwd, rank=True, dates=dates) + ic_pearson = cross_section_corr(factor, fwd, rank=False, dates=dates) + labels = assign_quantile_buckets(factor, cfg.n_quantiles) + quantiles = quantile_return_matrix(fwd, labels, cfg.n_quantiles, dates) + + usable_pair = _finite(pd.DataFrame({"f": factor, "r": fwd})) + cross_section_size = ( + usable_pair.groupby(level=DATE_LEVEL, sort=True).size().reindex(dates).fillna(0) + ) + factor_count = ( + _finite(factor.to_frame("f")) + .groupby(level=DATE_LEVEL, sort=True) + .size() + .reindex(dates) + .fillna(0) + ) + warmup_mask = ( + factor.groupby(level=SYMBOL_LEVEL, sort=False).cumcount() < spec.min_history_bars + ) + + return StandardEvalIR( + factor=factor, + forward_returns=fwd, + ic=ic, + quantile_returns=quantiles, + spec=spec, + cfg=cfg, + ctx=context, + ic_pearson=ic_pearson, + quantile_labels=labels, + quantile_turnover=quantile_turnover(labels, cfg.n_quantiles, dates), + cross_section_size=cross_section_size.astype(float), + factor_count=factor_count.astype(float), + realized_date=_realized_dates(dates, spec.forward_return_horizon), + dates=dates, + forward_return_source=fwd_source, + median_period_gap_days=median_gap, + rebalance_grid_check=grid_check, + warmup_mask=warmup_mask.rename("warmup"), + ) + + +__all__ = [ + "DECAY_LAGS", + "DEFAULT_PERIODS_PER_YEAR", + "PERIODS_PER_YEAR", + "REBALANCE_SPACING_DAYS", + "REBALANCE_SPACING_TOLERANCE", + "EvalContext", + "StandardEvalIR", + "assign_quantile_buckets", + "bucket_membership_weights", + "build_eval_ir", + "cross_section_corr", + "quantile_return_matrix", + "quantile_turnover", +] diff --git a/analytics/eval/render.py b/analytics/eval/render.py new file mode 100644 index 0000000..a0d53a1 --- /dev/null +++ b/analytics/eval/render.py @@ -0,0 +1,305 @@ +"""Presentation of a :class:`~analytics.eval.report.FactorEvalReport`. + +Two deterministic renderings of the same data: the Markdown page (the 10 fixed +sections of design §5) and the machine-readable dict (the cross-run comparable +record). Split out of ``report.py`` so the data model stays the data model. + +Import direction: ``report`` imports THIS module at runtime; the report types are +only needed for annotations here, so they are imported under ``TYPE_CHECKING`` — +no import cycle. + +Secret-safety: every string that reaches a rendered/exported artifact passes +through the D3 quality layer's ``sanitize_text`` (token values, ``.config.json`` +paths), and payload values through its bounded ``clean_value``. ``analytics`` +importing ``data.quality`` is downstream->upstream (``analytics/factor.py`` +already imports ``data.clean.schema``), so this reuses the redaction rules +instead of re-implementing them. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING + +from analytics.eval.sections import MANDATORY_SECTIONS, Skipped +from data.quality.report import clean_value, sanitize_text + +if TYPE_CHECKING: # pragma: no cover - annotations only + from analytics.eval.report import FactorEvalReport + from analytics.eval.sections import SectionLike + +# Bound on ONE rendered payload value, mirroring the D3 quality layer's bounded +# examples (MAX_EXAMPLES=5): ``clean_value``'s fallback is ``str(value)``, so a +# payload that accidentally holds a big object (a whole panel's repr) would dump +# it into the artifact. A report is a summary; redaction has already run inside +# ``clean_value``, so truncating afterwards can never re-expose a secret. +MAX_VALUE_CHARS = 200 +_TRUNCATED = "...[truncated]" + +# Rendered labels for the three verdict axes (design §6, v0.5). +AXIS_TITLES: dict[str, str] = { + "predictive": "Predictive", + "incremental": "Incremental", + "tradable": "Tradable", +} + +# Rendered headings for the 8 mandatory sections (rendered as 2..9). +SECTION_TITLES: dict[str, str] = { + "predictive_power": "Predictive Power", + "return_risk": "Return & Risk", + "stability_cost": "Stability & Cost", + "purity": "Purity", + "oos_generalization": "OOS & Generalization", + "execution_capacity": "Execution & Capacity", + "data_coverage": "Data & Coverage", + "caveats": "Caveats & Provenance", +} + + +def sanitize_payload(value: object) -> object: + """Recursively coerce a payload value to a deterministic, secret-free form. + + Mappings are key-sorted here, so a section's insertion order can never leak + into the artifact (a report must be byte-identical across runs). + + KEYS are redacted too, not just values: a payload key is just as capable of + carrying a token-shaped string into the artifact, and the D3 layer this + reuses already redacts its example keys (``_format_examples``) — leaving + keys raw here would be both leaky and inconsistent with it. + """ + if isinstance(value, Mapping): + # Sort the SOURCE by raw key first, then re-sort by the redacted key: two + # distinct secret-shaped keys can redact to the same marker, and this + # keeps which one survives (and the output order) deterministic. + cleaned = { + sanitize_text(str(k)): sanitize_payload(v) + for k, v in sorted(value.items(), key=lambda kv: str(kv[0])) + } + return dict(sorted(cleaned.items())) + if isinstance(value, (list, tuple)): + return [sanitize_payload(v) for v in value] + if value is None or isinstance(value, bool): + return value + cleaned = clean_value(value) + if isinstance(cleaned, str) and len(cleaned) > MAX_VALUE_CHARS: + return cleaned[:MAX_VALUE_CHARS] + _TRUNCATED + return cleaned + + +def format_value(value: object) -> str: + """Compact deterministic rendering of one payload value.""" + if isinstance(value, dict): + return "{" + ", ".join(f"{k}: {format_value(v)}" for k, v in value.items()) + "}" + if isinstance(value, list): + return "[" + ", ".join(format_value(v) for v in value) + "]" + return str(value) + + +def canonical_sections( + report: FactorEvalReport, mandatory: tuple[str, ...] +) -> list[tuple[str, SectionLike | None]]: + """Sections in CANONICAL order, independent of how they were assembled. + + The mandatory set first, in the fixed design §5 order (an absent one yields + ``None``, so every consumer must decide what to say about it), then any extra + section sorted by name. Assembly order must never leak into an artifact: two + reports carrying the same sections in a different order have to produce + byte-identical output, or a cross-run diff turns into noise. + """ + indexed = report.by_name() + ordered: list[tuple[str, SectionLike | None]] = [ + (name, indexed.get(name)) for name in mandatory + ] + extras = sorted( + (s for s in report.sections if s.name not in mandatory), key=lambda s: s.name + ) + return ordered + [(s.name, s) for s in extras] + + +def report_to_dict(report: FactorEvalReport) -> dict: + """Machine-readable record with a STABLE key order and CANONICAL section order.""" + verdict = report.require_verdict() + sections: list[dict] = [] + for name, section in canonical_sections(report, MANDATORY_SECTIONS): + if section is None: + # Unreachable via the public API (with_verdict validates, and every + # export requires a verdict); kept explicit so a hand-built report + # can never export a SHORT record that looks structurally valid. + # This mirrors the Markdown's _MISSING_ marker: the JSON states the + # hole rather than omitting the section. + sections.append( + { + "name": name, + "reason": "MISSING — the contract requires this section.", + "status": "missing", + } + ) + continue + if isinstance(section, Skipped): + sections.append( + {"name": section.name, "reason": section.reason, "status": "skipped"} + ) + continue + entry: dict[str, object] = { + "name": section.name, + "payload": sanitize_payload(section.payload), + "status": "ok", + } + if section.note: + entry["note"] = section.note + sections.append(dict(sorted(entry.items()))) + return { + "criteria_source": report.criteria_source, + "eval_config": sanitize_payload(vars(report.cfg)), + "schema_version": report.SCHEMA_VERSION, + "sections": sections, + "spec": sanitize_payload(vars(report.spec)), + # the ACTUAL criteria values used, whatever their source (design §6, v0.6). + "thresholds": sanitize_payload(vars(report.thresholds)), + # The DERIVED deployment label + the three axis-verdicts it was derived + # from (design §6, v0.5), so a cross-run record can compare axes, not just + # the single label. + "verdict": { + "reasons": [sanitize_text(r) for r in verdict.reasons], + "verdict": verdict.verdict, + "axes": { + name: { + "reasons": [sanitize_text(r) for r in axis.reasons], + "verdict": axis.verdict, + } + for name, axis in verdict.axes().items() + }, + }, + } + + +def render_report(report: FactorEvalReport, mandatory: tuple[str, ...]) -> str: + """Deterministic Markdown: the 10 fixed sections of design §5.""" + verdict = report.require_verdict() + spec = report.spec + lines = [f"# Factor Evaluation — {spec.factor_id} (v{spec.version})", ""] + + lines += ["## 0. Header & Provenance", ""] + for label, value in _provenance_rows(report): + lines.append(f"- {label}: {value}") + lines.append("") + + lines += ["## 1. Verdict & Scorecard", "", f"**{verdict.verdict}**", ""] + lines += [f"- {reason}" for reason in verdict.reasons] + lines.append("") + # The three axis-verdicts the deployment label was derived from (design §6, + # v0.5). Rendered under the label so the reader sees WHICH axis carried it. + lines.append("Axes:") + for name, axis in verdict.axes().items(): + detail = f" — {sanitize_text(axis.reasons[0])}" if axis.reasons else "" + lines.append(f"- {AXIS_TITLES.get(name, name)}: {axis.verdict}{detail}") + lines.append("") + if report.criteria_source == "declared": + criteria_prefix = ( + "Success criteria (PRE-REGISTERED via EvalConfig.success_criteria — " + "declared before the run)" + ) + else: + criteria_prefix = ( + "Success criteria (source: DEFAULT global bar — ⚠️ unvalidated defaults, " + "pending calibration; NOT pre-registered for this run)" + ) + lines.append( + criteria_prefix + + ": " + + ", ".join(f"{k}={v}" for k, v in sorted(vars(report.thresholds).items())) + ) + lines.append("") + + for index, (name, section) in enumerate(canonical_sections(report, mandatory)): + # the mandatory set renders as the fixed sections 2..9; extras follow + # unnumbered (they are additions to the contract, not part of it). + number = index + 2 if index < len(mandatory) else None + lines += _render_section(number, name, section) + return "\n".join(lines).rstrip() + "\n" + + +def _provenance_rows(report: FactorEvalReport) -> list[tuple[str, str]]: + spec, cfg = report.spec, report.cfg + rows: list[tuple[str, object]] = [ + ("factor_id", spec.factor_id), + ("description", spec.description), + ("family", spec.family), + ("hypothesis (expected IC sign, fixed pre-run)", f"{spec.expected_ic_sign:+d}"), + ("horizon / return basis", f"h={spec.forward_return_horizon} / {spec.return_basis}"), + ("input fields", ", ".join(spec.input_fields)), + ("price adjust / warm-up bars", f"{spec.price_adjust} / {spec.min_history_bars}"), + ("universe", f"{cfg.universe} (PIT={cfg.universe_is_pit})"), + ("window / rebalance", f"{cfg.start} .. {cfg.end} / {cfg.rebalance}"), + ("quantiles / long-short", f"{cfg.n_quantiles} / {cfg.long_short}"), + ("cost scenarios", ", ".join(f"{c}x" for c in cfg.cost_scenarios)), + ( + "processing", + f"winsorize={cfg.winsorize}, standardize={cfg.standardize}, " + f"neutralization={cfg.neutralization} ({cfg.industry_level})", + ), + ("oos split", cfg.oos_split), + ("independent cells", len(cfg.independent_cells)), + ( + "honesty flags", + f"exploratory={cfg.is_exploratory}, post_hoc_selected={cfg.post_hoc_selected}, " + f"tuned={cfg.tuned}, n_factors_screened={cfg.n_factors_screened}", + ), + ( + "verdict criteria source", + f"{report.criteria_source} " + + ( + "(pre-registered in EvalConfig.success_criteria)" + if report.criteria_source == "declared" + else "(global default bar — not pre-registered for this run)" + ), + ), + ("data snapshot", cfg.data_snapshot_id), + ] + if spec.is_intraday: + rows.append( + ( + "intraday contract", + f"cutoff={spec.decision_cutoff}, lag={spec.data_lag}, " + f"session_open={spec.session_open}, " + f"execution_model={spec.execution_model}, " + f"window={spec.execution_window}", + ) + ) + return [(label, sanitize_text(str(value))) for label, value in rows] + + +def _render_section( + number: int | None, name: str, section: SectionLike | None +) -> list[str]: + title = SECTION_TITLES.get(name, name) + heading = f"## {number}. {title}" if number is not None else f"## + {title}" + lines = [heading, ""] + if section is None: + # unreachable via the public API (with_verdict validates and render + # requires a verdict); kept explicit so a hand-built report can never + # render a silent hole. The JSON export marks it too ("status": + # "missing") — the two renderings must not disagree about a hole. + return lines + ["_MISSING — the contract requires this section._", ""] + if isinstance(section, Skipped): + return lines + [f"_Skipped: {section.reason}_", ""] + if section.note: + lines += [section.note, ""] + payload = sanitize_payload(section.payload) + if not payload: + return lines + ["_No metrics reported._", ""] + lines += [f"- {key}: {format_value(payload[key])}" for key in sorted(payload)] + lines.append("") + return lines + + +__all__ = [ + "MAX_VALUE_CHARS", + "AXIS_TITLES", + "SECTION_TITLES", + "canonical_sections", + "format_value", + "render_report", + "report_to_dict", + "sanitize_payload", +] diff --git a/analytics/eval/report.py b/analytics/eval/report.py new file mode 100644 index 0000000..63ec8a8 --- /dev/null +++ b/analytics/eval/report.py @@ -0,0 +1,296 @@ +"""``FactorEvalReport``: the standard evaluation output (provenance + sections + verdict). + +Enforcement layer #2 (design ``tmp/design/factor_eval_contract_v0.1.md`` §7): +:meth:`FactorEvalReport.validate_all_mandatory_present` raises when a mandatory +section is simply absent. A :class:`~analytics.eval.sections.Skipped` WITH a +reason counts as present — the contract demands DISCLOSURE, not results. + +A report is built in three explicit steps (the design's template method): + + assemble -> validate_all_mandatory_present -> with_verdict + +Rendering/exporting REQUIRES the verdict: publishing a report that dodges the +verdict is precisely what this contract exists to prevent. + +COMPLETENESS IS ENFORCED TWICE, on purpose (绝不静默漏段): + + 1. :meth:`with_verdict` re-runs ``validate_all_mandatory_present`` itself, so a + caller who skips step 2 still cannot obtain a verdict on an incomplete + report. ``assemble`` stays permissive (a report under construction is not + yet a claim); the VERDICT is the gate, because the verdict is the claim. + 2. Since ``render`` / ``to_dict`` / ``to_json`` all require a verdict, they are + unreachable on an unvalidated report through the public API. The remaining + path is a hand-rolled ``FactorEvalReport(..., verdict=...)`` that bypasses + both classmethod and step 2 — for THAT, both renderings mark an absent + mandatory section EXPLICITLY (Markdown ``_MISSING_``, JSON + ``{"status": "missing"}``) rather than quietly emitting a short record. + Silently dropping a section from the machine-readable export was the actual + hole: the Markdown said ``_MISSING_`` while the JSON just shipped 3 of 8 + sections and looked structurally valid. + +The section types live in ``sections.py`` and the two renderings in +``render.py``; this module is the data model + the verdict extraction. +""" + +from __future__ import annotations + +import json +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field, replace + +from analytics.eval.config import EvalConfig +from analytics.eval.render import render_report, report_to_dict +from analytics.eval.sections import ( + MANDATORY_SECTIONS, + VERDICT_KEYS, + Section, + SectionLike, + Skipped, +) +from analytics.eval.verdict import ( + VerdictInputs, + VerdictResult, + VerdictThresholds, + decide_verdict, +) +from factors.spec import FactorSpec + +SCHEMA_VERSION = "0.1" + + +def _payload_number(payload: Mapping[str, object], key: str, default: float) -> float: + value = payload.get(key, default) + try: + return float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return default + + +def _payload_flag(payload: Mapping[str, object], key: str, default: bool | None): + value = payload.get(key, default) + return value if isinstance(value, bool) else default + + +def extract_verdict_inputs( + spec: FactorSpec, cfg: EvalConfig, sections: Mapping[str, SectionLike] +) -> VerdictInputs: + """Pull the verdict's facts out of the section payloads (keys: VERDICT_KEYS). + + A skipped/missing section leaves its facts UNKNOWN (NaN / None / False / 0 + rebalances) rather than inventing them: unknown never earns an Adopt, and + only an EXPLICIT failure produces a hard Reject (see verdict.py). + + ``cfg`` supplies the one non-measured input: ``is_exploratory``, the run's + own declaration, which caps the verdict at Watch (design §6). It is read off + the CONFIG — where EvalConfig makes it mandatory — and never off the caveats + payload, so an evaluator cannot shed the cap by omitting a key. + """ + + def payload_of(name: str) -> Mapping[str, object]: + section = sections.get(name) + return section.payload if isinstance(section, Section) else {} + + coverage = payload_of("data_coverage") + predictive = payload_of("predictive_power") + returns = payload_of("return_risk") + purity = payload_of("purity") + oos = payload_of("oos_generalization") + execution = payload_of("execution_capacity") + + raw_costs = returns.get("net_long_short_by_cost", {}) + by_cost: tuple[tuple[float, float], ...] = () + if isinstance(raw_costs, Mapping): + by_cost = tuple( + (float(k), float(v)) + for k, v in sorted(raw_costs.items(), key=lambda kv: float(kv[0])) + ) + + settled = coverage.get("settled_rebalances", 0) + return VerdictInputs( + expected_ic_sign=spec.expected_ic_sign, + is_exploratory=cfg.is_exploratory, + settled_rebalances=int(settled) if isinstance(settled, (int, float)) else 0, + # Gate parts A and B. Absent -> NaN -> reported as UNKNOWN and FAILS the + # gate; an unmeasured sample size must never satisfy a sample gate. + effective_samples=_payload_number(coverage, "effective_samples", float("nan")), + span_days=_payload_number(coverage, "span_days", float("nan")), + ic_ir=_payload_number(predictive, "ic_ir", float("nan")), + ic_ir_ci_low=_payload_number(predictive, "ic_ir_ci_low", float("nan")), + ic_ir_ci_high=_payload_number(predictive, "ic_ir_ci_high", float("nan")), + ic_win_rate=_payload_number(predictive, "ic_win_rate", float("nan")), + ic_nw_t=_payload_number(predictive, "ic_nw_t", float("nan")), + monotonicity_spearman=_payload_number( + returns, "monotonicity_spearman", float("nan") + ), + # Incremental axis facts (design §6, v0.5). A Skipped purity section (no + # book) leaves the payload empty -> known_factors_supplied defaults False + # -> the axis is NOT_ASSESSED. A supplied-but-unmeasurable orthogonalized + # IC is NaN -> INSUFFICIENT_DATA, never a FAIL (unknown never convicts). + known_factors_supplied=bool( + _payload_flag(purity, "known_factors_supplied", False) + ), + incremental_ic_ir=_payload_number(purity, "incremental_ic_ir", float("nan")), + incremental_ic_mean=_payload_number( + purity, "incremental_ic_mean", float("nan") + ), + incremental_ic_ir_ci_low=_payload_number( + purity, "incremental_ic_ir_ci_low", float("nan") + ), + incremental_ic_ir_ci_high=_payload_number( + purity, "incremental_ic_ir_ci_high", float("nan") + ), + net_long_short_by_cost=by_cost, + oos_available=bool(_payload_flag(oos, "oos_available", False)), + oos_sign_consistent=bool(_payload_flag(oos, "sign_consistent", False)), + oos_sign_flipped=bool(_payload_flag(oos, "sign_flipped", False)), + oos_monotonicity_reversed=bool( + _payload_flag(oos, "monotonicity_reversed", False) + ), + tradable=_payload_flag(execution, "tradable", None), + capacity_sufficient=_payload_flag(execution, "capacity_sufficient", None), + ) + + +@dataclass(frozen=True) +class FactorEvalReport: + """Provenance (spec + cfg) + sections + verdict. Immutable.""" + + spec: FactorSpec + cfg: EvalConfig + sections: tuple[SectionLike, ...] + verdict: VerdictResult | None = None + thresholds: VerdictThresholds = field(default_factory=VerdictThresholds) + #: "declared" when the verdict bar came from EvalConfig.success_criteria (a + #: pre-registered bar), "default" otherwise. Stamped in the provenance box + + #: JSON so a reader can tell a pre-registered result from one judged against the + #: global default (design §6, v0.6). + criteria_source: str = "default" + + #: exported so a stored record can be matched to the schema that wrote it. + SCHEMA_VERSION = SCHEMA_VERSION + + @classmethod + def assemble( + cls, + spec: FactorSpec, + cfg: EvalConfig, + sections: Sequence[SectionLike], + thresholds: VerdictThresholds | None = None, + ) -> FactorEvalReport: + """Collect sections into a report (no verdict yet). Rejects duplicates.""" + if not isinstance(spec, FactorSpec): + raise TypeError( + f"FactorEvalReport needs a FactorSpec; got {type(spec).__name__}." + ) + if not isinstance(cfg, EvalConfig): + raise TypeError( + f"FactorEvalReport needs an EvalConfig; got {type(cfg).__name__}." + ) + collected = tuple(sections) + bad = [s for s in collected if not isinstance(s, (Section, Skipped))] + if bad: + raise TypeError( + f"every report section must be a Section or an explicit " + f"Skipped(reason); got {[type(s).__name__ for s in bad]}." + ) + counts = Counter(s.name for s in collected) + duplicates = sorted(name for name, n in counts.items() if n > 1) + if duplicates: + raise ValueError( + f"duplicate report section(s) {duplicates}: a collision would " + f"silently overwrite one section's findings with another's." + ) + return cls( + spec=spec, + cfg=cfg, + sections=collected, + thresholds=thresholds or VerdictThresholds(), + ) + + def by_name(self) -> dict[str, SectionLike]: + """Sections keyed by name (unique — ``assemble`` guarantees it).""" + return {s.name: s for s in self.sections} + + def validate_all_mandatory_present(self) -> None: + """Raise unless every mandatory section is present (enforcement #2).""" + present = {s.name for s in self.sections} + missing = [name for name in MANDATORY_SECTIONS if name not in present] + if missing: + raise ValueError( + f"factor evaluation report is missing mandatory section(s) " + f"{missing} for {self.spec.factor_id!r}. Every section in " + f"{MANDATORY_SECTIONS} must be produced, or explicitly returned as " + f"Skipped(name, reason) — a silently missing section is not allowed." + ) + + def with_verdict( + self, thresholds: VerdictThresholds | None = None + ) -> FactorEvalReport: + """Return a NEW report carrying the verdict (this object is unchanged). + + Validates FIRST, so "a verdicted report is a complete report" holds for + EVERY caller — not just the ones that remember to call + :meth:`validate_all_mandatory_present` themselves. Judging a factor on a + report with silently absent sections is exactly the self-deception this + contract exists to stop, and such a record would poison the cross-run + factor library (design §11). This is the same enforcement layer #2, just + made unbypassable. + """ + self.validate_all_mandatory_present() + # PRE-REGISTERED criteria win (design §6, v0.6): a bar declared on the frozen + # EvalConfig BEFORE evaluate() ran is pre-registered by construction. An + # explicit runtime `thresholds` override (or the assembled default) is used + # only when the config declared none. The chosen source is stamped so the + # report cannot hide which bar it was judged against. + if self.cfg.success_criteria is not None: + thr = self.cfg.success_criteria + source = "declared" + else: + thr = thresholds or self.thresholds + source = "default" + inputs = extract_verdict_inputs(self.spec, self.cfg, self.by_name()) + return replace( + self, + verdict=decide_verdict(inputs, thr), + thresholds=thr, + criteria_source=source, + ) + + def require_verdict(self) -> VerdictResult: + """The verdict, or a readable error — a report may not dodge it.""" + if self.verdict is None: + raise ValueError( + f"factor evaluation report for {self.spec.factor_id!r} has no " + f"verdict: call with_verdict() first. Publishing a report without " + f"a verdict is exactly what this contract forbids." + ) + return self.verdict + + # -- exports ---------------------------------------------------------- + + def to_dict(self) -> dict: + """Machine-readable record, stable key order (the cross-run record).""" + return report_to_dict(self) + + def to_json(self, *, indent: int | None = 2) -> str: + """Deterministic JSON (sorted keys) of :meth:`to_dict`.""" + return json.dumps( + self.to_dict(), indent=indent, sort_keys=True, ensure_ascii=False + ) + + def render(self) -> str: + """Deterministic Markdown: the 10 fixed sections of design §5.""" + return render_report(self, MANDATORY_SECTIONS) + + +__all__ = [ + "MANDATORY_SECTIONS", + "SCHEMA_VERSION", + "VERDICT_KEYS", + "FactorEvalReport", + "Section", + "SectionLike", + "Skipped", + "extract_verdict_inputs", +] diff --git a/analytics/eval/sections.py b/analytics/eval/sections.py new file mode 100644 index 0000000..3e63fc3 --- /dev/null +++ b/analytics/eval/sections.py @@ -0,0 +1,140 @@ +"""``Section`` / ``Skipped``: what one report section may be — and nothing else. + +A mandatory section is either produced (:class:`Section`) or explicitly skipped +with a reason (:class:`Skipped`). There is no third option: a silently missing +section is what :meth:`FactorEvalReport.validate_all_mandatory_present` raises +on (enforcement layer #2, design §7). This mirrors the project's standing rule +绝不静默降级 — a degraded path must SAY it degraded. + +Kept in its own module so both ``report`` (the data model) and ``render`` (the +presentation) can import these types without an import cycle. +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping +from dataclasses import dataclass, field + +from data.quality.report import sanitize_text + +# The 8 evaluator-produced sections, in FIXED order (design §5). They render as +# sections 2..9; sections 0 (provenance) and 1 (verdict) come from the report. +MANDATORY_SECTIONS: tuple[str, ...] = ( + "predictive_power", + "return_risk", + "stability_cost", + "purity", + "oos_generalization", + "execution_capacity", + "data_coverage", + "caveats", +) + +# --- payload keys the verdict reads (the contract PR-B must fill) ------------ +# Every key is a RAW fact; the hypothesis (expected_ic_sign) is applied by the +# verdict rules, never by the section. See analytics/eval/verdict.py. +VERDICT_KEYS: dict[str, tuple[str, ...]] = { + # The three-part sample gate's facts (design §6, v0.3). settled_rebalances is + # the RAW count of rebalances that settled; effective_samples is the IC + # series' N_eff = N / (1 + 2*sum_k rho_k), CLAMPED to [1, N]; span_days is + # the IC series' calendar span. The last two are computed from the IR + # (ir.ic + its date index) — a raw count is not a sample size under a daily + # rebalance, so the gate reads all three or reports UNKNOWN and fails. + "data_coverage": ("settled_rebalances", "effective_samples", "span_days"), + # ic_ir = mean(IC)/std(IC); ic_win_rate = share of periods whose IC carries + # the EXPECTED sign (hypothesis-relative BY DEFINITION); ic_nw_t = + # Newey-West autocorrelation-corrected t of the mean IC. + # ic_ir_ci_low/high = N_eff-based 95% CI of ic_ir (design §6, v0.6): the + # Predictive PASS gates on the LOWER bound, not the point. + "predictive_power": ( + "ic_ir", + "ic_ir_ci_low", + "ic_ir_ci_high", + "ic_win_rate", + "ic_nw_t", + ), + # monotonicity_spearman: RAW Spearman(bucket index, bucket mean return). + # net_long_short_by_cost: {cost multiplier: net (top - bottom) return}, RAW. + "return_risk": ("monotonicity_spearman", "net_long_short_by_cost"), + # The Incremental axis (design §6, v0.5). known_factors_supplied = was a + # known-factor BOOK supplied at all (False -> the axis is NOT_ASSESSED); + # incremental_ic_ir / incremental_ic_mean = mean(orthIC)/std(orthIC) and + # mean(orthIC) of the factor's IC AFTER residualizing it on the WHOLE book per + # date. RAW: the verdict applies expected_ic_sign, never the section. + "purity": ( + "known_factors_supplied", + "incremental_ic_ir", + "incremental_ic_mean", + # N_eff-based 95% CI of the orthogonalized ICIR (design §6, v0.6): the + # Incremental PASS gates on the LOWER bound. + "incremental_ic_ir_ci_low", + "incremental_ic_ir_ci_high", + ), + "oos_generalization": ( + "oos_available", + "sign_consistent", + "sign_flipped", + "monotonicity_reversed", + ), + "execution_capacity": ("tradable", "capacity_sufficient"), +} + + +@dataclass(frozen=True) +class Section: + """One produced report section: a name plus its metric payload. + + ``payload`` holds the section's facts (the verdict reads the documented keys + in :data:`VERDICT_KEYS`; everything else is rendered for the human). It is + DEEP-copied on construction — a caller mutating its dict afterwards must not + be able to rewrite a report — and always rendered/exported with sorted keys. + A shallow copy is not enough: ``net_long_short_by_cost`` (a nested dict the + verdict READS) would still be aliased to the caller's object. + """ + + name: str + payload: Mapping[str, object] = field(default_factory=dict) + note: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.name, str) or not self.name.strip(): + raise ValueError(f"Section.name must be a non-empty string; got {self.name!r}.") + if not isinstance(self.payload, Mapping): + raise ValueError( + f"Section({self.name!r}).payload must be a mapping of metric name " + f"-> value; got {type(self.payload).__name__}." + ) + object.__setattr__(self, "payload", copy.deepcopy(dict(self.payload))) + if self.note is not None: + object.__setattr__(self, "note", sanitize_text(str(self.note))) + + +@dataclass(frozen=True) +class Skipped: + """A section that was NOT produced — with a mandatory, explicit reason. + + The reason is the whole point: a skipped section is a DISCLOSED degradation + ("no OOS split configured", "capacity diagnostic disabled"), never a silent + hole in the report. + """ + + name: str + reason: str + + def __post_init__(self) -> None: + if not isinstance(self.name, str) or not self.name.strip(): + raise ValueError(f"Skipped.name must be a non-empty string; got {self.name!r}.") + if not isinstance(self.reason, str) or not self.reason.strip(): + raise ValueError( + f"Skipped({self.name!r}) requires a non-empty reason: a section may " + f"be skipped ONLY with an explicit, disclosed reason — never " + f"silently." + ) + object.__setattr__(self, "reason", sanitize_text(self.reason)) + + +SectionLike = Section | Skipped + + +__all__ = ["MANDATORY_SECTIONS", "VERDICT_KEYS", "Section", "SectionLike", "Skipped"] diff --git a/analytics/eval/standard.py b/analytics/eval/standard.py new file mode 100644 index 0000000..dc6d7c4 --- /dev/null +++ b/analytics/eval/standard.py @@ -0,0 +1,947 @@ +"""``StandardFactorEvaluator``: the default implementation of the 8 mandatory sections. + +Design ``tmp/design/factor_eval_contract_v0.1.md`` §10 step 2. Every section is a +cheap reduction of the eval-IR (:mod:`analytics.eval.ir`) — the IR is built ONCE, +vectorized, and nothing here touches the panel again. + +WHAT IS REAL AND WHAT IS SKIPPED (the point of the contract is that you can tell): + + predictive_power real — IC / ICIR / hypothesis win rate / Newey-West t + return_risk real — quantile NAVs / net long-short / monotonicity + stability_cost real — turnover / rank autocorrelation / cost gradient + purity real ONLY when EvalContext.known_factors is supplied, + Skipped otherwise (no anchors = no purity claim). When + real it also carries the Incremental verdict axis's + fact: the factor's IC AFTER residualizing it on the + WHOLE book jointly (incremental_ic_ir). + oos_generalization real ONLY when EvalConfig.oos_split is set, + Skipped otherwise ("no OOS evidence") + execution_capacity Skipped unless EvalContext.execution_capacity carries + facts MEASURED ELSEWHERE — this evaluator does not + run the I5b/I5f machinery and will not pretend to + data_coverage real + caveats real + +Because ``execution_capacity`` is Skipped by default, the Tradable axis is +NOT_ASSESSED, and because ``purity`` is Skipped without a book the Incremental axis +is NOT_ASSESSED too — so a default run CANNOT reach Adopt (design §6, v0.5: Adopt +needs all three axes PASS) and tops out at Watch. That is deliberate: an untested +execution path is not evidence of tradability, and a factor judged with no known +book has not shown it adds anything. Both are disclosed in the section reasons, +not hidden. + +NEVER FABRICATE, NEVER SILENTLY OMIT: a metric that cannot be computed is NaN or +an explicit Skipped(reason), never a plausible-looking number. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping + +import numpy as np +import pandas as pd + +from analytics.eval.config import EvalConfig +from analytics.eval.evaluator import FactorEvaluator +from analytics.eval.report import FactorEvalReport +from analytics.eval.verdict import VerdictThresholds +from analytics.eval.ir import ( + DECAY_LAGS, + EvalContext, + StandardEvalIR, + build_eval_ir, + cross_section_corr, +) +from analytics.eval.sections import Section, Skipped +from analytics.eval.stats import ( + DEFAULT_CONFIDENCE, + as_float, + effective_sample_size, + half_life, + hypothesis_win_rate, + information_ratio_ci, + mean_ci, + newey_west_t, + sortino, + spearman, +) +from analytics.factor import ic_summary +from analytics.performance import performance_summary +from analytics.quantstats_adapter import quantstats_performance +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL +from factors.spec import FactorSpec + +#: The synthetic-leg caveat the project pays for every time it forgets it (I5d). +LONG_SHORT_NOTE = ( + "QN-Q1 is a SYNTHETIC LONG-ONLY LEG DIFFERENCE (the top bucket's mean return " + "minus the bottom bucket's), NOT a dollar-neutral executed portfolio: no " + "short book, no borrow, no financing, and the two legs are never netted into " + "one order. Costs are charged as turnover x fee_rate on each leg. Sharpe / " + "Sortino / maxDD / vol below are computed on the HYPOTHESIS-ALIGNED " + "(expected_ic_sign x) base-cost spread, so a -1 factor reads in its own " + "direction. Buckets are formed from the FACTOR cross-section alone; a bucket " + "mean covers only the members whose forward return exists." +) + + +def _ic_span_days(ic: pd.Series) -> float: + """Calendar days spanned by the SETTLED part of the IC series (gate part B). + + Only periods that actually produced a finite IC count: a period that never + settled did not extend the evidence, so it must not extend the span either + (the same reason ``settled_rebalances`` counts finite ICs). NaN when the + index is not dates or fewer than two periods settled — the verdict reports + that as UNKNOWN and FAILS the gate rather than passing on a guess. + """ + settled = ic.dropna() + if len(settled) < 2: + return float("nan") + try: + dates = pd.DatetimeIndex(settled.index) + except (TypeError, ValueError): + return float("nan") + span = dates.max() - dates.min() + return float(span.days) + + +def _shift_within_symbol(series: pd.Series, lag: int) -> pd.Series: + """The panel value as of ``lag`` periods ago, never reaching across symbols.""" + return series.groupby(level=SYMBOL_LEVEL, sort=False).shift(lag) + + +def _mean(series: pd.Series) -> float: + clean = pd.Series(series, dtype=float).replace([np.inf, -np.inf], np.nan).dropna() + return float(clean.mean()) if len(clean) else float("nan") + + +def _residualize_cross_section(y: pd.Series, x: pd.Series) -> pd.Series: + """Per-date cross-sectional OLS residual of ``y`` on ``x`` — NO date loop. + + Single regressor, so the per-date beta is ``cov_t(x, y) / var_t(x)`` and the + whole thing is grouped transforms. A date whose regressor has no variance + yields NaN (the regression is undefined there) rather than a passthrough that + would look like a successful orthogonalization. + """ + frame = ( + pd.DataFrame({"y": y, "x": x}) + .replace([np.inf, -np.inf], np.nan) + .dropna() + ) + if frame.empty: + return pd.Series(dtype=float, index=frame.index) + grouped = frame.groupby(level=DATE_LEVEL, sort=False) + yc = frame["y"] - grouped["y"].transform("mean") + xc = frame["x"] - grouped["x"].transform("mean") + cov = (xc * yc).groupby(level=DATE_LEVEL, sort=False).transform("sum") + var = (xc * xc).groupby(level=DATE_LEVEL, sort=False).transform("sum") + beta = cov / var.where(var > 0) + return (yc - beta * xc).rename("residual") + + +def _residualize_on_book(factor: pd.Series, anchors: pd.DataFrame) -> pd.Series: + """Per-date residual of ``factor`` on the WHOLE anchor book — NO date loop. + + This is the Incremental axis's engine: the residual of the factor after ALL + known factors have been projected out per date, so its IC vs forward returns is + what the factor adds BEYOND the book (design §6, v0.5). Multi-regressor OLS is + done by sequential (modified Gram-Schmidt / Frisch-Waugh) orthogonalization — + each anchor is first residualized on the anchors already processed, then the + running factor residual is residualized on it. Projecting onto an orthogonal + basis of the book's span is EXACTLY the multi-regressor residual, and every + step reuses the vectorized single-regressor :func:`_residualize_cross_section`. + + The only loops are over the ANCHOR COLUMNS (a small constant, like + ``DECAY_LAGS``) — NOT over dates/periods, so this stays loop-free in the sense + design §8 cares about. A perfect collinearity leaves a zero residual, whose IC + is then NaN (an honest "undefined"), never a fabricated number. + """ + residual = factor + basis: list[pd.Series] = [] + for column in anchors.columns: + component = anchors[column].astype(float) + for prior in basis: + component = _residualize_cross_section(component, prior) + basis.append(component) + residual = _residualize_cross_section(residual, component) + return residual + + +class StandardFactorEvaluator(FactorEvaluator): + """Default evaluator: reduce the vectorized eval-IR into the 8 sections. + + Stateless — every section reads only the IR it is handed, so one instance may + evaluate any number of factors. + """ + + # -- the IR seam -------------------------------------------------------- + + def build_ir( + self, + factor_panel: pd.Series | pd.DataFrame, + spec: FactorSpec, + cfg: EvalConfig, + ctx: object | None = None, + ) -> StandardEvalIR: + """Build the design §8 four objects once (see :func:`analytics.eval.ir.build_eval_ir`).""" + if ctx is not None and not isinstance(ctx, EvalContext): + raise TypeError( + f"StandardFactorEvaluator expects an analytics.eval.ir.EvalContext " + f"(price_panel / forward_returns / known_factors / ...); got " + f"{type(ctx).__name__}." + ) + return build_eval_ir(factor_panel, spec, cfg, ctx) + + def evaluate_with_ir( + self, + factor_panel: pd.Series | pd.DataFrame, + spec: FactorSpec, + cfg: EvalConfig, + ctx: object | None = None, + thresholds: VerdictThresholds | None = None, + ) -> tuple[FactorEvalReport, StandardEvalIR]: + """Like :meth:`evaluate`, but also return the IR (for dashboard rendering). + + Mirrors the frozen ABC ``evaluate`` template step for step. It exists + because the contract ``evaluate`` returns ONLY the report (its signature is + frozen), whereas :func:`analytics.eval.figures.render_factor_dashboard` + needs the IR's ``ic`` / ``quantile_returns`` series. Building the IR once + here avoids a second ``build_ir`` pass, and reusing the identical + assemble -> validate -> with_verdict sequence keeps the two entry points in + lockstep (the report is byte-identical to ``evaluate``'s). + """ + ir = self.build_ir(factor_panel, spec, cfg, ctx) + sections = [getattr(self, name)(ir) for name in self.SECTION_ORDER] + report = FactorEvalReport.assemble(spec, cfg, sections, thresholds=thresholds) + report.validate_all_mandatory_present() + return report.with_verdict(thresholds), ir + + # -- 2. Predictive Power ------------------------------------------------ + + def predictive_power(self, ir: StandardEvalIR) -> Section | Skipped: + """Rank IC / ICIR / hypothesis win rate / Newey-West t / IC decay.""" + ic = ir.ic + if ic.notna().sum() == 0: + return Skipped( + "predictive_power", + reason=( + f"no evaluation period produced a finite rank IC across " + f"{ir.n_rebalances} period(s): every cross-section was empty, " + f"smaller than 2 usable (factor, forward-return) pairs, or had " + f"zero variance on one side. There is nothing to summarize." + ), + ) + sign = ir.spec.expected_ic_sign + rank_summary = ic_summary(ic) + pearson_summary = ic_summary(ir.ic_pearson) + nw = newey_west_t(ic) + # N_eff-based CIs (design §6, v0.6). ic_ir_ci_low is what the Predictive + # verdict axis gates on (the LOWER bound in the expected direction), NOT the + # point. IC-mean carries a CI too, reported for the reader but not gated on. + ir_ci = information_ratio_ci(ic, confidence=DEFAULT_CONFIDENCE) + mean_interval = mean_ci(ic, confidence=DEFAULT_CONFIDENCE) + + payload: dict[str, object] = { + "expected_ic_sign": sign, + "ic_mean": rank_summary["ic_mean"], + "ic_mean_se": mean_interval["se"], + "ic_mean_ci_low": mean_interval["ci_low"], + "ic_mean_ci_high": mean_interval["ci_high"], + "ic_std": as_float(ic.dropna().std(ddof=1)) if ic.notna().sum() > 1 else float("nan"), + "ic_ir": rank_summary["ic_ir"], + "ic_ir_se": ir_ci["se"], + "ic_ir_ci_low": ir_ci["ci_low"], + "ic_ir_ci_high": ir_ci["ci_high"], + "ic_ir_ci_confidence": ir_ci["confidence"], + "ic_ir_ci_n_eff": ir_ci["n_eff"], + "ic_win_rate": hypothesis_win_rate(ic, sign), + "ic_nw_t": nw["t"], + "ic_t_iid_unadjusted": nw["t_iid"], + "ic_nw_bandwidth_lags": int(nw["lags"]), + "ic_periods_finite": int(ic.notna().sum()), + "ic_periods_nan_dropped": int(nw["n_dropped"]), + "ic_pearson_mean": pearson_summary["ic_mean"], + "ic_pearson_ir": pearson_summary["ic_ir"], + } + for lag in DECAY_LAGS: + stale = cross_section_corr( + _shift_within_symbol(ir.factor, lag), + ir.forward_returns, + rank=True, + dates=ir.dates, + ) + payload[f"ic_decay_stale_lag_{lag}_mean"] = _mean(stale) + + note = ( + "Rank IC (Spearman) is the primary; the Pearson IC is secondary " + "(design §A). ic_win_rate is the share of finite periods whose IC " + "carries the EXPECTED sign, so 0.5 is a coin flip. " + "ic_nw_t is the Newey-West (Bartlett kernel, automatic bandwidth " + "floor(4*(n/100)^(2/9))) autocorrelation-corrected t of the mean IC; " + "ic_t_iid_unadjusted is the naive t that assumes independent periods " + "and OVERSTATES significance on an autocorrelated IC series — the two " + "are shown together so the gap is visible. The autocovariances are " + "computed ON THE TIME GRID: a NaN period (ic_periods_nan_dropped) " + "breaks the lag pair rather than being dropped first and silently " + "bridged, so a gap never manufactures an autocorrelation that is not " + "there. " + "ic_decay_stale_lag_k_mean is the mean rank IC of the factor AS OF k " + "periods ago against the SAME horizon-h forward return, i.e. how much " + "predictive power a stale signal retains (lag 0 is ic_mean). It is NOT " + "the IC at a longer forward horizon — that would need a second R_h. " + "ic_ir_ci_low/high is the ICIR's N_eff-based 95% confidence interval " + "(Lo Sharpe SE with N->N_eff); the Predictive verdict axis gates on the " + "LOWER bound in the expected direction, not the point, so a wide interval " + "from a noisy or autocorrelated IC series cannot buy a PASS." + ) + return Section("predictive_power", payload=payload, note=note) + + # -- 3. Return & Risk --------------------------------------------------- + + def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: + """Quantile NAVs / net long-short by cost / monotonicity / risk metrics.""" + quantiles = ir.quantile_returns + if quantiles.empty or quantiles.notna().to_numpy().sum() == 0: + return Skipped( + "return_risk", + reason=( + f"no (date, bucket) cell has a realized forward return across " + f"{ir.n_rebalances} period(s) and {ir.cfg.n_quantiles} buckets, " + f"so there is no quantile return matrix to reduce." + ), + ) + sign = ir.spec.expected_ic_sign + top, bottom = ir.cfg.long_short + buckets = list(quantiles.columns) + + bucket_mean = {int(q): as_float(quantiles[q].mean()) for q in buckets} + bucket_nav = { + int(q): as_float((1.0 + quantiles[q].dropna()).prod()) for q in buckets + } + bucket_periods = {int(q): int(quantiles[q].notna().sum()) for q in buckets} + + gross = (quantiles[top] - quantiles[bottom]).rename("spread") + turnover = ir.quantile_turnover + leg_turnover = ( + turnover[top].reindex(gross.index).fillna(0.0) + + turnover[bottom].reindex(gross.index).fillna(0.0) + ) + net_by_cost: dict[float, float] = {} + cumulative_by_cost: dict[float, float] = {} + base_net = None + for multiplier in ir.cfg.cost_scenarios: + net = gross - ir.ctx.fee_rate * multiplier * leg_turnover + net_by_cost[float(multiplier)] = as_float(net.mean()) + cumulative_by_cost[float(multiplier)] = as_float( + (1.0 + net.dropna()).prod() - 1.0 + ) + if abs(multiplier - 1.0) < 1e-9: + base_net = net + + payload: dict[str, object] = { + "long_short_legs": f"Q{top} - Q{bottom}", + "quantile_mean_return": bucket_mean, + "quantile_final_nav": bucket_nav, + "quantile_periods_with_return": bucket_periods, + # RAW Spearman of bucket index vs bucket mean return; the verdict is + # the ONE place that multiplies by expected_ic_sign. + "monotonicity_spearman": spearman( + [float(q) for q in buckets], [bucket_mean[int(q)] for q in buckets] + ), + "gross_long_short_mean": as_float(gross.mean()), + "net_long_short_by_cost": net_by_cost, + "net_long_short_cumulative_by_cost": cumulative_by_cost, + "fee_rate_base": ir.ctx.fee_rate, + "periods_per_year": ir.periods_per_year, + } + # N_eff-based CI of the base-cost QN-Q1 spread (design §6, v0.6). REPORTED + # for the reader (b); the Tradable axis still gates on the POINT base spread + # > 0 (the CI on this leg difference is point-only for now — c names only + # ICIR / incremental ICIR as the lower-CI magnitude criteria). + if base_net is not None: + base_ci = mean_ci(sign * base_net, confidence=DEFAULT_CONFIDENCE) + payload["net_long_short_base_se"] = base_ci["se"] + payload["net_long_short_base_ci_low"] = base_ci["ci_low"] + payload["net_long_short_base_ci_high"] = base_ci["ci_high"] + payload["net_long_short_base_ci_note"] = ( + "hypothesis-aligned base-cost spread CI (N_eff-based); REPORTED, not " + "gated — the Tradable axis PASS reads the POINT base spread > 0." + ) + if ir.periods_per_year_is_default: + payload["periods_per_year_basis"] = ( + f"DEFAULT {ir.periods_per_year}: rebalance label " + f"{ir.cfg.rebalance!r} is not a recognized frequency, so the " + f"annualization below is a fallback, not a derived fact." + ) + + if base_net is not None and base_net.notna().sum() >= 2: + aligned = (sign * base_net).dropna() + nav = (1.0 + aligned).cumprod() + simple = performance_summary(nav, periods_per_year=ir.periods_per_year) + payload["aligned_spread_annual_return"] = simple["annual_return"] + payload["aligned_spread_sharpe"] = simple["sharpe"] + payload["aligned_spread_volatility"] = simple["volatility"] + payload["aligned_spread_max_drawdown"] = simple["max_drawdown"] + payload["aligned_spread_sortino"] = sortino(aligned, ir.periods_per_year) + payload["aligned_spread_final_nav"] = as_float(nav.iloc[-1]) + # report-only cross-check; the simple numbers above stay authoritative + # and the backend is disclosed (INV-007), never silently faked. + cross = quantstats_performance( + aligned, + periods_per_year=ir.periods_per_year, + simple_fallback={ + "cagr": simple["annual_return"], + "sharpe": simple["sharpe"], + "volatility": simple["volatility"], + "max_drawdown": simple["max_drawdown"], + }, + ) + payload["quantstats_cross_check"] = cross + + return Section("return_risk", payload=payload, note=LONG_SHORT_NOTE) + + # -- 4. Stability & Cost ------------------------------------------------ + + def stability_cost(self, ir: StandardEvalIR) -> Section | Skipped: + """Turnover / factor rank autocorrelation -> half-life / cost gradient.""" + if ir.quantile_labels.empty: + return Skipped( + "stability_cost", + reason=( + "no (date, symbol) row carries a finite factor value, so there " + "is no bucket membership to measure turnover on and no " + "cross-section to autocorrelate." + ), + ) + top, bottom = ir.cfg.long_short + turnover = ir.quantile_turnover + + payload: dict[str, object] = { + "turnover_mean_by_quantile": { + int(q): as_float(turnover[q].mean()) for q in turnover.columns + }, + "turnover_mean_long_short_legs": as_float( + (turnover[top] + turnover[bottom]).mean() + ), + "fee_rate_base": ir.ctx.fee_rate, + } + + autocorr: dict[int, float] = {} + for lag in DECAY_LAGS: + rho = cross_section_corr( + ir.factor, _shift_within_symbol(ir.factor, lag), rank=True, dates=ir.dates + ) + autocorr[lag] = _mean(rho) + payload["factor_rank_autocorr_by_lag"] = autocorr + payload["half_life_periods"] = half_life(autocorr.get(1, float("nan"))) + + # The cost GRADIENT (design §4: base / 2x / 4x, never a single point). Read + # off the same net spreads return_risk reports, so the two cannot disagree. + gross = ir.quantile_returns[top] - ir.quantile_returns[bottom] + leg_turnover = ( + turnover[top].reindex(gross.index).fillna(0.0) + + turnover[bottom].reindex(gross.index).fillna(0.0) + ) + base_cost = float("nan") + drag: dict[float, float] = {} + for multiplier in ir.cfg.cost_scenarios: + cost = as_float((ir.ctx.fee_rate * multiplier * leg_turnover).mean()) + drag[float(multiplier)] = cost + if abs(multiplier - 1.0) < 1e-9: + base_cost = cost + payload["mean_cost_per_period_by_scenario"] = drag + payload["extra_cost_vs_base_by_scenario"] = { + m: as_float(c - base_cost) for m, c in drag.items() + } + payload["annualized_cost_drag_by_scenario"] = { + m: as_float(c * ir.periods_per_year) for m, c in drag.items() + } + + note = ( + "Turnover is sum_i |w_i(t) - w_i(t-1)| for an equal-weight bucket " + "(both sides of the trade), and turnover x fee_rate is the period's " + "cost — the project's convention. The FIRST period is charged for " + "establishing the book. factor_rank_autocorr_by_lag is the mean " + "per-date rank correlation of the factor with itself k periods " + "earlier; half_life_periods = log(0.5)/log(rho_1), defined only for " + "0 < rho_1 < 1 (a flipping or non-decaying signal has no half-life and " + "reports NaN). The cost gradient re-charges the SAME trades at k x the " + "fee: turnover and gross returns are identical across scenarios, only " + "the cost line moves." + ) + return Section("stability_cost", payload=payload, note=note) + + # -- 5. Purity ---------------------------------------------------------- + + def purity(self, ir: StandardEvalIR) -> Section | Skipped: + """Correlation with known anchors / orthogonalized IC. VIF is NOT computed.""" + known = ir.ctx.known_factors + if known is not None and not isinstance(known, pd.DataFrame): + # A wrong TYPE is a caller bug, not an absent anchor set: reporting it + # as "was not supplied" would be a misleading diagnostic. + raise TypeError( + f"EvalContext.known_factors must be a MultiIndex(date, symbol) " + f"DataFrame of anchor factors (one column each); got " + f"{type(known).__name__}." + ) + if known is None or len(known.columns) == 0: + return Skipped( + "purity", + reason=( + "EvalContext.known_factors was not supplied, so there is no " + "anchor panel to correlate against, orthogonalize on, or " + "compute a VIF from. Purity cannot be claimed from the factor " + "and its forward returns alone. Pass an already-processed " + "MultiIndex(date, symbol) frame of anchor factors (the project's " + "independently validated value_ep / value_bp / volatility_20, " + "plus classic size / momentum) to populate this section. " + f"Already stripped upstream per EvalConfig: " + f"neutralization={list(ir.cfg.neutralization)!r} " + f"({ir.cfg.industry_level}), winsorize={ir.cfg.winsorize!r}, " + f"standardize={ir.cfg.standardize!r}." + ), + ) + anchors = pd.DataFrame(known) + raw_ic = _mean(ir.ic) + payload: dict[str, object] = { + "family": ir.spec.family, + "neutralization_applied_upstream": list(ir.cfg.neutralization), + "industry_level": ir.cfg.industry_level, + "anchor_factors": [str(c) for c in anchors.columns], + "ic_mean_raw": raw_ic, + } + correlations: dict[str, float] = {} + orthogonal_ic: dict[str, float] = {} + for column in anchors.columns: + anchor = anchors[column].astype(float) + correlations[str(column)] = _mean( + cross_section_corr(ir.factor, anchor, rank=True, dates=ir.dates) + ) + residual = _residualize_cross_section(ir.factor, anchor) + orthogonal_ic[str(column)] = _mean( + cross_section_corr( + residual, ir.forward_returns, rank=True, dates=ir.dates + ) + ) + payload["mean_rank_corr_with_anchor"] = correlations + payload["ic_mean_orthogonalized_vs_anchor"] = orthogonal_ic + + # -- the Incremental axis's fact (design §6, v0.5) ------------------- + # The orthogonalized IC vs the WHOLE book (multi-regressor residual), NOT + # one anchor at a time: "does the factor add anything the book does not + # already have?" The verdict reads incremental_ic_ir exactly as it reads + # ic_ir for the Predictive axis; RAW, so the hypothesis is applied there. + book_residual = _residualize_on_book(ir.factor, anchors) + book_ortho_ic = cross_section_corr( + book_residual, ir.forward_returns, rank=True, dates=ir.dates + ) + book_summary = ic_summary(book_ortho_ic) + # The CI is computed on the ORTHOGONALIZED IC series' OWN N_eff (design §6, + # v0.6): the Incremental axis gates on incremental_ic_ir_ci_low, so the + # residual's own autocorrelation — not the raw IC's — sets the interval. + book_ir_ci = information_ratio_ci(book_ortho_ic, confidence=DEFAULT_CONFIDENCE) + payload["known_factors_supplied"] = True + payload["ic_mean_orthogonalized_vs_book"] = book_summary["ic_mean"] + payload["incremental_ic_mean"] = book_summary["ic_mean"] + payload["incremental_ic_ir"] = book_summary["ic_ir"] + payload["incremental_ic_ir_se"] = book_ir_ci["se"] + payload["incremental_ic_ir_ci_low"] = book_ir_ci["ci_low"] + payload["incremental_ic_ir_ci_high"] = book_ir_ci["ci_high"] + payload["incremental_ic_ir_ci_n_eff"] = book_ir_ci["n_eff"] + + payload["vif"] = None + payload["vif_status"] = ( + "NOT COMPUTED — a VIF needs a MULTI-regressor per-date cross-sectional " + "design; this section orthogonalizes against ONE anchor at a time " + "(vectorized single-regressor OLS). Read the pairwise correlations " + "together: several highly correlated anchors would each individually " + "look survivable here while being collectively collinear." + ) + note = ( + "mean_rank_corr_with_anchor is the mean per-date cross-sectional rank " + "correlation with each anchor. ic_mean_orthogonalized_vs_anchor is the " + "mean rank IC of the factor AFTER removing ONE anchor per date by " + "cross-sectional OLS (residual vs the same R_h); compare it with " + "ic_mean_raw — a collapse means the anchor explained the signal. " + "incremental_ic_ir / incremental_ic_mean drive the INCREMENTAL VERDICT " + "AXIS: they are the ICIR and mean rank IC of the factor residualized on " + "the WHOLE book jointly (multi-regressor, per date) — what the factor " + "adds BEYOND the known set. A value ~ 0 means the factor is redundant " + "with the book (a hard Reject on the Incremental axis), however strong " + "its RAW IC. Anchors are taken as supplied; this evaluator does not " + "verify that they were processed the same way as the factor." + ) + return Section("purity", payload=payload, note=note) + + # -- 6. OOS & Generalization (the project's core lesson) ---------------- + + def oos_generalization(self, ir: StandardEvalIR) -> Section | Skipped: + """Train/test sign consistency by REALIZED date. Drives the Adopt verdict.""" + cfg = ir.cfg + if cfg.oos_split is None: + return Skipped( + "oos_generalization", + reason=( + "cfg.oos_split not set — no OOS evidence. This run measured the " + "factor on ONE window with no holdout, so generalization is NOT " + "established and the verdict cannot reach Adopt. The project's " + "standing lesson (P3-3/P3-4: IC signs flipped train->test in " + "almost every cell) is that in-sample strength does not " + "extrapolate." + ), + ) + try: + split = pd.Timestamp(cfg.oos_split) + except (TypeError, ValueError) as exc: + # Loud, not a silent downgrade: a typo'd split must never be reported + # as "we chose not to do OOS". + raise ValueError( + f"EvalConfig.oos_split={cfg.oos_split!r} is not a parseable date, so " + f"the train/test boundary is undefined. Silently degrading to 'no " + f"OOS split' would report a CONFIG TYPO as a deliberate choice." + ) from exc + + sign = ir.spec.expected_ic_sign + realized = pd.to_datetime(ir.realized_date, errors="coerce") + is_test = realized >= split + is_train = realized < split + unrealized = realized.isna() + + train_ic = ir.ic[is_train.to_numpy()] + test_ic = ir.ic[is_test.to_numpy()] + train_mean, test_mean = _mean(train_ic), _mean(test_ic) + + payload: dict[str, object] = { + "oos_split": str(cfg.oos_split), + "split_basis": "realized date (t+h), never the signal date", + "train_periods_settled": int(train_ic.notna().sum()), + "test_periods_settled": int(test_ic.notna().sum()), + "periods_never_realized": int(unrealized.sum()), + "expected_ic_sign": sign, + "train_ic_mean": train_mean, + "test_ic_mean": test_mean, + "independent_cells_declared": len(cfg.independent_cells), + "independent_cells_evaluated": 0, + } + + available = train_ic.notna().sum() > 0 and test_ic.notna().sum() > 0 + payload["oos_available"] = bool(available) + + # The holdout, split in two by realized date: the contract's Adopt rule + # wants the expected sign in BOTH holdout subperiods (design §6), not just + # on the holdout average, which one lucky stretch could carry. + test_dates = list(ir.dates[is_test.to_numpy()]) + first_mean = second_mean = float("nan") + if len(test_dates) >= 2: + middle = len(test_dates) // 2 + first_mean = _mean(test_ic.reindex(test_dates[:middle])) + second_mean = _mean(test_ic.reindex(test_dates[middle:])) + payload["holdout_subperiod_1_ic_mean"] = first_mean + payload["holdout_subperiod_2_ic_mean"] = second_mean + + sign_consistent = bool( + available + and math.isfinite(first_mean) + and math.isfinite(second_mean) + and sign * first_mean > 0 + and sign * second_mean > 0 + ) + sign_flipped = bool( + available and math.isfinite(test_mean) and sign * test_mean < 0 + ) + test_monotonicity = float("nan") + if available: + test_quantiles = ir.quantile_returns[is_test.to_numpy()] + bucket_means = [as_float(test_quantiles[q].mean()) for q in test_quantiles.columns] + test_monotonicity = spearman( + [float(q) for q in test_quantiles.columns], bucket_means + ) + + payload["sign_consistent"] = sign_consistent + payload["sign_flipped"] = sign_flipped + payload["test_monotonicity_spearman"] = test_monotonicity + payload["test_monotonicity_aligned"] = as_float(sign * test_monotonicity) + # The contract's monotonicity_reversed means an INDEPENDENT-CELL reversal + # (verdict.py: "# independent-cell reversal (I5e)", and its Reject reason + # literally reads "reversed on an independent cell"). This evaluator scores + # ONE cell and has just reported independent_cells_evaluated=0, so writing + # a SAME-CELL reversal into that key would make the report claim an + # independent check ran and failed when none ran at all — a fabricated + # provenance in the very payload that discloses zero independent cells. + # Left False: unknown is not evidence, and the contract is explicit that an + # absent fact must never manufacture a hard Reject. The same-cell number is + # reported above as test_monotonicity_spearman, under its own honest name. + payload["monotonicity_reversed"] = False + payload["monotonicity_reversed_status"] = ( + "NOT ASSESSED — this key means a reversal on an INDEPENDENT cell " + "(different universe / non-overlapping window; the I5e signature: " + "CSI500 +1.0 -> CSI300 -0.5). This evaluator scores ONE cell and cannot " + "produce that evidence; a declared independent universe must be " + "evaluated as its OWN run and compared across runs. The same-cell " + "holdout figure is test_monotonicity_spearman " + f"({test_monotonicity:.4f}), aligned to the hypothesis " + f"({sign * test_monotonicity:+.4f}) — read it as a diagnostic, NOT as " + f"independent-cell evidence." + ) + + note = ( + "Periods are assigned to train/test by the REALIZED date (t+h), never " + "the signal date: a signal given at t is only out-of-sample once t+h " + "has happened (P3-3 fixed exactly this, and the fix moved the numbers " + "materially). sign_consistent requires the EXPECTED IC sign in BOTH " + "halves of the holdout, not merely on its average. sign_flipped means " + "the holdout mean IC CONTRADICTS the stated hypothesis — a hard " + "Reject. This evaluator scores ONE cell: independent_cells is a HUMAN " + "declaration it can neither run nor verify, so " + "independent_cells_evaluated is always 0 here and " + "monotonicity_reversed (which the contract defines as an " + "INDEPENDENT-cell reversal) is therefore always left False — see " + "monotonicity_reversed_status. A holdout split inside ONE " + "window/universe is NOT independent generalization: a declared " + "independent universe must be evaluated as its own run." + ) + return Section("oos_generalization", payload=payload, note=note) + + # -- 7. Execution & Capacity -------------------------------------------- + + def execution_capacity(self, ir: StandardEvalIR) -> Section | Skipped: + """I5b fill feasibility / I5f capacity — reported ONLY if measured elsewhere.""" + facts = ir.ctx.execution_capacity + if facts is None: + return Skipped( + "execution_capacity", + reason=( + f"NOT WIRED: this evaluator reduces the eval-IR (a factor panel " + f"and its forward returns) and never runs the backtest engine, so " + f"it observes neither I5b raw stk_limit fill feasibility " + f"(cfg.limit_feasibility={ir.cfg.limit_feasibility}) nor the I5f " + f"single-minute capacity diagnostic " + f"(cfg.capacity_notional={ir.cfg.capacity_notional}, " + f"max_participation_rate={ir.cfg.max_participation_rate}). Both " + f"need minute bars, raw stk_limit rows and the event engine — " + f"none of which is an IR input. A capacity number invented from " + f"the factor panel alone would be a fiction. Supply facts " + f"MEASURED by a P-I5b / P-I5f run via " + f"EvalContext.execution_capacity to populate this section. " + f"CONSEQUENCE: tradability is UNKNOWN, so this run cannot reach " + f"the Adopt verdict — an untested execution path is not evidence " + f"of tradability." + ), + ) + if not isinstance(facts, Mapping): + raise TypeError( + f"EvalContext.execution_capacity must be a mapping of measured " + f"execution facts (recognized: 'tradable', 'capacity_sufficient'); " + f"got {type(facts).__name__}." + ) + payload: dict[str, object] = dict(facts) + payload["source"] = ( + "MEASURED OUTSIDE this evaluator and supplied via " + "EvalContext.execution_capacity; these numbers are reported, not " + "produced here." + ) + for key in ("tradable", "capacity_sufficient"): + if not isinstance(facts.get(key), bool): + # Left absent/unknown on purpose: report.extract_verdict_inputs + # reads a non-bool as None, i.e. unknown, which can never earn an + # Adopt. Say so rather than let a missing key look measured. + payload[f"{key}_status"] = ( + f"UNKNOWN — EvalContext.execution_capacity carries no boolean " + f"{key!r}; the verdict treats it as not established." + ) + note = ( + "This section is a PASS-THROUGH of facts measured by the execution " + "machinery (I5b raw stk_limit fill gating / I5f single-minute capacity " + "at a target notional). The evaluator does not verify them, does not " + "re-run the engine, and cannot detect a stale or mismatched run." + ) + return Section("execution_capacity", payload=payload, note=note) + + # -- 8. Data & Coverage ------------------------------------------------- + + def data_coverage(self, ir: StandardEvalIR) -> Section | Skipped: + """Coverage / dropped symbols / NaN rates / cross-section sizes / sample size.""" + factor = ir.factor + total = int(len(factor)) + symbols = factor.index.get_level_values(SYMBOL_LEVEL) + evaluated = pd.Index(pd.unique(symbols)) + + finite = np.isfinite(factor.to_numpy(dtype=float)) + warmup = ir.warmup_mask.to_numpy(dtype=bool) + post_warmup = int((~warmup).sum()) + + # The verdict's THREE-PART sample gate (design §6, v0.3) reads + # settled_rebalances + effective_samples + span_days. All three come from + # the IR's IC series, computed ONCE here — a raw count is not a sample + # size under a daily rebalance. + ess = effective_sample_size(ir.ic) + span = _ic_span_days(ir.ic) + + payload: dict[str, object] = { + # -- the sample gate's three facts -- + "settled_rebalances": ir.settled_rebalances, + "effective_samples": as_float(ess["n_eff"]), + "span_days": span, + # how N_eff was reached, so the gate's arithmetic is auditable rather + # than a bare number the reader must trust. + "effective_samples_lags": as_float(ess["lags"]), + "effective_samples_lags_nw_floor": as_float(ess["lags_nw"]), + "effective_samples_sum_rho": as_float(ess["sum_rho"]), + "effective_samples_denominator": as_float(ess["denominator"]), + "effective_samples_note": ( + str(ess["status"]) + or ( + "N_eff = N / (1 + 2*sum_k rho_k) over the IC series, lags " + "truncated at the Newey-West floor then extended while rho>0, " + "clamped to [1, N]." + ) + ), + "evaluation_periods": ir.n_rebalances, + "declared_rebalance": ir.cfg.rebalance, + "median_period_gap_days": as_float(ir.median_period_gap_days), + "rebalance_grid_check": ir.rebalance_grid_check, + "symbols_evaluated": int(len(evaluated)), + "panel_rows": total, + "universe_is_pit": ir.cfg.universe_is_pit, + "forward_return_source": ir.forward_return_source, + "factor_nan_rate": as_float(1.0 - finite.sum() / total) if total else float("nan"), + "factor_nan_rate_excluding_warmup": ( + as_float(1.0 - (finite & ~warmup).sum() / post_warmup) + if post_warmup + else float("nan") + ), + "warmup_rows_excluded": int(warmup.sum()), + "min_history_bars": ir.spec.min_history_bars, + "forward_return_nan_rate": as_float( + 1.0 - np.isfinite(ir.forward_returns.to_numpy(dtype=float)).sum() / total + ) + if total + else float("nan"), + "cross_section_size_mean": as_float(ir.cross_section_size.mean()), + "cross_section_size_min": as_float(ir.cross_section_size.min()), + "cross_section_size_median": as_float(ir.cross_section_size.median()), + "cross_section_size_max": as_float(ir.cross_section_size.max()), + "periods_with_empty_cross_section": int((ir.cross_section_size == 0).sum()), + "input_fields_declared": list(ir.spec.input_fields), + } + + declared = ir.ctx.universe_symbols + if declared: + dropped = sorted(set(map(str, declared)) - set(map(str, evaluated))) + payload["universe_symbols_declared"] = len(declared) + payload["dropped_symbols_count"] = len(dropped) + payload["dropped_symbols_share"] = as_float(len(dropped) / len(declared)) + payload["dropped_symbols_examples"] = dropped[:5] + else: + payload["universe_symbols_declared"] = 0 + payload["dropped_symbols_count"] = None + payload["dropped_symbols_status"] = ( + "NOT ASSESSED — EvalContext.universe_symbols was not supplied, so " + "the evaluator cannot tell which universe members never reached the " + "factor panel. A silently dropped name is a coverage bias (I5d " + "dropped 103 of 995 minute-uncovered constituents and had to say so)." + ) + + note = ( + "settled_rebalances counts periods with a FINITE rank IC — the periods " + "that actually produced evidence; the last h periods have no realized " + "forward return and never settle. ONE ROW OF THE FACTOR PANEL IS ONE " + "EVALUATION PERIOD: this evaluator does NOT resample. It instead CHECKS " + "the supplied spacing against declared_rebalance and refuses to run on a " + "grid that contradicts it (see rebalance_grid_check), because " + "min_rebalances — the gate that decides INSUFFICIENT-DATA — would " + "otherwise pass on a sample that does not exist at the declared " + "frequency. A 'NOT CHECKED' outcome means the declaration is UNVERIFIED " + "and the sample size below is simply the rows supplied. " + + ( + "universe_is_pit=False: constituents are NOT point-in-time, so these " + "results carry survivorship bias." + if not ir.cfg.universe_is_pit + else "universe_is_pit=True: constituents are point-in-time." + ) + ) + return Section("data_coverage", payload=payload, note=note) + + # -- 9. Caveats & Provenance -------------------------------------------- + + def caveats(self, ir: StandardEvalIR) -> Section | Skipped: + """Post-hoc / exploratory / tuned / multiple testing / sample size.""" + cfg, spec = ir.cfg, ir.spec + items: list[str] = [] + if cfg.is_exploratory: + items.append( + "EXPLORATORY: this is not a return claim and must not be read as one." + ) + if cfg.post_hoc_selected: + items.append( + "POST-HOC SELECTED: the factor was chosen after seeing results on " + "(some of) this data, so this run quantifies it — it does not " + "CONFIRM it. Independent confirmation needs a window and/or universe " + "that took no part in the screening (P3-6 -> P3-7)." + ) + if cfg.tuned: + items.append( + "TUNED: at least one parameter was fitted, so in-sample fit is " + "optimistic by construction." + ) + if not cfg.universe_is_pit: + items.append( + "NON-PIT UNIVERSE: constituents are not point-in-time — survivorship " + "bias is present and not corrected." + ) + if cfg.oos_split is None: + items.append( + "NO OOS SPLIT: a single window with no holdout. The project's own " + "record (P3-2's single-year outperformance did not survive P3-3/P3-4) " + "is that this cannot be extrapolated." + ) + if len(cfg.independent_cells) == 0: + items.append( + "NO INDEPENDENT CELL DECLARED: even a passing OOS split inside ONE " + "window/universe is not independent generalization (I5d held on " + "CSI500 and reversed on CSI300 in I5e)." + ) + items.append( + "SYNTHETIC LONG-SHORT: the QN-Q1 spread is a long-only leg difference, " + "not a dollar-neutral executed portfolio." + ) + if ir.settled_rebalances < 24: + items.append( + f"SMALL SAMPLE: {ir.settled_rebalances} settled rebalance(s). The " + f"project has repeatedly seen ~21-period cells reverse each other." + ) + + payload: dict[str, object] = { + "caveats": items, + "is_exploratory": cfg.is_exploratory, + "post_hoc_selected": cfg.post_hoc_selected, + "tuned": cfg.tuned, + "universe_is_pit": cfg.universe_is_pit, + "n_factors_screened": cfg.n_factors_screened, + "independent_cells_declared": len(cfg.independent_cells), + "settled_rebalances": ir.settled_rebalances, + "factor_version": spec.version, + "data_snapshot_id": cfg.data_snapshot_id, + "window": f"{cfg.start} .. {cfg.end}", + } + screened = cfg.n_factors_screened + if screened: + payload["bonferroni_alpha_for_5pct_family"] = as_float(0.05 / screened) + payload["multiple_testing_note"] = ( + f"{screened} factor(s) were screened to arrive at this one. A " + f"nominal 5% test over {screened} candidates expects " + f"{0.05 * screened:.2f} false positive(s); the Bonferroni-corrected " + f"per-factor alpha is {0.05 / screened:.5f}. The reported " + f"Newey-West t is NOT corrected for this." + ) + else: + payload["multiple_testing_note"] = ( + "n_factors_screened is None: the multiple-testing background is " + "UNDECLARED, so the reported significance cannot be discounted for " + "how many candidates were looked at. Absence of a declaration is not " + "evidence that only one factor was tried." + ) + return Section("caveats", payload=payload) + + +__all__ = ["LONG_SHORT_NOTE", "StandardFactorEvaluator"] diff --git a/analytics/eval/stats.py b/analytics/eval/stats.py new file mode 100644 index 0000000..29c5c70 --- /dev/null +++ b/analytics/eval/stats.py @@ -0,0 +1,417 @@ +"""Statistics the eval sections reduce the IR with — pure, small, testable. + +Only what the rest of ``analytics`` does not already provide: +``analytics/performance.py`` supplies annual return / max drawdown / volatility / +Sharpe from a nav, and ``analytics/factor.py`` supplies ``ic_summary``; this +module adds the pieces specific to judging a FACTOR rather than a portfolio. + +The headline one is :func:`newey_west_t`. Design §A: "IC t 必须自相关校正 +(NW/block)". An IC series is autocorrelated (today's cross-sectional ranking +overlaps heavily with yesterday's), and the textbook IID t divides by a standard +error that pretends the periods are independent — so it OVERSTATES significance, +sometimes by a lot. Both are reported side by side so the gap is visible instead +of implied. +""" + +from __future__ import annotations + +import math + +import numpy as np +import pandas as pd + + +def newey_west_lag(n: int) -> int: + """Newey-West (1994) automatic bandwidth: ``floor(4 * (n/100)^(2/9))``. + + The usual default. Clamped to ``[0, n-1]``: a bandwidth at or beyond the + sample length has no lagged pairs left to estimate. + """ + if n < 2: + return 0 + return max(0, min(int(math.floor(4.0 * (n / 100.0) ** (2.0 / 9.0))), n - 1)) + + +def newey_west_t(series: pd.Series, lags: int | None = None) -> dict[str, float]: + """Autocorrelation-corrected t of the MEAN of ``series`` (e.g. an IC series). + + Uses the Bartlett-kernel HAC variance + + var = gamma_0 + 2 * sum_{j=1..L} (1 - j/(L+1)) * gamma_j + + with ``gamma_j`` the sample autocovariance at lag j, and ``se = sqrt(var/n)``. + The loop is over the ~4 BANDWIDTH LAGS, not over the periods. + + GAPS ARE HANDLED ON THE TIME GRID, NOT BY DROPPING FIRST. An IC series has + holes: a degenerate cross-section here and there yields NaN. Dropping those + and *then* lagging would pair observations that are j SURVIVING ROWS apart + while calling them j PERIODS apart — silently bridging every hole and + reporting an autocovariance that does not exist. Since the whole reason this + statistic exists is that the naive IID t overstates significance on an + autocorrelated series, a quietly wrong autocovariance here would defeat its + purpose. Instead the series is kept on its own time grid and the deviations + are ZERO-FILLED at the holes, so any lagged product touching a hole vanishes: + the pair is DROPPED rather than bridged. Each ``gamma_j`` is divided by ``n`` + (the count of finite observations), the standard amplitude-modulation + convention, which keeps the Bartlett kernel's variance non-negative. + + Returns + ------- + ``{"t": , "t_iid": , "mean", "se", "se_iid", "lags", "n", + "n_dropped"}``. Non-estimable pieces are NaN, never a fabricated 0. + """ + raw = pd.Series(series, dtype=float).replace([np.inf, -np.inf], np.nan) + values = raw.to_numpy(dtype=float) + valid = np.isfinite(values) + n = int(valid.sum()) + nan = float("nan") + out = { + "t": nan, "t_iid": nan, "mean": nan, "se": nan, "se_iid": nan, + "lags": float(0), "n": float(n), "n_dropped": float(len(values) - n), + } + if n < 2: + return out + + observed = values[valid] + mean = float(observed.mean()) + out["mean"] = mean + + std = float(observed.std(ddof=1)) + if math.isfinite(std) and std > 0: + out["se_iid"] = std / math.sqrt(n) + out["t_iid"] = mean / out["se_iid"] + + bandwidth = newey_west_lag(n) if lags is None else max(0, min(int(lags), n - 1)) + out["lags"] = float(bandwidth) + + # Zero-filled deviations ON THE TIME GRID: dev[t] is 0 wherever the period is + # missing, so dev[t] * dev[t-j] is 0 unless BOTH periods are observed -- a hole + # breaks the lag pair instead of bridging it. + deviations = np.where(valid, values - mean, 0.0) + variance = float(deviations @ deviations) / n # gamma_0 + for j in range(1, bandwidth + 1): + gamma_j = float(deviations[j:] @ deviations[:-j]) / n + variance += 2.0 * (1.0 - j / (bandwidth + 1.0)) * gamma_j + # A Bartlett-kernel HAC variance is guaranteed non-negative, but it CAN come + # out at (numerically) zero — a constant series, or one whose autocovariances + # cancel. Report NaN rather than an infinite t. + if not math.isfinite(variance) or variance <= 0: + return out + se = math.sqrt(variance / n) + out["se"] = se + out["t"] = mean / se + return out + + +def effective_sample_size( + series: pd.Series, lags: int | None = None +) -> dict[str, float | str]: + """Effective sample size of an autocorrelated series: ``N / (1 + 2*sum_k rho_k)``. + + Gate part A of design §6 (v0.3). A RAW COUNT IS NOT A SAMPLE SIZE: consecutive + daily IC observations overlap heavily, so 500 raw points can carry only a few + dozen independent ones. This is block averaging — correlated MD frames are not + independent samples either — applied to the IC series. + + LAG TRUNCATION: ``newey_west_lag(n)`` is the FLOOR, then the window is + EXTENDED while rho-hat stays positive (Geyer's initial positive sequence), + capped at ``n-1``. + + Why not the NW bandwidth alone, as the HAC t uses? Because it is the wrong + tool for THIS quantity and fails in the PERMISSIVE direction. NW(1994)'s + ``4*(n/100)^(2/9)`` is 5 lags at n=500 — fine for a HAC variance, but an + AR(1) with rho=0.95 has an integrated autocorrelation time of + (1+rho)/(1-rho) ~ 39, so truncating at 5 measures N_eff ~ 53 when the truth + is ~13. A gate fed a 4x-too-large N_eff is not a gate. Sharing the + bandwidth as a FLOOR keeps the two statistics from disagreeing about the + autocorrelation structure in the direction that matters (the ESS window is + never SHORTER than the NW-t's view of it, and both read the same rho-hats + off the same grid with the same gap rule); it simply does not stop looking + while the series is still visibly correlated. An explicit ``lags`` disables + the extension — the caller chose the truncation. + + GAPS: identical convention to :func:`newey_west_t` — deviations are ZERO-FILLED + on the time grid, so a lagged product touching a hole vanishes and the PAIR IS + DROPPED rather than bridged. Never resampled, never dropped-then-lagged. + + GUARDS — the result is ALWAYS a finite, non-negative number, never NaN/inf/negative: + * no finite observation -> 0.0 (an empty sample is 0 effective samples). + * constant series (gamma_0 = 0) -> 1.0. It carries exactly one distinct + value, i.e. one sample's worth of information: the perfectly-correlated + limit, and the 0/0 the ratio would otherwise be. + * ``1 + 2*sum_k rho_k <= 0`` (a net anti-correlated or merely noisy rho-hat) + -> clamped to N, the same answer as the tiny-positive-denominator case it + is the continuation of. The estimator is saying "at least as informative as + i.i.d."; we credit exactly i.i.d. and no more. + * ``N_eff`` is CLAMPED TO [1, N]. You cannot hold more independent + observations than observations, and crediting anti-correlation with + N_eff > N is precisely how a noisy rho-hat would buy evidence a run does + not have. Clamping DOWN is the conservative direction for a gate that + requires N_eff to be LARGE. + + Returns + ------- + ``{"n_eff", "n", "lags", "lags_nw", "sum_rho", "denominator", "status"}``. + ``status`` is "" unless a guard fired, and says which. + """ + raw = pd.Series(series, dtype=float).replace([np.inf, -np.inf], np.nan) + values = raw.to_numpy(dtype=float) + valid = np.isfinite(values) + n = int(valid.sum()) + nan = float("nan") + out: dict[str, float | str] = { + "n_eff": float(n), "n": float(n), "lags": 0.0, "lags_nw": 0.0, + "sum_rho": nan, "denominator": nan, "status": "", + } + if n == 0: + out["n_eff"] = 0.0 + out["status"] = "no finite observation: 0 effective samples." + return out + if n == 1: + out["n_eff"] = 1.0 + out["status"] = "a single finite observation." + return out + + mean = float(values[valid].mean()) + # Zero-filled deviations ON THE TIME GRID (see newey_west_t): a hole breaks + # the lag pair instead of bridging it. + deviations = np.where(valid, values - mean, 0.0) + # All autocovariances in ONE vectorized pass (lags 0..len-1); the only loop + # anywhere here is numpy's, and never over rebalance periods. + gammas = np.correlate(deviations, deviations, mode="full")[len(deviations) - 1 :] / n + gamma_0 = float(gammas[0]) + if not math.isfinite(gamma_0) or gamma_0 <= 0: + out["n_eff"] = 1.0 + out["status"] = ( + "constant series (zero variance): one distinct value carries one " + "sample's worth of information." + ) + return out + + rho = gammas / gamma_0 + # Cap lags at n-1 exactly like newey_west_t does. + max_lag = min(len(deviations) - 1, n - 1) + bandwidth_nw = ( + newey_west_lag(n) if lags is None else max(0, min(int(lags), n - 1)) + ) + out["lags_nw"] = float(bandwidth_nw) + window = bandwidth_nw + if lags is None and bandwidth_nw < max_lag: + # extend past the NW floor while the autocorrelation is still positive + tail = rho[bandwidth_nw + 1 : max_lag + 1] + non_positive = np.flatnonzero(tail <= 0) + window += int(non_positive[0]) if non_positive.size else int(tail.size) + out["lags"] = float(window) + + sum_rho = float(rho[1 : window + 1].sum()) if window >= 1 else 0.0 + out["sum_rho"] = sum_rho + denominator = 1.0 + 2.0 * sum_rho + out["denominator"] = denominator + if not math.isfinite(denominator) or denominator <= 0: + out["n_eff"] = float(n) + out["status"] = ( + f"1 + 2*sum(rho) = {denominator:.4f} <= 0 (net anti-correlated or a " + f"noisy rho-hat): clamped to N={n}. Anti-correlation is never credited " + f"with more independent samples than observations." + ) + return out + + n_eff = n / denominator + clamped = min(max(n_eff, 1.0), float(n)) + out["n_eff"] = float(clamped) + if clamped != n_eff: + out["status"] = f"N_eff {n_eff:.2f} clamped into [1, {n}]." + return out + + +#: Two-sided normal z-scores for the confidence levels the CI helpers support. +#: A small table rather than a dependency on scipy — the eval layer is pure +#: numpy/pandas, and these three levels cover every use. +_Z_TWO_SIDED: dict[float, float] = { + 0.90: 1.6448536269514722, + 0.95: 1.959963984540054, + 0.99: 2.5758293035489004, +} +#: The project's standing confidence level for the verdict CIs (design §6, v0.6). +DEFAULT_CONFIDENCE = 0.95 + + +def _z_for(confidence: float) -> float: + """The two-sided normal z for ``confidence``, or a readable error.""" + z = _Z_TWO_SIDED.get(round(float(confidence), 4)) + if z is None: + raise ValueError( + f"confidence must be one of {sorted(_Z_TWO_SIDED)} (a normal-approx CI " + f"table, no scipy dependency); got {confidence!r}." + ) + return z + + +def mean_ci( + series: pd.Series, confidence: float = DEFAULT_CONFIDENCE +) -> dict[str, float]: + """CI of the MEAN of an autocorrelated series, using N_eff (design §6, v0.6). + + ``SE(mean) = std / sqrt(N_eff)`` — the standard error DEFLATED by the effective + sample size (:func:`effective_sample_size`), not the raw count, so an + autocorrelated IC / spread series gets the WIDER interval its overlapping + observations deserve. CI = mean +/- z * SE at the stated confidence. + + Returns ``{point, se, ci_low, ci_high, n_eff, z, confidence}``; every numeric + piece is NaN when fewer than two finite observations exist or the std is zero. + """ + z = _z_for(confidence) + clean = pd.Series(series, dtype=float).replace([np.inf, -np.inf], np.nan).dropna() + nan = float("nan") + out = { + "point": nan, "se": nan, "ci_low": nan, "ci_high": nan, + "n_eff": nan, "z": z, "confidence": float(confidence), + } + if len(clean) < 2: + out["point"] = float(clean.mean()) if len(clean) else nan + return out + mean = float(clean.mean()) + std = float(clean.std(ddof=1)) + n_eff = float(effective_sample_size(clean)["n_eff"]) + out["point"] = mean + out["n_eff"] = n_eff + if not math.isfinite(std) or std <= 0 or not math.isfinite(n_eff) or n_eff <= 0: + return out + se = std / math.sqrt(n_eff) + out["se"] = se + out["ci_low"] = mean - z * se + out["ci_high"] = mean + z * se + return out + + +def information_ratio_ci( + series: pd.Series, confidence: float = DEFAULT_CONFIDENCE +) -> dict[str, float]: + """CI of the INFORMATION RATIO (ICIR = mean/std) of an IC series, using N_eff. + + Design §6, v0.6. The ICIR is a Sharpe ratio of the IC series, so its standard + error is the Lo (2002) iid-Sharpe SE + + SE(IR) = sqrt((1 + 0.5 * IR^2) / N_eff) + + with the sample count N replaced by the EFFECTIVE count N_eff + (:func:`effective_sample_size`) to absorb the IC series' autocorrelation — the + ``0.5 * IR^2`` term is the extra uncertainty from estimating the denominator + (std) as well as the mean. CI = IR +/- z * SE. This is the LOWER bound the + verdict gates on (:mod:`analytics.eval.verdict`). + + Returns ``{point, se, ci_low, ci_high, n_eff, z, confidence}``; NaN pieces when + the IR itself is undefined (< 2 finite obs, or a zero-variance IC series). + """ + z = _z_for(confidence) + clean = pd.Series(series, dtype=float).replace([np.inf, -np.inf], np.nan).dropna() + nan = float("nan") + out = { + "point": nan, "se": nan, "ci_low": nan, "ci_high": nan, + "n_eff": nan, "z": z, "confidence": float(confidence), + } + if len(clean) < 2: + return out + mean = float(clean.mean()) + std = float(clean.std(ddof=1)) + n_eff = float(effective_sample_size(clean)["n_eff"]) + out["n_eff"] = n_eff + if not math.isfinite(std) or std <= 0: + return out # a zero-variance IC series has no ICIR (and no CI) + ir = mean / std + out["point"] = ir + if not math.isfinite(ir) or not math.isfinite(n_eff) or n_eff <= 0: + return out + se = math.sqrt((1.0 + 0.5 * ir * ir) / n_eff) + out["se"] = se + out["ci_low"] = ir - z * se + out["ci_high"] = ir + z * se + return out + + +def hypothesis_win_rate(series: pd.Series, expected_sign: int) -> float: + """Share of finite periods whose value carries the EXPECTED sign. + + ``VERDICT_KEYS`` documents ``ic_win_rate`` as hypothesis-relative BY + DEFINITION, so the sign is applied here and not by the verdict. Exactly zero + is not the expected sign (it is not evidence for the hypothesis). NaN when no + finite period exists — never a 0.0 that would read as "always wrong". + """ + clean = pd.Series(series, dtype=float).replace([np.inf, -np.inf], np.nan).dropna() + if clean.empty: + return float("nan") + return float((expected_sign * clean > 0).mean()) + + +def half_life(autocorr: float) -> float: + """Periods for a signal with lag-1 autocorrelation ``autocorr`` to halve. + + Assumes the AR(1) decay ``rho^k``: ``log(0.5) / log(rho)``. Defined only for + ``0 < rho < 1``; a non-positive rho (the signal flips rather than decays) or a + rho >= 1 (no decay) has no half-life and returns NaN rather than a number that + would be read as one. + """ + if not isinstance(autocorr, (int, float)) or not math.isfinite(autocorr): + return float("nan") + if not 0.0 < autocorr < 1.0: + return float("nan") + return float(math.log(0.5) / math.log(autocorr)) + + +def sortino(returns: pd.Series, periods_per_year: int) -> float: + """Annualized Sortino: mean / downside deviation * sqrt(ppy). + + Downside deviation uses the returns BELOW zero, with the sum divided by the + FULL sample size (the standard definition — a strategy that is rarely + negative should be rewarded for it, not have its few bad periods averaged + among themselves). NaN when there is no downside at all: the ratio is + genuinely undefined, and +inf would render as a spectacular fake. + """ + clean = pd.Series(returns, dtype=float).replace([np.inf, -np.inf], np.nan).dropna() + if len(clean) < 2: + return float("nan") + downside = clean.where(clean < 0, 0.0) + dd = math.sqrt(float((downside**2).sum()) / len(clean)) + if not math.isfinite(dd) or dd <= 0: + return float("nan") + return float(clean.mean() / dd * math.sqrt(periods_per_year)) + + +def spearman(x: pd.Series | list, y: pd.Series | list) -> float: + """Spearman correlation of two short aligned vectors (NaN pairs dropped). + + Used for the bucket-index vs bucket-return monotonicity. RAW: the hypothesis + is applied by the verdict, never here. + """ + pair = ( + pd.DataFrame({"x": pd.Series(list(x), dtype=float), "y": pd.Series(list(y), dtype=float)}) + .replace([np.inf, -np.inf], np.nan) + .dropna() + ) + if len(pair) < 2 or pair["x"].nunique() < 2 or pair["y"].nunique() < 2: + return float("nan") + return float(pair["x"].corr(pair["y"], method="spearman")) + + +def as_float(value: object) -> float: + """Coerce to a plain float for a payload; anything unusable becomes NaN.""" + try: + out = float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return float("nan") + return out if math.isfinite(out) else float("nan") + + +__all__ = [ + "DEFAULT_CONFIDENCE", + "as_float", + "effective_sample_size", + "half_life", + "hypothesis_win_rate", + "information_ratio_ci", + "mean_ci", + "newey_west_lag", + "newey_west_t", + "sortino", + "spearman", +] diff --git a/analytics/eval/verdict.py b/analytics/eval/verdict.py new file mode 100644 index 0000000..3a7edc0 --- /dev/null +++ b/analytics/eval/verdict.py @@ -0,0 +1,940 @@ +"""Verdict rules: three independent axis-verdicts + a derived deployment label. + +Design doc ``tmp/design/factor_eval_contract_v0.1.md`` §6 (v0.5). The RULE +STRUCTURE is fixed (a factual check — no eyeballing a factor into "promising"); +the THRESHOLDS are config (:class:`VerdictThresholds`). + +WHY THREE AXES (v0.5 — the multi-factor point). Judging a factor in ISOLATION is +the wrong unit for a cross-sectional MULTI-FACTOR book. A single scalar verdict +answered "is there a signal?" but never "does it add anything the book does not +already have?" — so a factor that merely re-expresses value_ep would sail through +on its own raw IC. The verdict is now split into three questions that are +answered INDEPENDENTLY and reported side by side: + + Predictive is there a real, OUT-OF-SAMPLE predictive signal? + Incremental does it add alpha BEYOND the known factor set? (the new axis) + Tradable can the signal actually be harvested? + +Each axis takes one of four states: ``PASS`` / ``FAIL`` / ``INSUFFICIENT_DATA`` / +``NOT_ASSESSED``. A single DEPLOYMENT LABEL (Adopt / Watch / Reject / +INSUFFICIENT-DATA) is DERIVED from the three (see :func:`_derive_deployment`), +preserving the old asymmetric gate + exploratory cap that the scalar verdict won. + +THE DEPLOYMENT DERIVATION (design §6 table) + + any axis FAIL -> REJECT (FAIL is evaluated FIRST, so + it BYPASSES the sample gate: + a thin-sample sign-flip still + Rejects) + no FAIL, all three PASS -> ADOPT (capped to WATCH if the run + declares itself exploratory) + no FAIL, >=1 PASS, rest unresolved -> WATCH (reason names the unresolved + axes) + no FAIL, no PASS -> INSUFFICIENT-DATA + + ``default run -> at most WATCH`` is INTENDED: with no known_factors the + Incremental axis is NOT_ASSESSED and with no execution facts the Tradable axis + is NOT_ASSESSED, so a default run can never have all three PASS and tops out + at WATCH. That is the honest reading, not a defect — a factor evaluated with + no book and no execution evidence has not earned an Adopt. + +THE ASYMMETRIC GATE — FAIL BYPASSES THE SAMPLE GATE (v0.4, kept) + The sample gate exists to prevent OVERCLAIMING, so it guards the POSITIVE + claims (a PASS on Predictive/Incremental) and nothing else. A FAIL is a + NEGATIVE FINDING: thin data plus a VISIBLE failure — an out-of-sample sign + flip, an untradable spread, a loss at every cost level — is already enough to + say "do not trade this". Each axis therefore decides its FAIL BEFORE consulting + the sample gate (Predictive and Tradable have KNOWN-fact FAILs that never touch + the gate; only the Incremental FAIL, which is a statistical "it adds nothing" + claim, waits for a sufficient sample). The costs are asymmetric — a false + Reject skips one possibly-good factor; a false Adopt trades a bad one with real + money — so the rule is too: quick to reject, slow to adopt. + + WHAT THIS FIXES (concretely): a regime-flipping factor — the project's + signature failure mode (I5e, P3-3, P3-4) — has an IC series that is a STEP + FUNCTION, so N_eff collapses to ~3 out of ~300 raw periods. That N_eff is + CORRECT (two regimes really are about two observations), but a gate-first order + would report INSUFFICIENT-DATA instead of Reject about precisely the failure + the project most wants flagged. The flip is a fact the run MEASURED; the gate + has no business suppressing it. Predictive FAIL runs first, so it Rejects. + +UNKNOWN NEVER CONVICTS. Every FAIL requires a KNOWN fact — ``tradable is False`` +(never a falsy ``None``), an explicit flip/reversal flag, a non-empty set of +FINITE all-non-positive spreads, or a FINITE orthogonalized ICIR that measurably +fails to clear on a sufficient sample. A ``None`` / ``NaN`` / absent fact yields +NO FAIL: it falls through to INSUFFICIENT_DATA (Predictive/Incremental) or +NOT_ASSESSED (Tradable). And unknown never passes the gate as a PASS either +(same conservatism, opposite direction). + +THE SAMPLE GATE IS THREE PARTS (v0.3, kept). A RAW COUNT IS NOT A SAMPLE SIZE. +Consecutive daily IC observations are heavily autocorrelated: 500 raw points can +carry only a few dozen independent ones. The gate (raw floor + N_eff + calendar +span) governs whether Predictive/Incremental may be PASS vs INSUFFICIENT_DATA — +same thresholds, now applied PER AXIS. + + raw floor (min_rebalances) a LOW precondition, NOT the sample gate: below + a handful of points the IC autocorrelation — + and hence N_eff itself — is not estimable. + (A) min_effective_samples N_eff = N / (1 + 2*sum_k rho_k) over the IC + series. THE part that does the work. + (B) min_span_days calendar span of the IC series, which is + frequency-independent. + +THE EXPLORATORY CAP (v0.2, kept) caps the DEPLOYMENT LABEL, not an axis: +``EvalConfig.is_exploratory=True`` turns an all-PASS ADOPT into WATCH. Adopt IS a +performance claim, and an exploratory run declares it is not making one. Because +EvalConfig §4 forces ``post_hoc_selected=True`` => ``is_exploratory=True``, this +also closes the post-hoc -> Adopt hole. The cap only downgrades the LABEL; the +axes and their per-axis evidence are reported unchanged. + +TWO CONVENTIONS THAT MAKE THE RULES SIGN-SAFE + 1. Section payloads carry RAW facts: ``monotonicity_spearman`` is the plain + Spearman of bucket index vs bucket mean return, and + ``net_long_short_by_cost`` is the plain (top - bottom) leg difference. The + HYPOTHESIS (``expected_ic_sign``) is applied in exactly ONE place — here — + so a low-vol factor (sign -1) reads in its own direction. + 2. ``ic_win_rate`` and the orthogonalized ICs are hypothesis-relative by the + same rule: direction is applied by multiplying with ``expected_ic_sign``. + +PRE-REGISTERED CRITERIA + CONFIDENCE INTERVALS (v0.6, change #3). The verdict +thresholds are pre-registered per run: ``EvalConfig.success_criteria`` (a frozen +``VerdictThresholds`` constructed BEFORE ``evaluate`` runs) is the declared bar; +when absent the documented global default is used, and the report stamps which +(``criteria_source``). And the magnitude PASS test is now on the LOWER CONFIDENCE +BOUND, not the naked point: the standard layer attaches an N_eff-based CI to every +gated estimate (ICIR, incremental ICIR), and :func:`_clears_magnitude` requires +the aligned lower bound to clear the bar. A thin/noisy estimate has a wide CI and a +low lower bound, so it fails the positive claim automatically. This does NOT +duplicate the sample gate — the gate asks "can we estimate a CI at all?" +(INSUFFICIENT below it); the lower-CI test asks "is it convincingly above the bar?" +(PASS above it). FAIL stays POINT-based and known-fact-only; a NaN CI bound never +convicts. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field + +# -- deployment labels (the DERIVED, user-facing verdict) ---------------- + +INSUFFICIENT_DATA = "INSUFFICIENT-DATA" +ADOPT = "Adopt" +WATCH = "Watch" +REJECT = "Reject" + +VERDICTS: tuple[str, ...] = (INSUFFICIENT_DATA, ADOPT, WATCH, REJECT) + +# -- axis-verdict states (each of the three axes takes one) -------------- +# +# Deliberately their OWN vocabulary, distinct from the deployment labels above: +# an axis answers PASS/FAIL/INSUFFICIENT_DATA/NOT_ASSESSED, and the deployment +# label is derived from the three. (The axis "INSUFFICIENT_DATA" uses an +# underscore; the deployment "INSUFFICIENT-DATA" a hyphen — they are related +# ideas at different scopes, not the same value.) + +AXIS_PASS = "PASS" +AXIS_FAIL = "FAIL" +AXIS_INSUFFICIENT_DATA = "INSUFFICIENT_DATA" +AXIS_NOT_ASSESSED = "NOT_ASSESSED" + +AXIS_VERDICTS: tuple[str, ...] = ( + AXIS_PASS, + AXIS_FAIL, + AXIS_INSUFFICIENT_DATA, + AXIS_NOT_ASSESSED, +) + +#: the three axes, in report order. +AXIS_NAMES: tuple[str, ...] = ("predictive", "incremental", "tradable") + + +# -- threshold domains --------------------------------------------------- +# +# These thresholds decide the axis PASS/FAIL and so the VERDICT — the contract's +# central claim. They are validated for TYPE and RANGE like everything else here. +# +# ⚠️ Structural validation only: it checks a threshold is a sane number, NOT that +# it is the RIGHT number. The defaults remain UNVALIDATED-BY-DATA and still need +# one round of empirical calibration against real runs (design §11). + +#: thresholds compared against a MAGNITUDE: finite and >= 0, no natural upper bound. +_MAGNITUDE_THRESHOLDS: tuple[str, ...] = ( + "min_abs_icir", "min_abs_nw_t", "min_incremental_abs_icir", +) + +#: thresholds living inside a closed metric domain: lower bound INCLUSIVE, upper +#: bound EXCLUSIVE. Both metrics max out at 1.0 and both rules compare with a +#: strict '>', so a threshold AT 1.0 could never be exceeded and would silently +#: make the rule it gates unsatisfiable — the same class of silent breakage as +#: NaN/inf, it just fails strict instead of permissive. +_BOUNDED_THRESHOLDS: dict[str, tuple[float, float]] = { + "min_ic_win_rate": (0.0, 1.0), + "min_monotonicity_spearman": (0.0, 1.0), +} + + +def _check_real(name: str, value: object) -> None: + """Reject non-numbers / bools / NaN / inf with a READABLE ValueError.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError( + f"VerdictThresholds.{name} must be a real number (never a bool: True " + f"is an int subclass and would silently become 1); got {value!r}." + ) + if isinstance(value, float) and not math.isfinite(value): + raise ValueError( + f"VerdictThresholds.{name} must be finite; got {value!r}. NaN/inf " + f"would SILENTLY disable the rule it gates — every '>' comparison " + f"against NaN is False, and nothing exceeds +inf." + ) + + +@dataclass(frozen=True) +class VerdictThresholds: + """Thresholds behind the (fixed) axis rules. + + ⚠️ UNVALIDATED DEFAULTS — plausible starting points, NOT calibrated numbers. + Design §11 lists their calibration as an open item. Pass a tuned instance to + override; never silently re-interpret a threshold as "validated" because a + report printed it. + """ + + # -- the three-part sample gate (design §6, v0.3) --------------------- + # + # RAW FLOOR — deliberately LOW, and deliberately NOT the sample gate any more. + # Its only job is to refuse a sample too small to estimate the IC + # autocorrelation from at all. Parts A and B do the real gating. + min_rebalances: int = 12 + # (A) effective sample size N_eff = N / (1 + 2*sum_k rho_k) of the IC series. + # Note N_eff <= N always (clamped), so this also implies "at least 24 settled + # rebalances". + min_effective_samples: float = 24.0 + # (B) calendar span of the IC series, in days — frequency-independent. 365 is + # a FLOOR, not sufficiency (P3-2's single-year outperformance died in + # P3-3/P3-4). + min_span_days: int = 365 + # |mean(IC)/std(IC)| — magnitude only; direction is checked out-of-sample. + min_abs_icir: float = 0.30 + # (v0.7) Incremental axis bar: |orthogonalized ICIR| of the residual AFTER the + # book. SEPARATE from min_abs_icir because orthogonalization removes variance, + # so the residual ICIR lives on a structurally SMALLER scale than the raw ICIR + # — reusing the raw 0.30 bar was a category error (it silently failed genuinely + # partial-incremental factors). Default ~= half the raw bar; UNVALIDATED (§11). + min_incremental_abs_icir: float = 0.15 + # share of periods whose IC carries the EXPECTED sign (0.5 = a coin flip). + min_ic_win_rate: float = 0.55 + # |Newey-West corrected t| of the mean IC (~5% two-sided). + min_abs_nw_t: float = 2.0 + # hypothesis-aligned Spearman of bucket index vs bucket mean return. + # (v0.7) DIRECTION gate, not strength: default 0.0 = "aligned monotonicity must + # be strictly > 0" (buckets ordered the RIGHT way / NOT reversed), NOT "strongly + # monotone". A 0.8 strength bar rejected tail-concentrated real factors whose + # power sits in one extreme (e.g. jump-amount-corr: ICIR -0.40, NW-t -14.8, yet + # Q3-humped). Raise it to demand strength. UNVALIDATED (§11). + min_monotonicity_spearman: float = 0.0 + + def __post_init__(self) -> None: + n = self.min_rebalances + if isinstance(n, bool) or not isinstance(n, int) or n < 1: + raise ValueError( + f"VerdictThresholds.min_rebalances must be an int >= 1 (the raw " + f"floor below which the IC autocorrelation is not estimable); got " + f"{n!r}." + ) + eff = self.min_effective_samples + _check_real("min_effective_samples", eff) + if eff < 1: + raise ValueError( + f"VerdictThresholds.min_effective_samples must be >= 1: N_eff is " + f"clamped to [1, N], so a threshold below 1 can never fail and " + f"would SILENTLY disable gate part A. Got {eff!r}." + ) + span = self.min_span_days + if isinstance(span, bool) or not isinstance(span, int) or span < 0: + raise ValueError( + f"VerdictThresholds.min_span_days must be an int >= 0 calendar days " + f"(0 disables gate part B explicitly); got {span!r}." + ) + for name in _MAGNITUDE_THRESHOLDS: + value = getattr(self, name) + _check_real(name, value) + if value < 0: + raise ValueError( + f"VerdictThresholds.{name} is compared against a magnitude and " + f"must be non-negative; got {value!r}." + ) + for name, (low, high) in _BOUNDED_THRESHOLDS.items(): + value = getattr(self, name) + _check_real(name, value) + if not low <= value < high: + raise ValueError( + f"VerdictThresholds.{name} must be in [{low}, {high}) — its " + f"metric never exceeds {high} and the rule compares with a " + f"strict '>', so a threshold of {high} or above can never be " + f"exceeded and would SILENTLY make the rule it gates " + f"unsatisfiable. Got {value!r}." + ) + + +@dataclass(frozen=True) +class VerdictInputs: + """The exact facts the axis rules read (extracted from section payloads). + + Every field is either a hard fact or an explicit unknown (NaN / None / + False), so the rules never guess. + + ``is_exploratory`` is the one field that is NOT a measurement: it is the + run's own DECLARATION, read straight off :class:`~analytics.eval.config. + EvalConfig` rather than out of a section payload. Sourcing the cap from a + section payload would let an evaluator escape it by omitting the key. + """ + + expected_ic_sign: int # from FactorSpec: +1 / -1 + settled_rebalances: int = 0 # data_coverage (the RAW count) + # EvalConfig declaration (NOT a metric): True caps the deployment label at Watch. + is_exploratory: bool = False + #: data_coverage: N_eff of the IC series, CLAMPED to [1, N]. NaN = UNKNOWN, + #: which FAILS the gate (never passes it). + effective_samples: float = float("nan") + #: data_coverage: calendar days spanned by the IC series (gate part B). + span_days: float = float("nan") + # -- Predictive axis facts ------------------------------------------- + ic_ir: float = float("nan") # predictive_power (POINT estimate) + #: 95% CI bounds of ic_ir, computed with N_eff (design §6, v0.6). RAW (the + #: verdict applies expected_ic_sign); the Predictive PASS gates on the LOWER + #: bound in the expected direction, NOT the naked point. NaN = UNKNOWN -> the + #: axis cannot PASS, and a NaN bound never manufactures a FAIL. + ic_ir_ci_low: float = float("nan") + ic_ir_ci_high: float = float("nan") + ic_win_rate: float = float("nan") # predictive_power (hypothesis-relative) + ic_nw_t: float = float("nan") # predictive_power + monotonicity_spearman: float = float("nan") # return_risk (RAW) + oos_available: bool = False # oos_generalization + oos_sign_consistent: bool = False # expected sign holds in BOTH subperiods + oos_sign_flipped: bool = False # explicit flip -> Predictive FAIL + oos_monotonicity_reversed: bool = False # independent-cell reversal (I5e) + # -- Incremental axis facts (v0.5, NEW) ------------------------------ + #: purity: was a known-factor BOOK supplied at all? False (the default) means + #: the Incremental axis is NOT_ASSESSED. This is a declaration of what was + #: SUPPLIED, distinct from whether the orthogonalized IC could be computed. + known_factors_supplied: bool = False + #: purity: mean(orthIC)/std(orthIC) of the factor's IC AFTER residualizing it + #: on the WHOLE known-factor book per date (mirror of ``ic_ir``, but on the + #: residual). NaN = UNKNOWN -> Incremental INSUFFICIENT_DATA, never a FAIL. + incremental_ic_ir: float = float("nan") + #: purity: mean of the same orthogonalized IC series (direction + magnitude), + #: reported alongside; the axis reads direction off ``incremental_ic_ir``. + incremental_ic_mean: float = float("nan") + #: 95% CI bounds of incremental_ic_ir, computed with the ORTHOGONALIZED IC + #: series' OWN N_eff. Same lower-bound PASS rule as ic_ir_ci_* (design §6, + #: v0.6). NaN never convicts. + incremental_ic_ir_ci_low: float = float("nan") + incremental_ic_ir_ci_high: float = float("nan") + # -- return_risk / execution facts ----------------------------------- + #: return_risk: {cost multiplier: net (top - bottom) return}, RAW sign. + net_long_short_by_cost: tuple[tuple[float, float], ...] = () + tradable: bool | None = None # execution_capacity; None = not supplied + capacity_sufficient: bool | None = None # None = not supplied + + +@dataclass(frozen=True) +class AxisVerdict: + """One axis's verdict (PASS / FAIL / INSUFFICIENT_DATA / NOT_ASSESSED) + why.""" + + verdict: str + reasons: tuple[str, ...] = field(default_factory=tuple) + + def __post_init__(self) -> None: + if self.verdict not in AXIS_VERDICTS: + raise ValueError( + f"axis verdict must be one of {AXIS_VERDICTS}; got {self.verdict!r}." + ) + object.__setattr__(self, "reasons", tuple(self.reasons)) + + +@dataclass(frozen=True) +class VerdictResult: + """The three axis-verdicts + the DERIVED deployment label (with its reasons). + + ``verdict`` / ``reasons`` are the deployment-level answer (what a caller reads + first); ``predictive`` / ``incremental`` / ``tradable`` are the three + independent axes it was derived from. The axes default to NOT_ASSESSED so a + hand-built result (e.g. a placeholder verdict on an incomplete report) stays + constructible positionally. + """ + + verdict: str + reasons: tuple[str, ...] = field(default_factory=tuple) + predictive: AxisVerdict = field( + default_factory=lambda: AxisVerdict(AXIS_NOT_ASSESSED) + ) + incremental: AxisVerdict = field( + default_factory=lambda: AxisVerdict(AXIS_NOT_ASSESSED) + ) + tradable: AxisVerdict = field( + default_factory=lambda: AxisVerdict(AXIS_NOT_ASSESSED) + ) + + def __post_init__(self) -> None: + if self.verdict not in VERDICTS: + raise ValueError( + f"verdict must be one of {VERDICTS}; got {self.verdict!r}." + ) + object.__setattr__(self, "reasons", tuple(self.reasons)) + + def axes(self) -> dict[str, AxisVerdict]: + """The three axis verdicts keyed by name, in report order.""" + return { + "predictive": self.predictive, + "incremental": self.incremental, + "tradable": self.tradable, + } + + +# -- the axis magnitude/level comparisons (design §6, v0.6: gate on the CI) ------ +# +# CHANGE #3 (pre-registered criteria + confidence intervals) IS this change. Every +# axis PASS magnitude comparison funnels through these two helpers, so the CI logic +# is localized here instead of scattered across the three axes. +# +# TWO-LAYER RELATIONSHIP WITH THE SAMPLE GATE — they are DIFFERENT checks, do NOT +# read one as subsuming the other: +# * the raw-floor / N_eff / span sample gate is the "can we even ESTIMATE a CI?" +# precondition. Below it the axis is INSUFFICIENT_DATA and the CI is not +# consulted at all. +# * the lower-CI comparison here is the "is the estimate CONVINCINGLY above the +# pre-registered bar?" PASS test, ABOVE the gate. The gate can pass (enough +# data) while the lower CI still fails — a noisy estimate whose interval +# straddles the bar even at sufficient N_eff. That run is NOT a PASS, but it is +# not gated out either; it reads as "promising, unconfirmed". + + +def _clears_magnitude( + point_estimate: float, threshold: float, *, lower: float | None = None +) -> bool: + """Does a magnitude estimate clear ``threshold``? + + Point path (``lower`` omitted — e.g. the Newey-West t, itself an inference + statistic): ``|point_estimate| > threshold``. + + CI path (design §6, v0.6 — ICIR, incremental ICIR): a supplied ``lower`` is the + estimate's LOWER CONFIDENCE BOUND already ALIGNED to the expected direction, and + the estimate clears only if that lower bound exceeds the bar — "convincingly + above", not the naked point. A thin/noisy estimate -> wide CI -> low lower bound + -> fails automatically. A NaN ``lower`` cannot clear (unknown never PASSes); and + it never manufactures a FAIL — the FAIL branches are POINT-based and never call + this with a ``lower``. + """ + if lower is not None: + return math.isfinite(lower) and lower > threshold + return math.isfinite(point_estimate) and abs(point_estimate) > threshold + + +def _clears_level(point_estimate: float, threshold: float) -> bool: + """point_estimate > threshold (a directed level), estimate KNOWN and finite.""" + return math.isfinite(point_estimate) and point_estimate > threshold + + +def _aligned_lower_bound(sign: int, ci_low: float, ci_high: float) -> float: + """The lower CI bound of ``sign * estimate`` (the bound in the EXPECTED direction). + + ``ci_low``/``ci_high`` bracket the RAW estimate; multiplying the interval by the + hypothesis sign and taking the smaller endpoint yields the worst-case value in + the direction the factor claims. NaN when either bound is unknown — the caller + then treats the estimate as not-convincingly-clearing (never a FAIL). + """ + if not (math.isfinite(ci_low) and math.isfinite(ci_high)): + return float("nan") + return min(sign * ci_low, sign * ci_high) + + +# -- shared fact readers ------------------------------------------------- + + +def _sample_gate_failures( + inputs: VerdictInputs, thr: VerdictThresholds +) -> list[str]: + """Every part of the three-part sample gate that did NOT pass (design §6). + + Returns ALL failures rather than the first: "which part failed" is the whole + diagnostic value, and a run can fail more than one. Empty list = gate passed. + An UNKNOWN (non-finite) fact fails as UNKNOWN rather than being compared. + """ + failures: list[str] = [] + + if inputs.settled_rebalances < thr.min_rebalances: + failures.append( + f"raw floor: settled rebalances {inputs.settled_rebalances} < required " + f"{thr.min_rebalances}; below this the IC autocorrelation — and so the " + f"effective sample size itself — is not estimable." + ) + + n_eff = inputs.effective_samples + if not math.isfinite(n_eff): + failures.append( + f"effective samples (A): UNKNOWN — data_coverage reported no finite " + f"'effective_samples' (required >= {thr.min_effective_samples}). A " + f"sample-adequacy gate cannot pass on a sample size nobody measured." + ) + elif n_eff < thr.min_effective_samples: + failures.append( + f"effective samples (A): {n_eff:.2f} < required " + f"{thr.min_effective_samples} — from {inputs.settled_rebalances} raw " + f"settled rebalance(s), whose IC observations are autocorrelated and " + f"therefore NOT that many independent pieces of evidence." + ) + + span = inputs.span_days + if not math.isfinite(span): + failures.append( + f"calendar span (B): UNKNOWN — data_coverage reported no finite " + f"'span_days' (required >= {thr.min_span_days})." + ) + elif span < thr.min_span_days: + failures.append( + f"calendar span (B): {span:.0f} calendar day(s) < required " + f"{thr.min_span_days}; a dense-but-short window is not a sample at any " + f"rebalance frequency." + ) + + return failures + + +def _base_spread(inputs: VerdictInputs) -> float: + """Hypothesis-aligned net long-short return at the BASE (1.0x) cost.""" + for multiplier, value in inputs.net_long_short_by_cost: + if abs(multiplier - 1.0) < 1e-9: + return inputs.expected_ic_sign * value + return float("nan") + + +def _all_spreads_negative(inputs: VerdictInputs) -> bool: + """True only when EVERY known cost scenario is non-positive (design §6). + + An empty/unknown set is NOT "all negative" — unknown is not evidence. + """ + aligned = [ + inputs.expected_ic_sign * v + for _, v in inputs.net_long_short_by_cost + if math.isfinite(v) + ] + return bool(aligned) and all(v <= 0 for v in aligned) + + +def _has_in_sample_point_signal(inputs: VerdictInputs, thr: VerdictThresholds) -> bool: + """POINT-estimate in-sample signal: |ICIR|, aligned monotonicity, NW-t, win rate. + + Every component must be a finite, known number: an unknown is not a signal. + This is the v0.5 point check, kept for the FAIL determination — a factor whose + POINT metrics do not clear has no signal at all (a negative finding). The CI + lower-bound test (design §6, v0.6) is applied SEPARATELY, only to decide PASS vs + "promising but unconfirmed" for a factor that DID clear on the point. + """ + aligned_monotonicity = inputs.expected_ic_sign * inputs.monotonicity_spearman + return ( + _clears_magnitude(inputs.ic_ir, thr.min_abs_icir) + and _clears_magnitude(inputs.ic_nw_t, thr.min_abs_nw_t) + and _clears_level(inputs.ic_win_rate, thr.min_ic_win_rate) + and _clears_level(aligned_monotonicity, thr.min_monotonicity_spearman) + ) + + +# -- Axis A: Predictive -------------------------------------------------- + + +def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdict: + """Is there a real, OUT-OF-SAMPLE predictive signal? (design §6, Axis A). + + Order (FAIL before the gate — the asymmetric rule): + 1. FAIL a KNOWN out-of-sample reversal (sign flip or independent-cell + monotonicity reversal). Decided BEFORE the gate, so it Rejects even + on a thin sample. Requires oos_available AND a set reversal flag — + unknown never convicts. + 2. INSUFFICIENT_DATA the sample gate failed OR there is no OOS evidence. + In-sample metrics are still REPORTED (in the section), just not a + PASS: no out-of-sample evidence means no predictive claim. The two + reasons (gate vs no-OOS) are named distinctly. + 3. FAIL sufficient data + OOS, but a NEGATIVE finding: the OOS sign is not + consistent OR the in-sample POINT metrics do not clear (no signal at + all). POINT-based, so a merely wide CI never manufactures a FAIL. + 4. PASS OOS sign consistent AND point metrics clear AND the ICIR LOWER CI + bound (N_eff-based, expected direction) exceeds the bar — design §6, + v0.6: convincingly above, not the naked point. + 5. INSUFFICIENT_DATA point metrics clear + OOS consistent, but the ICIR lower + CI does NOT clear (or is unknown): promising, yet not confirmed at + the pre-registered bar. This is the "gate passed, CI still straddles + the bar" case — distinct from the sample gate (step 2). + """ + sign = inputs.expected_ic_sign + # 1. FAIL on a KNOWN out-of-sample reversal — before the gate. + if inputs.oos_available and ( + inputs.oos_sign_flipped or inputs.oos_monotonicity_reversed + ): + reasons: list[str] = [] + if inputs.oos_sign_flipped: + reasons.append( + "out-of-sample IC sign flipped against the stated hypothesis." + ) + if inputs.oos_monotonicity_reversed: + reasons.append( + "quantile monotonicity reversed on an independent cell." + ) + return AxisVerdict(AXIS_FAIL, tuple(reasons)) + + # 2. INSUFFICIENT_DATA on the sample gate OR no OOS evidence. + reasons = _sample_gate_failures(inputs, thr) + if not inputs.oos_available: + reasons.append( + "no out-of-sample split: generalization NOT established, so there is " + "no predictive PASS to claim (in-sample metrics are reported, not a " + "PASS)." + ) + if reasons: + return AxisVerdict(AXIS_INSUFFICIENT_DATA, tuple(reasons)) + + # 3. FAIL: a measured negative finding (OOS sign not consistent, or no point + # signal at all). POINT-based — a wide CI is handled below, not here. + point_signal = _has_in_sample_point_signal(inputs, thr) + if (not inputs.oos_sign_consistent) or (not point_signal): + reasons = [] + if not inputs.oos_sign_consistent: + reasons.append( + "out-of-sample: the expected IC sign is NOT consistent across both " + "holdout subperiods (we looked out-of-sample and the signal was " + "not there)." + ) + if not point_signal: + reasons.append( + "in-sample POINT metrics do NOT clear the predictive thresholds " + "(ICIR / NW-t / win rate / monotonicity)." + ) + return AxisVerdict(AXIS_FAIL, tuple(reasons)) + + # 4/5. Point signal present + OOS consistent. The #3 lower-CI test decides + # PASS vs "promising but unconfirmed" (INSUFFICIENT_DATA). This is ABOVE + # the sample gate and does NOT duplicate it (see _clears_magnitude). + icir_lower = _aligned_lower_bound(sign, inputs.ic_ir_ci_low, inputs.ic_ir_ci_high) + if _clears_magnitude(inputs.ic_ir, thr.min_abs_icir, lower=icir_lower): + return AxisVerdict( + AXIS_PASS, + ( + "expected IC sign holds in both out-of-sample subperiods.", + f"in-sample signal convincingly above the bar: the ICIR lower CI " + f"bound (N_eff-based, expected direction) {icir_lower:+.3f} > " + f"{thr.min_abs_icir}; point |ICIR| {abs(inputs.ic_ir):.3f}, " + f"|NW-t| {abs(inputs.ic_nw_t):.2f}, win rate " + f"{inputs.ic_win_rate:.3f}.", + ), + ) + lower_txt = "UNKNOWN" if not math.isfinite(icir_lower) else f"{icir_lower:+.3f}" + return AxisVerdict( + AXIS_INSUFFICIENT_DATA, + ( + f"the point metrics clear and the out-of-sample sign is consistent, but " + f"the ICIR LOWER CI bound ({lower_txt}, N_eff-based) does not exceed the " + f"pre-registered bar {thr.min_abs_icir}: promising, but not confirmed. " + f"The sample size passed the gate; the ESTIMATE is still too imprecise " + f"to claim a PASS (a wider bar than the raw count implies).", + ), + ) + + +# -- Axis B: Incremental ------------------------------------------------- + + +def _incremental_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdict: + """Does it add alpha BEYOND the known factor set? (design §6, Axis B). + + The orthogonalized IC is the factor's IC AFTER residualizing it on the WHOLE + known-factor book per date (computed in PR-B's ``standard.py``; consumed here + as ``incremental_ic_ir``, exactly as the Predictive axis consumes ``ic_ir``). + + Order: + * NOT_ASSESSED no known_factors were supplied (the default). + * INSUFFICIENT_DATA a book was supplied but the sample is too thin to + assess it (same N_eff/span gate), OR the orthogonalized + IC could not be measured (NaN — unknown, never a FAIL), + OR the POINT clears in the expected direction but the LOWER + CI bound does not (promising, unconfirmed — design §6, v0.6). + * PASS the orthogonalized IC is significantly in the EXPECTED + direction: its LOWER CI bound (N_eff-based) clears + min_abs_icir. + * FAIL book + sufficient sample, but the orthogonalized IC POINT + is convincingly ~ 0 or in the WRONG direction (redundant / + anti-incremental — the factor duplicates the book). FAIL is + POINT-based, so a merely wide CI never manufactures it. + """ + sign = inputs.expected_ic_sign + if not inputs.known_factors_supplied: + return AxisVerdict( + AXIS_NOT_ASSESSED, + ( + "no known-factor book supplied: incremental value BEYOND the " + "existing factors cannot be assessed. Pass EvalContext." + "known_factors to evaluate this axis.", + ), + ) + + gate_failures = _sample_gate_failures(inputs, thr) + if gate_failures: + return AxisVerdict( + AXIS_INSUFFICIENT_DATA, + tuple(gate_failures) + + ( + "a book was supplied, but the sample is too thin to judge whether " + "the factor is incremental to it.", + ), + ) + + if not math.isfinite(inputs.incremental_ic_ir): + return AxisVerdict( + AXIS_INSUFFICIENT_DATA, + ( + "the orthogonalized IC (factor residualized on the known-factor " + "book) could not be measured — reported as UNKNOWN, which is not a " + "FAIL: unknown never convicts.", + ), + ) + + # FAIL is POINT-based (v0.5 semantics unchanged): a measured orthogonalized IC + # that is ~ 0 or in the wrong direction. A merely WIDE CI is handled below, not + # here, so an imprecise-but-positive estimate is never mislabelled "redundant". + aligned_point = sign * inputs.incremental_ic_ir + if not (aligned_point > 0 and _clears_magnitude(inputs.incremental_ic_ir, thr.min_incremental_abs_icir)): + direction = ( + "in the WRONG direction (anti-incremental)" + if aligned_point < 0 + else "~ 0 (redundant with the book)" + ) + return AxisVerdict( + AXIS_FAIL, + ( + f"orthogonalized ICIR {inputs.incremental_ic_ir:+.3f} is " + f"{direction}: after residualizing on the known-factor book the " + f"factor adds no signal of magnitude > {thr.min_incremental_abs_icir} in its " + f"stated direction.", + ), + ) + + # The point clears in the expected direction. The #3 lower-CI test (design §6, + # v0.6) decides PASS vs "promising but unconfirmed", ABOVE the sample gate. + incr_lower = _aligned_lower_bound( + sign, inputs.incremental_ic_ir_ci_low, inputs.incremental_ic_ir_ci_high + ) + if _clears_magnitude(inputs.incremental_ic_ir, thr.min_incremental_abs_icir, lower=incr_lower): + return AxisVerdict( + AXIS_PASS, + ( + f"orthogonalized ICIR lower CI bound (N_eff-based, expected " + f"direction) {incr_lower:+.3f} > {thr.min_incremental_abs_icir}: the factor " + f"convincingly adds a signal the book does not already carry " + f"(point {inputs.incremental_ic_ir:+.3f}).", + ), + ) + lower_txt = "UNKNOWN" if not math.isfinite(incr_lower) else f"{incr_lower:+.3f}" + return AxisVerdict( + AXIS_INSUFFICIENT_DATA, + ( + f"the orthogonalized ICIR point {inputs.incremental_ic_ir:+.3f} is in " + f"the expected direction, but its LOWER CI bound ({lower_txt}, " + f"N_eff-based) does not exceed the pre-registered bar {thr.min_incremental_abs_icir}: " + f"promising incremental value, but not confirmed. Sufficient sample; the " + f"estimate is still too imprecise to claim a PASS.", + ), + ) + + +# -- Axis C: Tradable ---------------------------------------------------- + + +def _tradable_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdict: + """Can the signal actually be harvested? (design §6, Axis C). + + Execution facts (I5b fill feasibility / I5f capacity) are MEASURED ELSEWHERE + and are not IR inputs, so the DEFAULT is NOT_ASSESSED. + + * NOT_ASSESSED no execution facts supplied (tradable is None AND + capacity is None). + * FAIL a KNOWN execution failure: ``tradable is False`` OR + ``capacity_sufficient is False`` OR net long-short + non-positive at EVERY cost scenario. (No sample gate — a + "cannot execute" is not a statistical claim.) + * PASS tradable AND capacity sufficient AND net positive at base + cost. + * INSUFFICIENT_DATA partial facts (e.g. tradable known but capacity not, or + the base-cost spread unknown): established neither way, + and unknown must not be spun into either a PASS or a FAIL. + """ + if inputs.tradable is None and inputs.capacity_sufficient is None: + return AxisVerdict( + AXIS_NOT_ASSESSED, + ( + "no execution facts supplied (I5b fill feasibility / I5f capacity " + "are measured elsewhere, not IR inputs): tradability not assessed.", + ), + ) + + fail_reasons: list[str] = [] + if inputs.tradable is False: + fail_reasons.append("not tradable: the spread cannot be executed.") + if inputs.capacity_sufficient is False: + fail_reasons.append( + "capacity insufficient at the target notional." + ) + if _all_spreads_negative(inputs): + fail_reasons.append( + "net long-short return is non-positive in EVERY cost scenario." + ) + if fail_reasons: + return AxisVerdict(AXIS_FAIL, tuple(fail_reasons)) + + base = _base_spread(inputs) + if ( + inputs.tradable is True + and inputs.capacity_sufficient is True + and _clears_level(base, 0.0) + ): + return AxisVerdict( + AXIS_PASS, + ( + "fills feasible and capacity sufficient at the target notional, " + f"and the net long-short return is positive at base cost " + f"({base:+.4f}).", + ), + ) + + return AxisVerdict( + AXIS_INSUFFICIENT_DATA, + ( + "tradability marginal: fills and/or capacity are not both established, " + "or the base-cost net spread is unknown/non-positive. Established " + "neither way — unknown is not a PASS and not a FAIL.", + ), + ) + + +# -- the deployment derivation ------------------------------------------- + + +_CAP_REASON = ( + "CAPPED AT WATCH: this run declares EvalConfig.is_exploratory=True, so no " + "Adopt claim is available to it regardless of the axes — Adopt IS a " + "performance claim and an exploratory run declares it is not making one. The " + "per-axis PASS evidence below stands on its own; to claim Adopt, re-run as a " + "non-exploratory confirmation." +) + + +def _tag(name: str, axis: AxisVerdict) -> list[str]: + """Per-axis reasons, prefixed with the axis name + its state (for the label).""" + return [f"[{name} {axis.verdict}] {reason}" for reason in axis.reasons] + + +def _derive_deployment( + predictive: AxisVerdict, + incremental: AxisVerdict, + tradable: AxisVerdict, + inputs: VerdictInputs, +) -> tuple[str, tuple[str, ...]]: + """Derive the single deployment label from the three axes (design §6 table). + + Order preserves the asymmetric gate + exploratory cap: + any FAIL -> REJECT (FAIL is checked FIRST, so a thin-sample failure still + Rejects); all three PASS -> ADOPT (capped to WATCH if exploratory); + >=1 PASS with the rest unresolved -> WATCH; no PASS and no FAIL -> + INSUFFICIENT-DATA. + """ + axes = ( + ("predictive", predictive), + ("incremental", incremental), + ("tradable", tradable), + ) + fails = [(name, axis) for name, axis in axes if axis.verdict == AXIS_FAIL] + passes = [(name, axis) for name, axis in axes if axis.verdict == AXIS_PASS] + unresolved = [ + (name, axis) + for name, axis in axes + if axis.verdict in (AXIS_INSUFFICIENT_DATA, AXIS_NOT_ASSESSED) + ] + + # 1. any FAIL -> REJECT. Checked first, so it bypasses the sample gate. + if fails: + names = ", ".join(name for name, _ in fails) + reasons = [ + f"REJECT: the {names} axis/axes FAILED. A factor that fails ANY axis " + f"is rejected, and this decision precedes the sample gate — a " + f"thin-sample failure still Rejects." + ] + for name, axis in fails: + reasons += _tag(name, axis) + return REJECT, tuple(reasons) + + # 2. all three PASS -> ADOPT (capped to WATCH if the run is exploratory). + evidence = [reason for name, axis in passes for reason in _tag(name, axis)] + if len(passes) == 3: + if inputs.is_exploratory: + return WATCH, (_CAP_REASON, *evidence) + return ADOPT, tuple(evidence) + + # 3. >=1 PASS, rest unresolved -> WATCH. + if passes: + pass_names = ", ".join(name for name, _ in passes) + unresolved_names = ", ".join( + f"{name} ({axis.verdict})" for name, axis in unresolved + ) + reasons = [ + f"WATCH: {pass_names} established; unresolved: {unresolved_names}. A " + f"positive claim on some axes but not all, so no Adopt." + ] + for name, axis in passes: + reasons += _tag(name, axis) + for name, axis in unresolved: + reasons += _tag(name, axis) + return WATCH, tuple(reasons) + + # 4. no FAIL, no PASS -> INSUFFICIENT-DATA. + reasons = [ + "INSUFFICIENT-DATA: no axis reached a positive PASS and none failed — " + "nothing was demonstrated and nothing was refuted." + ] + for name, axis in unresolved: + reasons += _tag(name, axis) + return INSUFFICIENT_DATA, tuple(reasons) + + +def decide_verdict( + inputs: VerdictInputs, thresholds: VerdictThresholds | None = None +) -> VerdictResult: + """Score the three axes and derive the deployment label. Pure + deterministic.""" + thr = thresholds or VerdictThresholds() + predictive = _predictive_axis(inputs, thr) + incremental = _incremental_axis(inputs, thr) + tradable = _tradable_axis(inputs, thr) + label, reasons = _derive_deployment(predictive, incremental, tradable, inputs) + return VerdictResult( + verdict=label, + reasons=reasons, + predictive=predictive, + incremental=incremental, + tradable=tradable, + ) + + +__all__ = [ + "INSUFFICIENT_DATA", + "ADOPT", + "WATCH", + "REJECT", + "VERDICTS", + "AXIS_PASS", + "AXIS_FAIL", + "AXIS_INSUFFICIENT_DATA", + "AXIS_NOT_ASSESSED", + "AXIS_VERDICTS", + "AXIS_NAMES", + "VerdictThresholds", + "VerdictInputs", + "AxisVerdict", + "VerdictResult", + "decide_verdict", +] diff --git a/config/phase_c_jump_amount_corr.yaml b/config/phase_c_jump_amount_corr.yaml new file mode 100644 index 0000000..cd8971a --- /dev/null +++ b/config/phase_c_jump_amount_corr.yaml @@ -0,0 +1,111 @@ +# PR-C — First real factor evaluation: price-jump turnover correlation. +# +# Reproduces the Kaiyuan report §6 factor (价格跳跃成交额相关性, full-A RankIC +# -10.23%, market-cap + industry neutral) as a first-class JumpAmountCorrFactor and +# runs it through the FROZEN StandardFactorEvaluator on REAL cached A-share data. +# CACHE-ONLY: the minute read is provably live-call-free; daily / universe / +# covariate endpoints go through the read-through cache (warm -> 0 gap fetches). +# +# Universe = CSI500 (000905.SH), PIT membership; window = 2021-07-01 .. 2026-06-30 +# (the project's minute-coverage window; the report notes the factor is stronger in +# mid/small caps). Daily rebalance (project + contract default). The evaluator runs +# TWICE (see qt/eval_jump_amount_corr.py): once with NO book (Incremental +# NOT_ASSESSED) and once with the confirmed book (value_ep/value_bp/volatility_20) +# to measure whether jump-amount-corr adds alpha BEYOND value / low-vol. + +project: + name: quantitative_trading_pr_c_jump_amount_corr + timezone: Asia/Shanghai +data: + source: tushare + freq: D + start: '2021-07-01' + end: '2026-06-30' + external_secret_file: /home/shaofl/Projects/financial_projects/.config.json + tushare_token_key: tushare.token + output_name: jump_amount_corr_daily + cache: + enabled: true + root_dir: artifacts/cache/tushare/v1 + refresh_recent_days: 14 + refresh_dimension_days: 30 + force_refresh: [] +universe: + type: index + index_code: 000905.SH + symbols: [] + min_listing_days: 60 + filters: + missing_close: true + suspended: false + st: false + limit_up_down: false +# factors = the confirmed BOOK used for the Incremental axis (value + low-vol). The +# SUBJECT factor (jump_amount_corr_20) is minute-derived and computed by the runner +# from the intraday cache — it is NOT a config-listed daily factor. +factors: +- name: value_ep + enabled: true +- name: value_bp + enabled: true +- name: volatility_20 + enabled: true + params: + window: 20 + price_col: close +processing: + drop_missing: true + standardize: + enabled: true + method: zscore + # winsorize is a P0 no-op in this codebase; the EvalConfig declares winsorize=None + # accordingly (nothing is clipped), so the report never overstates preprocessing. + winsorize: + enabled: false + method: mad + n: 3.0 + # Industry + market-cap neutralization (SW-L1), matching the report's neutral + # column and the EvalConfig neutralization declaration. + neutralize: + enabled: true + industry_col: industry + size_col: market_cap + industry_level: L1 +alpha: + model: equal_weight + params: {} +portfolio: + # Required by the schema but unused: this runner evaluates a factor, it does not + # build/execute a portfolio. + constructor: topn_equal_weight + top_n: 50 + long_only: true + max_weight: null + turnover_cap: null +backtest: + # Required by the schema but unused (no backtest is run). The EvalConfig uses a + # DAILY rebalance (contract default), independent of this field. + initial_nav: 1.0 + rebalance: monthly + event_order: close_to_next_period + cash_return: 0.0 +cost: + fee_rate: 0.001 + slippage_rate: 0.0 + turnover_formula: l1 +analytics: + forward_return_periods: + - 1 + quantiles: 5 + benchmark: null +# OOS split (window midpoint) so the OOS section runs and the Predictive axis can be +# assessed (sign consistency across both holdout subperiods). +oos: + split_date: '2024-01-01' +output: + root_dir: artifacts + data_dir: artifacts/data + factor_dir: artifacts/factors + report_dir: artifacts/reports + log_dir: artifacts/logs + overwrite: true diff --git a/data/clean/intraday_aggregate.py b/data/clean/intraday_aggregate.py index d3058a9..ece083e 100644 --- a/data/clean/intraday_aggregate.py +++ b/data/clean/intraday_aggregate.py @@ -76,6 +76,20 @@ MMP_LOOKBACK = 20 DEFAULT_EPSILON = 1e-6 +# Price-jump turnover-correlation factor (PR-C, Kaiyuan report §6). The daily +# value is a trailing-``JUMP_LOOKBACK_DAYS``-trading-day lagged correlation between +# the traded ``amount`` at price-JUMP minutes and the amount at the STRICTLY-next +# minute; a jump minute is one whose within-(symbol, day) amplitude z-score exceeds +# ``JUMP_Z``. These are part of the FACTOR DEFINITION (reproduced from the report), +# not tuned knobs. Requires at least ``JUMP_MIN_PAIRS`` jump-pairs in the window. +JUMP_LOOKBACK_DAYS = 20 +JUMP_MIN_PAIRS = 10 +JUMP_Z = 1.0 +# One minute in seconds: the "strictly next minute" test. A gap != 60s (the lunch +# break, the session close, a missing bar) is NOT the next minute, so that jump +# contributes no pair — exactly the report's within-session adjacency rule. +_ONE_MINUTE_SECONDS = 60.0 + def compute_minute_mmp( open_: np.ndarray, @@ -375,6 +389,163 @@ def mmp_valid_minute_counts( return pd.Series(counts, index=index, dtype=int).sort_index() +def _empty_jump_series(name: str) -> pd.Series: + """Schema-shaped empty ``MultiIndex(date, symbol)`` jump-factor Series.""" + index = pd.MultiIndex.from_arrays( + [pd.DatetimeIndex([]), pd.Index([], dtype=object)], + names=DAILY_INDEX_NAMES, + ) + return pd.Series([], index=index, dtype=float, name=name) + + +def compute_jump_amount_corr( + bars: pd.DataFrame, + *, + lookback_days: int = JUMP_LOOKBACK_DAYS, + min_pairs: int = JUMP_MIN_PAIRS, + jump_z: float = JUMP_Z, + name: str = "jump_amount_corr", +) -> pd.Series: + """PIT-safe daily "price-jump turnover correlation" factor from 1min ``bars``. + + Reproduces the Kaiyuan report §6 factor (``价格跳跃成交额相关性``, full-A RankIC + -10.23%): the trailing-``lookback_days``-trading-day lagged Pearson correlation + between the traded ``amount`` at price-JUMP minutes and the amount at the + STRICTLY-next minute. The daily value at ``(date d, symbol s)`` uses ONLY bars + at dates ``<= d`` (the trailing window ending at d), so a factor value never + sees a future bar (invariant #1); it is meant to trade close-to-close from d+1. + + Definition (LOCKED, per bar of one (symbol, day) session): + * ``amplitude = (high - low) / open`` (guard open > 0, amount finite); + * ``jump`` = within-(symbol, day) amplitude z-score (ddof=1) ``> jump_z``; + * pair each jump minute ``t`` with the STRICTLY-next minute (same session, + ``bar_end`` gap exactly 60s — this excludes the lunch break AND the close); + * ``factor(s, d)`` = Pearson corr(amount[jump t], amount[t+1]) over ALL + jump-pairs whose date is in the trailing ``lookback_days`` TRADING DAYS + (the symbol's own minute-trading days) ending at d; NaN when fewer than + ``min_pairs`` pairs fall in the window (or the correlation is undefined). + + Vectorized (no per-rebalance-date python loop): the trailing-window correlation + is a rolling sum of per-day sufficient statistics (n, sum x, sum y, sum x^2, + sum y^2, sum xy) over the trading-day axis, then Pearson's closed form. Rolling + over ROWS of the per-day-sorted stats == a trailing window of trading days, + because consecutive rows are consecutive trading days. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the + grouping is strictly per symbol (no cross-symbol leakage). + lookback_days: trailing trading-day window length (part of the definition). + min_pairs: minimum jump-pairs in the window for a finite value. + jump_z: within-day amplitude z-score threshold defining a jump minute. + name: the returned Series name (the factor-panel column name). + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily + factor value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if min_pairs < 2: + # Pearson correlation needs at least 2 points; below that it is undefined. + raise ValueError(f"min_pairs must be >= 2; got {min_pairs!r}.") + if len(bars) == 0: + return _empty_jump_series(name) + + work = bars.reset_index()[ + [SYMBOL_LEVEL, "bar_end", "open", "high", "low", "amount"] + ].copy() + # Guard bad rows BEFORE anything else: a non-positive open makes the amplitude + # meaningless and a non-finite amount would poison the correlation. + work = work[(work["open"] > 0.0) & np.isfinite(work["amount"].to_numpy(dtype=float))] + if work.empty: + return _empty_jump_series(name) + work[DATE_LEVEL] = work["bar_end"].dt.normalize() + # Sort so the "strictly next minute" shift and the per-symbol trading-day + # rolling both see bars in chronological order within each (symbol, day). + work = work.sort_values([SYMBOL_LEVEL, "bar_end"], kind="mergesort") + + work["amp"] = (work["high"] - work["low"]) / work["open"] + by_session = work.groupby([SYMBOL_LEVEL, DATE_LEVEL], sort=False) + mean_amp = by_session["amp"].transform("mean") + std_amp = by_session["amp"].transform("std") # ddof=1 (pandas default) + zscore = (work["amp"] - mean_amp) / std_amp + next_bar_end = by_session["bar_end"].shift(-1) + amt_next = by_session["amount"].shift(-1) + gap = (next_bar_end - work["bar_end"]).dt.total_seconds() + is_jump = (zscore > jump_z) & (gap == _ONE_MINUTE_SECONDS) + + pairs = pd.DataFrame( + { + SYMBOL_LEVEL: work[SYMBOL_LEVEL].to_numpy(), + DATE_LEVEL: work[DATE_LEVEL].to_numpy(), + "x": work["amount"].to_numpy(dtype=float), + "y": amt_next.to_numpy(dtype=float), + } + ).loc[is_jump.to_numpy()] + + # Per-(symbol, day) sufficient statistics of the jump-pairs. + if pairs.empty: + stats = pd.DataFrame( + columns=["cnt", "sx", "sy", "sxx", "syy", "sxy"], dtype=float + ) + stats.index = pd.MultiIndex.from_arrays( + [pd.DatetimeIndex([]), pd.Index([], dtype=object)], + names=[SYMBOL_LEVEL, DATE_LEVEL], + ) + else: + pairs = pairs.assign( + xx=pairs["x"] * pairs["x"], + yy=pairs["y"] * pairs["y"], + xy=pairs["x"] * pairs["y"], + ) + stats = pairs.groupby([SYMBOL_LEVEL, DATE_LEVEL], sort=True).agg( + cnt=("x", "size"), + sx=("x", "sum"), + sy=("y", "sum"), + sxx=("xx", "sum"), + syy=("yy", "sum"), + sxy=("xy", "sum"), + ) + + # Trading-day axis = every (symbol, day) the symbol has minute bars on, so the + # rolling window counts TRADING DAYS (days with no jump still occupy a row, as + # zeros — they consume one of the trailing ``lookback_days`` slots). + axis = ( + work[[SYMBOL_LEVEL, DATE_LEVEL]] + .drop_duplicates() + .sort_values([SYMBOL_LEVEL, DATE_LEVEL], kind="mergesort") + ) + full_index = pd.MultiIndex.from_arrays( + [axis[SYMBOL_LEVEL].to_numpy(), axis[DATE_LEVEL].to_numpy()], + names=[SYMBOL_LEVEL, DATE_LEVEL], + ) + dense = stats.reindex(full_index).fillna(0.0) + rolled = ( + dense.groupby(level=SYMBOL_LEVEL, sort=False) + .rolling(lookback_days, min_periods=1) + .sum() + ) + # groupby.rolling prepends the group key -> drop it, keep (symbol, date). + rolled.index = rolled.index.droplevel(0) + + n = rolled["cnt"].to_numpy(dtype=float) + sx = rolled["sx"].to_numpy(dtype=float) + sy = rolled["sy"].to_numpy(dtype=float) + with np.errstate(invalid="ignore", divide="ignore"): + cov = n * rolled["sxy"].to_numpy(dtype=float) - sx * sy + var_x = n * rolled["sxx"].to_numpy(dtype=float) - sx * sx + var_y = n * rolled["syy"].to_numpy(dtype=float) - sy * sy + den = np.sqrt(var_x * var_y) + corr = np.where((n >= min_pairs) & (den > 0.0), cov / den, np.nan) + corr = np.clip(corr, -1.0, 1.0) + + out = pd.Series(corr, index=rolled.index, name=name) + out.index = out.index.set_names([SYMBOL_LEVEL, DATE_LEVEL]) + return out.reorder_levels([DATE_LEVEL, SYMBOL_LEVEL]).sort_index() + + def resample_intraday_bars(bars: pd.DataFrame, freq: str) -> pd.DataFrame: """Derive coarser intraday bars from normalized 1min ``bars``. diff --git a/factors/base.py b/factors/base.py index 17533de..e7ed297 100644 --- a/factors/base.py +++ b/factors/base.py @@ -6,23 +6,147 @@ event order (see CONTRACTS.md), ``momentum_20[t] = close[t]/close[t-20] - 1`` is acceptable because rebalancing happens AFTER the close of t and holding starts the next trading day. + +MANDATORY SPEC (factor-evaluation contract, enforcement layer #1): + every CONCRETE subclass MUST declare a :class:`~factors.spec.FactorSpec` in + its own class body, as a plain class attribute or as a ``property``. Writing + a factor therefore MEANS declaring its hypothesis (``expected_ic_sign``), its + PIT contract and its inputs — there is no "add the spec later" path. + + "Concrete" is meant literally: a class that still has unimplemented abstract + methods (a shared factor-family base that leaves ``compute`` to its children) + is NOT yet a factor and is not asked for a spec — it would have to invent a + bogus one. The requirement fires on the first class that could actually be + instantiated and evaluated. """ from __future__ import annotations -from abc import ABC, abstractmethod +import functools +import inspect +from abc import ABC, ABCMeta, abstractmethod import pandas as pd +from factors.spec import FactorSpec + +# Sentinel: ``None`` is a plausible (wrong) declared value, so it cannot mark +# "nothing was declared". +_UNDECLARED = object() + +# Declaration forms a subclass may use for ``spec``: +# * a FactorSpec instance -> validated right here, at class definition; +# * a property/cached_property -> needs an instance, so _FactorMeta.__call__ +# validates whatever it returns at construction time. +# An explicit allow-list on purpose: ``hasattr(type(x), "__get__")`` accepts a +# plain function, a staticmethod and a classmethod too (all are descriptors), +# which is exactly the "author forgot @property" bug this must reject. +_DEFERRED_SPEC_TYPES = (property, functools.cached_property) + + +def _validate_instance_spec(factor: object) -> FactorSpec: + """Return ``factor.spec``, asserting it really is a :class:`FactorSpec`.""" + spec = factor.spec # a property is evaluated here; errors propagate as-is + if not isinstance(spec, FactorSpec): + raise TypeError( + f"{type(factor).__name__}.spec must be a FactorSpec instance; got " + f"{type(spec).__name__}. Declare it as a class attribute, or as a " + f"property returning a FactorSpec when the id depends on " + f"constructor params (e.g. momentum_{{window}})." + ) + return spec + + +def _validate_declared_spec(cls: type) -> None: + """Class-definition-time check: this class body declares a usable ``spec``. + + Skipped while the class is still ABSTRACT (``compute`` not implemented yet): + such a class is a shared family base, not a factor, and demanding a spec + would force it to invent one. The moment a subclass becomes concrete, the + requirement applies in full. + """ + declared = cls.__dict__.get("spec", _UNDECLARED) + if declared is _UNDECLARED: + # ABCMeta.__new__ has already computed this by the time we run. + if getattr(cls, "__abstractmethods__", frozenset()): + return # still abstract: not a factor yet, so not asked for a spec + raise TypeError( + f"Factor subclass {cls.__name__!r} must declare a FactorSpec as " + f"'spec' in its own class body — a class attribute, or a property " + f"returning one when the id depends on constructor params. Writing " + f"a factor means declaring its hypothesis (expected_ic_sign), its " + f"PIT contract and its input fields (see factors/spec.py)." + ) + if isinstance(declared, FactorSpec): + return # static class attribute: already a validated spec + if isinstance(declared, _DEFERRED_SPEC_TYPES): + return # needs an instance: _FactorMeta.__call__ validates the result + if isinstance(declared, (staticmethod, classmethod)) or inspect.isfunction(declared): + raise TypeError( + f"Factor subclass {cls.__name__!r} declares 'spec' as a plain " + f"{type(declared).__name__} — did you forget @property? Accessing " + f"'spec' would then yield the {type(declared).__name__} itself, not " + f"a FactorSpec, so the mandatory-spec contract would go unenforced " + f"until something tried to read it. Declare 'spec' as a FactorSpec " + f"class attribute, or decorate it with @property." + ) + raise TypeError( + f"Factor subclass {cls.__name__!r} declares spec={declared!r}, which is " + f"neither a FactorSpec nor a property returning one." + ) + + +class _FactorMeta(ABCMeta): + """Metaclass validating the ``spec`` at class definition AND at construction. + + WHY A METACLASS FOR THE DECLARATION CHECK (and not ``__init_subclass__``): + the check must skip classes that are still abstract, and + ``__abstractmethods__`` is computed by ``ABCMeta.__new__`` — which runs + ``__init_subclass__`` on the way, i.e. TOO EARLY: the hook would read the + PARENT's abstract set and mis-judge every intermediate base. ``__new__`` + after ``super().__new__`` is the first point where abstractness is known. + + WHY A METACLASS FOR THE INSTANCE CHECK (and not ``Factor.__init__``): + concrete factors define their own ``__init__`` and do not call + ``super().__init__()``, so a base-class ``__init__`` would simply never run — + the check would be silently dead. And a ``property`` spec cannot be evaluated + at class-definition time, since its ``factor_id`` depends on constructor + params that do not exist yet. ``ABCMeta.__call__`` is the one hook that sees + a FULLY built instance without touching a single factor constructor, so the + spec is checked at construction: an invalid spec cannot even be instantiated, + let alone reach an evaluator. + """ + + def __new__( + mcls, + name: str, + bases: tuple[type, ...], + namespace: dict, + **kwargs: object, + ) -> type: + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + _validate_declared_spec(cls) + return cls + + def __call__(cls, *args: object, **kwargs: object) -> object: + instance = super().__call__(*args, **kwargs) + _validate_instance_spec(instance) + return instance + -class Factor(ABC): +class Factor(ABC, metaclass=_FactorMeta): """Abstract cross-sectional factor. - Subclasses set the class attribute ``name`` (used as the factor-panel column) - and implement ``compute``. + Subclasses set the class attribute ``name`` (used as the factor-panel column), + declare a :class:`~factors.spec.FactorSpec` as ``spec``, and implement + ``compute``. """ name: str + # Annotation only: it must NOT land in ``Factor.__dict__``, or every subclass + # would inherit a "declared" spec and the check could never fire. (``Factor`` + # itself is abstract, so _validate_declared_spec skips it either way.) + spec: FactorSpec @abstractmethod def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/candidates.py b/factors/compute/candidates.py index 94b40c9..8e6a41a 100644 --- a/factors/compute/candidates.py +++ b/factors/compute/candidates.py @@ -29,10 +29,22 @@ from factors.base import Factor from factors.compute.momentum import MomentumFactor +from factors.spec import FactorSpec # daily_basic-derived value fields the pipeline can enrich + surface (P3-5). VALUE_FIELDS: tuple[str, ...] = ("value_ep", "value_bp") +# Per-field evaluation contract metadata for the value pack. expected_ic_sign=+1 +# for both: the classic value prior (cheap earns more than expensive), and the +# ONE hypothesis this project has confirmed on independent samples — P3-5 test +# IC 3/3 positive (0.037~0.056), P3-7 SUPPORTED on 2/2 holdout cells (SSE50 + +# CSI300, magnitude decayed), P3-8 GENERALIZES to CSI500 (test IC +0.0145 / +# +0.0127). Sign confirmed; profitability at portfolio level is NOT. +_VALUE_META: dict[str, str] = { + "value_ep": "Earnings yield 1/pe (daily_basic, published same day; pe<=0 -> NaN).", + "value_bp": "Book yield 1/pb (daily_basic, published same day; pb<=0 -> NaN).", +} + class ReversalFactor(Factor): """Short-horizon reversal: the exact negative of ``momentum_w``. @@ -48,6 +60,29 @@ def __init__(self, window: int = 20, price_col: str = "close") -> None: self._momentum = MomentumFactor(window=window, price_col=price_col) self.name = f"reversal_{window}" + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=-1: reversal IS -momentum by construction, so its + hypothesis is the exact negation of MomentumFactor's +1 prior (short- + horizon losers bounce back). Project evidence: no signal — P3-5 found + reversal_5/20 sign-flipping across cells. + """ + base = self._momentum.spec + return FactorSpec( + factor_id=self.name, + version="1.0", + description=f"Short-horizon reversal: the exact negative of {base.factor_id}.", + expected_ic_sign=-base.expected_ic_sign, + is_intraday=False, + forward_return_horizon=base.forward_return_horizon, + return_basis=base.return_basis, + input_fields=base.input_fields, + family="reversal", + min_history_bars=base.min_history_bars, + ) + def compute(self, panel: pd.DataFrame) -> pd.Series: return (-self._momentum.compute(panel)).rename(self.name) @@ -70,6 +105,35 @@ def __init__(self, window: int = 20, price_col: str = "close") -> None: self._price_col = price_col self.name = f"volatility_{window}" + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=-1: the low-volatility anomaly (low-vol stocks earn + MORE than high-vol ones), i.e. the factor as defined (higher = more + volatile) should correlate NEGATIVELY with forward returns. This is the + project's second independently-confirmed hypothesis: P3-5 test IC 3/3 + negative (-0.044~-0.079), P3-7 SUPPORTED on 2/2 holdout cells, P3-8 + GENERALIZES to CSI500 (test IC -0.0272). + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Trailing {self._window}-bar volatility: std (ddof=1) of daily " + f"{self._price_col} returns over a FULL window." + ), + expected_ic_sign=-1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + input_fields=(self._price_col,), + family="lowvol", + # pct_change loses row 0 and the rolling std needs ``window`` returns + # -> the leading ``window`` rows are NaN. + min_history_bars=self._window, + ) + def compute(self, panel: pd.DataFrame) -> pd.Series: if self._price_col not in panel.columns: raise ValueError( @@ -107,6 +171,34 @@ def __init__(self, window: int = 20, amount_col: str = "amount") -> None: self._amount_col = amount_col self.name = f"liquidity_{window}" + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=-1: the illiquidity-premium prior — less-traded stocks + earn more, so this factor (higher = MORE traded) should correlate + NEGATIVELY with forward returns. Project evidence: P3-5 test IC 3/3 + negative but small in magnitude; P3-6 showed adding it to the + value+lowvol subset is no free lunch (better on 1 cell, worse on 2). + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Log of the trailing {self._window}-bar mean turnover " + f"'{self._amount_col}' (non-positive mean -> NaN)." + ), + expected_ic_sign=-1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + input_fields=(self._amount_col,), + family="liquidity", + # rolling mean over ``window`` amounts -> first valid row is + # ``window-1`` (no pct_change involved, unlike volatility). + min_history_bars=self._window - 1, + ) + def compute(self, panel: pd.DataFrame) -> pd.Series: if self._amount_col not in panel.columns: raise ValueError( @@ -152,6 +244,36 @@ def __init__( self._close_col = close_col self.name = f"overnight_mom_{window}" + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=+1: the overnight-return premium prior (persistent + overnight demand keeps paying), i.e. momentum-like in direction. Project + evidence is mixed/weak: P3-5 test IC 2/3 positive (+0.008/+0.016) and + negative on the 2020-2022 cell. + + NOT is_intraday: it is derived from DAILY open/close bars, so it carries + no minute decision-cutoff / execution contract. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Sum of the last {self._window} overnight log returns " + f"log({self._open_col}[t] / {self._close_col}[t-1])." + ), + expected_ic_sign=+1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + input_fields=(self._open_col, self._close_col), + family="momentum", + # row 0 has no prior close and the rolling sum needs ``window`` full + # overnight returns -> the leading ``window`` rows are NaN. + min_history_bars=self._window, + ) + def compute(self, panel: pd.DataFrame) -> pd.Series: missing = [c for c in (self._open_col, self._close_col) if c not in panel.columns] if missing: @@ -188,6 +310,27 @@ def __init__(self, field: str) -> None: self.name = field self._field = field + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property because the id IS the chosen field. + + expected_ic_sign=+1 for both fields — see ``_VALUE_META`` above for the + prior and the project's independent confirmation (P3-5/P3-7/P3-8). + ``min_history_bars=0``: the ratio is published same-day, no warm-up. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=_VALUE_META[self._field], + expected_ic_sign=+1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + input_fields=(self._field,), + family="value", + min_history_bars=0, + ) + def compute(self, panel: pd.DataFrame) -> pd.Series: if self._field not in panel.columns: raise ValueError( diff --git a/factors/compute/financial.py b/factors/compute/financial.py index 93b2577..35bf93b 100644 --- a/factors/compute/financial.py +++ b/factors/compute/financial.py @@ -12,12 +12,39 @@ import pandas as pd from factors.base import Factor +from factors.spec import FactorSpec # financial fields that may be requested as a factor (P1; grossprofit_margin # joined in P3-5 as the conservative quality candidate — same ann_date as-of # machinery, no new temporal logic). SUPPORTED_FIELDS: tuple[str, ...] = ("roe", "netprofit_yoy", "grossprofit_margin") +# Per-field evaluation contract metadata (hypothesis fixed BEFORE any run). +# All three carry the conventional +1 prior: more profitable / faster-growing / +# higher-margin firms are hypothesized to earn higher cross-sectional returns. +# The project's own evidence is weak-to-null for roe / netprofit_yoy (P3-1: IC +# 0.0006 / 0.0001 on SSE50; P3-3/P3-4: signs flip across cells) and +# grossprofit_margin showed no signal in P3-5 — the prior is still the STATED +# hypothesis; the verdict is what checks it. +_FIELD_META: dict[str, tuple[int, str, str]] = { + # field: (expected_ic_sign, family, description) + "roe": ( + +1, + "quality", + "Return on equity (fina_indicator), PIT-aligned by ann_date disclosure.", + ), + "netprofit_yoy": ( + +1, + "growth", + "Net-profit YoY growth (fina_indicator), PIT-aligned by ann_date.", + ), + "grossprofit_margin": ( + +1, + "quality", + "Gross-profit margin (fina_indicator), PIT-aligned by ann_date.", + ), +} + class FinancialFactor(Factor): """Surface a PIT-aligned financial column (e.g. ``roe``) as a factor.""" @@ -31,6 +58,28 @@ def __init__(self, field: str = "roe") -> None: self.name = field self._field = field + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property because the id IS the chosen field. + + ``min_history_bars=0``: this factor does no temporal logic of its own — + the value is already PIT-aligned upstream by ``ann_date`` as-of, so there + is no warm-up window to exclude. + """ + sign, family, description = _FIELD_META[self._field] + return FactorSpec( + factor_id=self.name, + version="1.0", + description=description, + expected_ic_sign=sign, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + input_fields=(self._field,), + family=family, + min_history_bars=0, + ) + def compute(self, panel: pd.DataFrame) -> pd.Series: """Return the as-of financial column as a MultiIndex(date, symbol) series.""" if self._field not in panel.columns: diff --git a/factors/compute/intraday_derived.py b/factors/compute/intraday_derived.py new file mode 100644 index 0000000..1f0947c --- /dev/null +++ b/factors/compute/intraday_derived.py @@ -0,0 +1,118 @@ +"""Daily factors DERIVED from intraday (minute) bars but executed close-to-close. + +The single member today is :class:`JumpAmountCorrFactor` (PR-C), the Kaiyuan +report §6 "price-jump turnover correlation" factor. Like the value / MMP factors, +the heavy computation runs UPSTREAM (``data.clean.intraday_aggregate. +compute_jump_amount_corr`` aggregates 1min bars into a daily +``MultiIndex(date, symbol)`` column) and the Factor here simply SELECTS its column +off the panel the runner already enriched. Keeping the minute aggregation in the +data-clean layer preserves the layering: ``factors`` never fetches and never sees a +forward return. + +WHY ``is_intraday=False`` FOR A MINUTE-DERIVED FACTOR (deliberate, documented): + ``FactorSpec.is_intraday`` flags an intraday-EXECUTION contract — the minute + tail model that DECIDES at 14:50 and FILLS at 14:51, whose holding period runs + exec(T) -> exec(T_next) (I5a). This factor has minute INPUT but its signal is a + DAILY value traded at the daily close and held close-to-close (t -> t+1), just + like the report's monthly rebalance on the daily close and the project's daily + default. It carries no 14:50 decision cutoff, no execution window, no exec-to- + exec holding period — so ``is_intraday`` is False, ``return_basis`` is + ``"close_to_close"``, and the five minute-block spec fields are all None. (The + base ``FactorSpec`` deliberately does NOT force is_intraday from a minute + provenance; a daily signal computed from minute data is a legitimate case.) +""" + +from __future__ import annotations + +import pandas as pd + +from data.clean.intraday_aggregate import ( + JUMP_LOOKBACK_DAYS, + JUMP_MIN_PAIRS, +) +from factors.base import Factor +from factors.spec import FactorSpec + + +class JumpAmountCorrFactor(Factor): + """Price-jump turnover-correlation factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the + panel (produced by ``compute_jump_amount_corr``); it does NO minute work of its + own, mirroring the value / financial factors that surface an enriched column. + + Args: + lookback_days: trailing trading-day window; part of the factor DEFINITION + (reproduced from the report), not a tuned knob. It only names the + column so a non-default window cannot silently mislabel it. + """ + + name: str = f"jump_amount_corr_{JUMP_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = JUMP_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"jump-amount-corr lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"jump_amount_corr_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=-1: the report's RankIC mean is -10.23% (full A, market-cap + + industry neutral) — high jump-amount-correlation predicts LOWER forward + returns. The sign is fixed BEFORE the run; a validated prototype reproduced + it (mean RankIC -0.074 on 2022-2024 sampled names). is_intraday=False by the + module docstring's reasoning (daily signal traded close-to-close). + min_history_bars=0: the warm-up is DATA-dependent (a value appears once + >= ``JUMP_MIN_PAIRS`` jump-pairs accumulate in the trailing window), not a + fixed leading count — the honest NaN rate is reported by data_coverage + rather than hidden behind a fabricated warm-up window. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Price-jump turnover correlation (Kaiyuan report §6): trailing " + f"{self._lookback_days}-trading-day lagged Pearson corr between the " + f"traded amount at price-JUMP minutes (within-day amplitude z-score " + f">1) and the amount at the strictly-next minute. Derived from 1min " + f"bars but a DAILY signal traded close-to-close; >= {JUMP_MIN_PAIRS} " + f"jump-pairs required else NaN." + ), + expected_ic_sign=-1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from. These + # are declared for honest provenance disclosure (data_coverage lists + # them); the daily panel surfaces the pre-aggregated column itself. + input_fields=("high", "low", "open", "amount"), + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily jump-amount-corr column off ``panel``. + + The runner runs ``compute_jump_amount_corr`` on the minute cache upstream + and joins the result as ``self.name``; here we only surface it, so this + factor does no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"JumpAmountCorrFactor needs the pre-aggregated '{self.name}' column " + f"on the panel (produced upstream by compute_jump_amount_corr and " + f"joined by the runner); panel has {list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = ["JumpAmountCorrFactor"] diff --git a/factors/compute/momentum.py b/factors/compute/momentum.py index 9d074eb..7bb758f 100644 --- a/factors/compute/momentum.py +++ b/factors/compute/momentum.py @@ -22,6 +22,7 @@ import pandas as pd from factors.base import Factor +from factors.spec import FactorSpec class MomentumFactor(Factor): @@ -54,6 +55,35 @@ def window(self) -> int: def price_col(self) -> str: return self._price_col + @property + def spec(self) -> FactorSpec: + """Evaluation contract. A property, not a class attribute: ``factor_id`` + must track the ACTUAL window (mirrors the ``self.name`` idiom above). + + expected_ic_sign=+1: the classic cross-sectional momentum prior (winners + keep winning). NOTE the project's own evidence disagrees in magnitude — + P3-3/P3-4 found momentum_20 IC ~= 0 and sign-flipping across cells — but + the prior stays the STATED hypothesis, which is the point: the sign is + fixed before the run and the verdict then checks it factually. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Trailing {self._window}-bar price momentum: " + f"{self._price_col}[t] / {self._price_col}[t-{self._window}] - 1." + ), + expected_ic_sign=+1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + input_fields=(self._price_col,), + family="momentum", + # value at t needs bars t-window..t -> the leading ``window`` rows + # of every symbol are NaN by construction. + min_history_bars=self._window, + ) + def compute(self, panel: pd.DataFrame) -> pd.Series: """Compute per-symbol momentum aligned to the panel index. diff --git a/factors/spec.py b/factors/spec.py new file mode 100644 index 0000000..4ac5ef4 --- /dev/null +++ b/factors/spec.py @@ -0,0 +1,249 @@ +"""``FactorSpec``: the factor-INTRINSIC half of the factor-evaluation contract. + +Two objects together form the provenance an evaluator requires (design doc +``tmp/design/factor_eval_contract_v0.1.md`` §1): + + * :class:`FactorSpec` (HERE) — identity + PIT contract + declared inputs. + It describes the factor itself, travels with it, and barely ever changes. + * ``analytics.eval.EvalConfig`` — the per-run parameters + honesty flags. + +WHY THIS LIVES IN ``factors/`` (layering invariant #3, 分层解耦): + the project layering is ``data -> universe -> factors -> alpha -> portfolio + -> runtime -> analytics``. ``Factor`` must HOLD its spec (the base class + enforces it at class-definition time), so if ``FactorSpec`` lived in + ``analytics/`` then ``factors`` would import ``analytics`` — an UPWARD + dependency. ``analytics`` importing ``factors.spec`` is downstream->upstream + and therefore fine. This module deliberately imports nothing from the + project (no pandas either): it is a pure declaration. + +The construction-time validators below are enforcement layer #1 of three (design +§7): declare the hypothesis and the PIT contract, or you cannot even build the +object. ``expected_ic_sign`` may NOT be None — a factor author must commit to a +direction BEFORE the run, so the verdict can be a factual sign check rather than +an after-the-fact story. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +# The ONLY basis an intraday factor may declare: the minute tail model's holding +# period runs execution-anchor to execution-anchor, exec(T) -> exec(T_next), and +# is NEVER close-to-close (I5a). Cross-checked in _check_intraday_block. +INTRADAY_RETURN_BASIS = "exec_to_exec" + +# How the evaluated forward return is measured. +# close_to_close -> daily close(t+h)/close(t) - 1 (the daily pipeline) +# exec_to_exec -> execution-anchored, e.g. the minute tail model's +# exec(T_next)/exec(T) - 1 (NEVER close-to-close; I5a). +RETURN_BASES: tuple[str, ...] = ("close_to_close", INTRADAY_RETURN_BASIS) + +# Price-adjustment conventions. The framework front-adjusts (qfq) in memory and +# stores raw; no other basis is supported today, so anything else is an error +# rather than a silently-wrong evaluation. +PRICE_ADJUSTMENTS: tuple[str, ...] = ("qfq",) + +# The minute block: all five are required when ``is_intraday`` is True and must +# all be None otherwise. A half-declared intraday contract is the exact failure +# mode this guards (three timestamps kept separate all the way through: signal +# cutoff / execution timestamp / holding period — see runtime/intraday_*). +INTRADAY_FIELDS: tuple[str, ...] = ( + "decision_cutoff", + "data_lag", + "session_open", + "execution_model", + "execution_window", +) + + +@dataclass(frozen=True) +class FactorSpec: + """Identity + PIT contract + declared inputs of ONE factor (immutable). + + Attributes + ---------- + factor_id : canonical unique name; equals the ``Factor.name`` panel column. + version : bump on any redefinition, so cross-run records stay comparable. + description : one line — what does this factor measure? + expected_ic_sign : +1 or -1, the hypothesis, FIXED BEFORE THE RUN. Drives the + OOS sign check in the verdict. **None is forbidden** by design. + is_intraday : whether the factor is derived from intraday bars; decides + whether the minute block is required. + forward_return_horizon : h > 0, the horizon (in evaluation periods) the + factor claims to predict; the IC is aligned to it. + return_basis : one of :data:`RETURN_BASES`. + input_fields : the panel columns the factor actually reads, so an evaluator + can check availability + coverage instead of guessing. + price_adjust : one of :data:`PRICE_ADJUSTMENTS`. + family : orthogonality grouping (momentum/value/lowvol/microstructure/...). + min_history_bars : leading warm-up bars that are NaN by construction, so the + evaluator does not charge the factor for its own warm-up window. + decision_cutoff, data_lag, session_open, execution_model, execution_window : + the intraday block (see :data:`INTRADAY_FIELDS`). + """ + + factor_id: str + version: str + description: str + expected_ic_sign: int + is_intraday: bool + forward_return_horizon: int + return_basis: str + input_fields: tuple[str, ...] + price_adjust: str = "qfq" + family: str | None = None + min_history_bars: int = 0 + decision_cutoff: str | None = None + data_lag: str | None = None + session_open: str | None = None + execution_model: str | None = None + execution_window: str | None = None + + def __post_init__(self) -> None: + self._check_identity() + self._check_hypothesis() + self._check_measurement() + self._check_inputs() + self._check_intraday_block() + + # -- validators (enforcement layer #1) -------------------------------- + + def _check_identity(self) -> None: + for field_name in ("factor_id", "version", "description"): + value = getattr(self, field_name) + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"FactorSpec.{field_name} must be a non-empty string; got " + f"{value!r}." + ) + + def _check_hypothesis(self) -> None: + sign = self.expected_ic_sign + # Two Python gotchas guarded at once: ``True`` is an int subclass and + # ``1.0 == 1``, so both would sneak past a bare ``in (1, -1)`` and land a + # non-int in the declared-int field (and in the exported record). + if isinstance(sign, bool) or not isinstance(sign, int) or sign not in (1, -1): + raise ValueError( + f"FactorSpec.expected_ic_sign must be +1 or -1 (never None): the " + f"factor author must commit to a direction BEFORE the run so the " + f"verdict stays a factual sign check. Got {sign!r} for " + f"{self.factor_id!r}." + ) + if not isinstance(self.is_intraday, bool): + raise ValueError( + f"FactorSpec.is_intraday must be a bool; got {self.is_intraday!r}." + ) + + def _check_measurement(self) -> None: + horizon = self.forward_return_horizon + if isinstance(horizon, bool) or not isinstance(horizon, int) or horizon <= 0: + raise ValueError( + f"FactorSpec.forward_return_horizon must be a positive int (the " + f"horizon the factor claims to predict); got {horizon!r} for " + f"{self.factor_id!r}." + ) + if self.return_basis not in RETURN_BASES: + raise ValueError( + f"FactorSpec.return_basis must be one of {RETURN_BASES}; got " + f"{self.return_basis!r} for {self.factor_id!r}." + ) + if self.price_adjust not in PRICE_ADJUSTMENTS: + raise ValueError( + f"FactorSpec.price_adjust must be one of {PRICE_ADJUSTMENTS} (the " + f"only supported basis today); got {self.price_adjust!r} for " + f"{self.factor_id!r}." + ) + bars = self.min_history_bars + if isinstance(bars, bool) or not isinstance(bars, int) or bars < 0: + raise ValueError( + f"FactorSpec.min_history_bars must be a non-negative int; got " + f"{bars!r} for {self.factor_id!r}." + ) + if self.family is not None and ( + not isinstance(self.family, str) or not self.family.strip() + ): + raise ValueError( + f"FactorSpec.family must be None or a non-empty string; got " + f"{self.family!r} for {self.factor_id!r}." + ) + + def _check_inputs(self) -> None: + fields = self.input_fields + # A bare string would silently become a tuple of single characters. + if isinstance(fields, str) or not isinstance(fields, Sequence): + raise ValueError( + f"FactorSpec.input_fields must be a sequence of panel column " + f"names (not a bare string); got {fields!r} for {self.factor_id!r}." + ) + normalized = tuple(fields) + if not normalized: + raise ValueError( + f"FactorSpec.input_fields must be non-empty: an evaluator checks " + f"availability + coverage of the columns the factor reads " + f"({self.factor_id!r})." + ) + bad = [f for f in normalized if not isinstance(f, str) or not f.strip()] + if bad: + raise ValueError( + f"FactorSpec.input_fields entries must be non-empty strings; got " + f"{bad!r} for {self.factor_id!r}." + ) + # Store a tuple even when handed a list, so the frozen spec stays + # immutable + hashable. + object.__setattr__(self, "input_fields", normalized) + + def _check_intraday_block(self) -> None: + present = [f for f in INTRADAY_FIELDS if getattr(self, f) is not None] + if self.is_intraday: + missing = [f for f in INTRADAY_FIELDS if getattr(self, f) is None] + if missing: + raise ValueError( + f"FactorSpec({self.factor_id!r}) is_intraday=True requires the " + f"whole minute block {INTRADAY_FIELDS}; missing {missing}. A " + f"half-declared intraday contract (cutoff without execution " + f"model, ...) is exactly what this guard rejects." + ) + # Present-but-blank is the SAME half-declared contract as missing: a + # block of empty strings declares nothing while passing a None check. + blank = [ + f + for f in INTRADAY_FIELDS + if not isinstance(getattr(self, f), str) or not getattr(self, f).strip() + ] + if blank: + raise ValueError( + f"FactorSpec({self.factor_id!r}) minute block entries must be " + f"non-empty strings; got blank/non-string {blank}. An empty " + f"cutoff or execution window declares nothing — it is the same " + f"half-declared intraday contract as omitting the field." + ) + if self.return_basis != INTRADAY_RETURN_BASIS: + raise ValueError( + f"FactorSpec({self.factor_id!r}) is_intraday=True requires " + f"return_basis={INTRADAY_RETURN_BASIS!r}; got " + f"{self.return_basis!r}. An intraday factor's holding period is " + f"execution-anchored (exec(T) -> exec(T_next)) and is NEVER " + f"close-to-close (I5a)." + ) + # NOTE the CONVERSE is deliberately NOT enforced: exec_to_exec does + # NOT imply is_intraday. A DAILY-computed signal executed on the + # minute tail is a legitimate combination — it is exactly this + # project's I5a/I5b path (a daily factor decided at 14:50, filled at + # 14:51, held exec-to-exec). Locking the converse in would reject it. + # Please do not "tighten" this into an iff. + elif present: + raise ValueError( + f"FactorSpec({self.factor_id!r}) is_intraday=False requires the " + f"whole minute block to be None; got {present} set. A daily factor " + f"declaring an execution window would misdescribe its PIT contract." + ) + + +__all__ = [ + "FactorSpec", + "RETURN_BASES", + "PRICE_ADJUSTMENTS", + "INTRADAY_FIELDS", + "INTRADAY_RETURN_BASIS", +] diff --git a/qt/cli.py b/qt/cli.py index 02fff19..ab0ae9e 100644 --- a/qt/cli.py +++ b/qt/cli.py @@ -203,6 +203,31 @@ def _cmd_run_phase_i5d_intraday_groups(args: argparse.Namespace) -> int: return 0 +def _cmd_run_eval_jump_amount_corr(args: argparse.Namespace) -> int: + """Run the two real jump-amount-corr factor evaluations (cache-only) + reports.""" + from qt.eval_jump_amount_corr import run_eval_jump_amount_corr + + try: + result = run_eval_jump_amount_corr(args.config) + except (ConfigError, ValueError, FileNotFoundError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + nb, wb = result.no_book_metrics, result.with_book_metrics + print( + f"OK run-eval-jump-amount-corr: covered={result.covered_symbols}/" + f"{result.requested_symbols}, stk_mins_live_calls={result.minute_live_calls}, " + f"factor_rows={result.factor_rows} ({result.elapsed:.1f}s)\n" + f"no-book: {nb['deployment']} (predictive={nb['predictive']}) " + f"ic_mean={nb['ic_mean']:.4f} ic_ir={nb['ic_ir']:.3f} N_eff={nb['effective_samples']:.1f}\n" + f"with-book: {wb['deployment']} (incremental={wb['incremental']}) " + f"incr_ic_ir={wb['incremental_ic_ir']:.3f}\n" + f"reports: {result.reports.no_book_md} | {result.reports.with_book_md}\n" + f"dashboards: {result.reports.no_book_dashboard} | " + f"{result.reports.with_book_dashboard}" + ) + return 0 + + def _cmd_data_update(args: argparse.Namespace) -> int: """Warm/update the tushare caches (P4-3); never runs a backtest.""" from qt.data_updater import format_summary, run_data_update @@ -334,6 +359,13 @@ def build_parser() -> argparse.ArgumentParser: p_i5d.add_argument("--config", required=True, help="Path to the YAML config.") p_i5d.set_defaults(func=_cmd_run_phase_i5d_intraday_groups) + p_jac = sub.add_parser( + "run-eval-jump-amount-corr", + help="Run the first real factor evaluation (jump-amount-corr, CSI500, cache-only).", + ) + p_jac.add_argument("--config", required=True, help="Path to the YAML config.") + p_jac.set_defaults(func=_cmd_run_eval_jump_amount_corr) + for name, func, help_text in ( ("fetch-data", _cmd_fetch_data, "Run the spine, report data fetch."), ("compute-factors", _cmd_compute_factors, "Run the spine, report factor compute."), diff --git a/qt/eval_jump_amount_corr.py b/qt/eval_jump_amount_corr.py new file mode 100644 index 0000000..267d071 --- /dev/null +++ b/qt/eval_jump_amount_corr.py @@ -0,0 +1,477 @@ +"""run-eval-jump-amount-corr: the FIRST real factor evaluation (PR-C). + +Reproduces the Kaiyuan report §6 "price-jump turnover correlation" factor as a +first-class :class:`~factors.compute.intraday_derived.JumpAmountCorrFactor` and +runs it through the FROZEN :class:`~analytics.eval.StandardFactorEvaluator` on REAL +cached A-share data (CSI500, PIT membership), closing the contract-driven loop. + +The factor is a DAILY signal derived from 1min bars (see +``data.clean.intraday_aggregate.compute_jump_amount_corr``): the trailing-20- +trading-day lagged correlation between the amount at price-jump minutes and the +amount at the strictly-next minute. It is executed CLOSE-TO-CLOSE (daily default), +so ``is_intraday=False`` (the reasoning is documented on the factor's spec). + +CACHE-ONLY: every input is read from the persistent tushare cache +(``artifacts/cache/tushare/v1``). The minute read is provably live-call-free (the +minute store has no fetch closure — a miss simply yields no rows); the daily / +universe / covariate endpoints go through the shared read-through cache, which on +a fully-warmed cache does zero gap fetches (disclosed via the run-log cache-stats +line). Forward returns are computed ONLY at the evaluator/analytics boundary from +``ctx.price_panel`` — the factor computation never sees a future return. + +The evaluator is run TWICE: once with NO known-factor book (the Incremental axis +is NOT_ASSESSED) and once with the project's independently-confirmed book +(value_ep / value_bp / volatility_20) so the Incremental axis measures whether +jump-amount-corr adds alpha BEYOND value / low-vol. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from pathlib import Path + +import pandas as pd + +from analytics.eval import ( + EvalConfig, + EvalContext, + FactorEvalReport, + StandardFactorEvaluator, +) +from analytics.eval.figures import render_factor_dashboard +from data.cache.intraday_cache import ENDPOINT as INTRADAY_ENDPOINT +from data.cache.intraday_cache import READ_COLUMNS +from data.cache.intraday_parquet_store import IntradayParquetStore +from data.clean.intraday_aggregate import ( + JUMP_LOOKBACK_DAYS, + JUMP_MIN_PAIRS, + compute_jump_amount_corr, +) +from data.clean.intraday_schema import RAW_INTRADAY_FREQ, normalize_intraday_bars +from data.clean.schema import CORE_COLUMNS, DATE_LEVEL +from factors.compute.intraday_derived import JumpAmountCorrFactor +from factors.spec import FactorSpec +from qt.config import RootConfig, load_config +from qt.pipeline import ( + _build_cache, + _build_universe, + _load_panel, + _log_run_cache_stats, + _make_logger, + _maybe_enrich_covariates, + _maybe_enrich_value, + _process_factors, +) + +_LOGGER_NAME = "qt.eval_jump_amount_corr" +_REPORT_STEM = "eval_jump_amount_corr" + + +# --------------------------------------------------------------------------- # +# Minute loading (cache-only, per-symbol -> memory-bounded) +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class _JumpMinuteLoad: + """Diagnostics from the cache-only per-symbol minute read + aggregation.""" + + factor: pd.Series # MultiIndex(date, symbol) raw jump-amount-corr + requested: int + covered: tuple[str, ...] # symbols that produced >= 1 finite factor value + empty_symbols: tuple[str, ...] # requested but no cached minute / no value + raw_rows: int + live_calls: int # provably 0 (store read has no fetch closure) + + +def _load_jump_factor_panel( + cfg: RootConfig, + symbols: list[str], + spec: FactorSpec, + logger, + *, + lookback_days: int, + min_pairs: int, +) -> _JumpMinuteLoad: + """Compute the raw jump-amount-corr panel per symbol from the minute cache. + + Memory-bounded: one symbol's minute history is read, aggregated to its daily + factor series, and discarded before the next — the multi-year all-symbol minute + panel is NEVER materialized. Read-only: :meth:`IntradayParquetStore.read_range` + has no fetch closure, so ``stk_mins`` live calls are provably zero (a symbol + with no cached minute simply yields no rows and is disclosed as empty). + """ + root = cfg.data.cache.root_dir + store = IntradayParquetStore(root) + start = pd.Timestamp(cfg.data.start).normalize() + end = pd.Timestamp(cfg.data.end).normalize() + pd.Timedelta("23:59:59") + + series: list[pd.Series] = [] + covered: list[str] = [] + empty: list[str] = [] + raw_rows = 0 + for i, sym in enumerate(symbols): + part = store.read_range(INTRADAY_ENDPOINT, sym, RAW_INTRADAY_FREQ, start, end) + if part.empty: + empty.append(sym) + continue + raw_rows += len(part) + bars = normalize_intraday_bars( + part.rename(columns={"bar_end": "time"})[READ_COLUMNS], + freq=RAW_INTRADAY_FREQ, + ) + s = compute_jump_amount_corr( + bars, lookback_days=lookback_days, min_pairs=min_pairs, name=spec.factor_id + ) + if s.notna().any(): + series.append(s) + covered.append(sym) + else: + empty.append(sym) + if (i + 1) % 100 == 0: + logger.info( + "minute aggregation: %d/%d symbols processed (%d with a value)", + i + 1, len(symbols), len(covered), + ) + + if not series: + raise ValueError( + "run-eval-jump-amount-corr blocked: no requested symbol produced a " + f"cached minute jump-amount-corr value over [{cfg.data.start}, " + f"{cfg.data.end}]. The minute cache is required (this runner never warms " + "it); check coverage." + ) + factor = pd.concat(series).sort_index() + logger.info( + "minute aggregation (cache-only): %d/%d symbols with a value, %d raw 1min " + "rows read, %d factor rows, stk_mins_live_calls=0", + len(covered), len(symbols), raw_rows, len(factor), + ) + return _JumpMinuteLoad( + factor=factor, + requested=len(symbols), + covered=tuple(covered), + empty_symbols=tuple(empty), + raw_rows=raw_rows, + live_calls=0, + ) + + +# --------------------------------------------------------------------------- # +# Evaluation core (network-free seam: given panels, run the two evaluations) +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class _RunReports: + """The two evaluation reports (no-book / with-book) + their file paths.""" + + no_book: FactorEvalReport + with_book: FactorEvalReport + no_book_md: Path + no_book_json: Path + with_book_md: Path + with_book_json: Path + no_book_dashboard: Path + with_book_dashboard: Path + + +def evaluate_two_runs( + factor_panel: pd.Series | pd.DataFrame, + spec: FactorSpec, + eval_cfg: EvalConfig, + price_panel: pd.DataFrame, + book: pd.DataFrame, + *, + universe_symbols: tuple[str, ...], + fee_rate: float, + report_dir: Path, + stem: str = _REPORT_STEM, +) -> _RunReports: + """Run the StandardFactorEvaluator TWICE (no book / with book) and write reports. + + This is the network-free seam: given the PROCESSED factor panel, the qfq price + panel (for forward returns), and the PROCESSED known-factor book, it does the + two ``evaluate`` calls and writes ``{stem}_no_book`` / ``{stem}_with_book`` as + Markdown + JSON. Run 1 omits ``known_factors`` (Incremental NOT_ASSESSED); run 2 + supplies the book (Incremental measured). + """ + evaluator = StandardFactorEvaluator() + report_dir.mkdir(parents=True, exist_ok=True) + + ctx_no_book = EvalContext( + price_panel=price_panel, + universe_symbols=universe_symbols, + fee_rate=fee_rate, + ) + # evaluate_with_ir yields the SAME report as evaluate() plus the IR the + # research-style dashboard needs (per-period IC + quantile return series). + report_no_book, ir_no_book = evaluator.evaluate_with_ir( + factor_panel, spec, eval_cfg, ctx_no_book + ) + + ctx_with_book = EvalContext( + price_panel=price_panel, + universe_symbols=universe_symbols, + fee_rate=fee_rate, + known_factors=book, + ) + report_with_book, ir_with_book = evaluator.evaluate_with_ir( + factor_panel, spec, eval_cfg, ctx_with_book + ) + + nb_md, nb_json = _write_report(report_no_book, report_dir, f"{stem}_no_book") + wb_md, wb_json = _write_report(report_with_book, report_dir, f"{stem}_with_book") + nb_png = render_factor_dashboard( + report_no_book, ir_no_book, report_dir / f"{stem}_no_book_dashboard.png" + ) + wb_png = render_factor_dashboard( + report_with_book, ir_with_book, report_dir / f"{stem}_with_book_dashboard.png" + ) + return _RunReports( + no_book=report_no_book, + with_book=report_with_book, + no_book_md=nb_md, + no_book_json=nb_json, + with_book_md=wb_md, + with_book_json=wb_json, + no_book_dashboard=nb_png, + with_book_dashboard=wb_png, + ) + + +def _write_report(report: FactorEvalReport, report_dir: Path, stem: str) -> tuple[Path, Path]: + """Write ``report`` as deterministic Markdown + JSON; return the two paths.""" + md_path = report_dir / f"{stem}.md" + json_path = report_dir / f"{stem}.json" + md_path.write_text(report.render(), encoding="utf-8") + json_path.write_text(report.to_json(), encoding="utf-8") + return md_path, json_path + + +# --------------------------------------------------------------------------- # +# Metric extraction (for the CLI line + the handoff) +# --------------------------------------------------------------------------- # +def _section_payload(report: FactorEvalReport, name: str) -> dict: + section = report.by_name().get(name) + return dict(getattr(section, "payload", {}) or {}) + + +def extract_metrics(report: FactorEvalReport) -> dict: + """Pull the headline verdict + gated metrics out of a finished report.""" + verdict = report.require_verdict() + pred = _section_payload(report, "predictive_power") + purity = _section_payload(report, "data_coverage") + incr = _section_payload(report, "purity") + return { + "deployment": verdict.verdict, + "predictive": verdict.predictive.verdict, + "incremental": verdict.incremental.verdict, + "tradable": verdict.tradable.verdict, + "ic_mean": pred.get("ic_mean"), + "ic_ir": pred.get("ic_ir"), + "ic_ir_ci_low": pred.get("ic_ir_ci_low"), + "ic_ir_ci_high": pred.get("ic_ir_ci_high"), + "ic_ir_ci_n_eff": pred.get("ic_ir_ci_n_eff"), + "ic_win_rate": pred.get("ic_win_rate"), + "ic_nw_t": pred.get("ic_nw_t"), + "settled_rebalances": purity.get("settled_rebalances"), + "effective_samples": purity.get("effective_samples"), + "span_days": purity.get("span_days"), + "incremental_ic_ir": incr.get("incremental_ic_ir"), + "incremental_ic_ir_ci_low": incr.get("incremental_ic_ir_ci_low"), + "incremental_ic_ir_ci_high": incr.get("incremental_ic_ir_ci_high"), + "incremental_ic_ir_ci_n_eff": incr.get("incremental_ic_ir_ci_n_eff"), + "incremental_ic_mean": incr.get("incremental_ic_mean"), + } + + +# --------------------------------------------------------------------------- # +# EvalConfig construction (HONEST provenance of what the runner actually did) +# --------------------------------------------------------------------------- # +def _build_eval_config(cfg: RootConfig) -> EvalConfig: + """Build the per-run EvalConfig, declaring EXACTLY what the pipeline applied. + + The declarations must not overstate: this codebase's winsorize step is a P0 + no-op, so ``winsorize`` is declared None (nothing was clipped) even though the + config may toggle it. z-score + industry/size neutralization ARE applied, so + they are declared. ``oos_split`` (from the config's ``oos`` block) makes the + OOS section run so the Predictive axis can be assessed. ``is_exploratory=True``: + this is a reproduction on a shorter window / narrower neutralization than the + report, not a return claim (it caps the deployment label at Watch). + """ + if cfg.oos is None: + raise ValueError( + "run-eval-jump-amount-corr requires an 'oos' section (split_date) so the " + "Predictive axis has an out-of-sample split to assess; add e.g. " + "oos: {split_date: '2024-01-01'}." + ) + return EvalConfig( + universe=cfg.universe.index_code or cfg.universe.type, + universe_is_pit=cfg.universe.type == "index", + start=cfg.data.start, + end=cfg.data.end, + is_exploratory=True, + post_hoc_selected=False, + rebalance="daily", + n_quantiles=int(cfg.analytics.quantiles), + cost_scenarios=(1.0, 2.0, 4.0), + oos_split=cfg.oos.split_date, + # winsorize is a P0 no-op in this codebase -> declare None (nothing clipped). + winsorize=None, + standardize="zscore" if cfg.processing.standardize.enabled else None, + neutralization=("industry", "size") if cfg.processing.neutralize.enabled else (), + industry_level=cfg.processing.neutralize.industry_level, + tuned=False, + # We evaluated ONE pre-registered factor whose sign came from the report + # (not a screen of our own); the report's own 17-factor screen is a caveat + # noted in the run's prose, not our multiple-testing background. + n_factors_screened=1, + data_snapshot_id=cfg.data.cache.root_dir, + ) + + +# --------------------------------------------------------------------------- # +# Result container + the full glue +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class JumpEvalResult: + """Immutable summary of one run-eval-jump-amount-corr run.""" + + config: RootConfig + spec: FactorSpec + requested_symbols: int + covered_symbols: int + empty_symbols: int + factor_rows: int + minute_raw_rows: int + minute_live_calls: int + no_book_metrics: dict + with_book_metrics: dict + reports: _RunReports + log_path: Path + elapsed: float + + +def _check_preconditions(cfg: RootConfig) -> None: + """Fail readably if the config cannot drive a real, cache-only CSI500 eval.""" + if cfg.data.source != "tushare": + raise ValueError( + "run-eval-jump-amount-corr needs data.source='tushare' (real cached " + f"A-share data); got {cfg.data.source!r}." + ) + if not cfg.data.cache.enabled: + raise ValueError( + "run-eval-jump-amount-corr needs data.cache.enabled=true (it reads the " + "persistent tushare cache and never warms live)." + ) + if cfg.universe.type != "index": + raise ValueError( + "run-eval-jump-amount-corr needs universe.type='index' (PIT membership, " + f"e.g. 000905.SH for CSI500); got {cfg.universe.type!r}." + ) + if not cfg.processing.neutralize.enabled: + raise ValueError( + "run-eval-jump-amount-corr expects processing.neutralize.enabled=true " + "(industry + size neutralization, matching the report's neutral column " + "and the EvalConfig declaration)." + ) + + +def run_eval_jump_amount_corr(config_path: str) -> JumpEvalResult: + """Run the two real jump-amount-corr evaluations (cache-only) and write reports.""" + cfg = load_config(config_path) + _check_preconditions(cfg) + + log_path = Path(cfg.output.log_dir) / f"{_REPORT_STEM}.log" + logger = _make_logger(log_path, name=_LOGGER_NAME) + started = time.monotonic() + + factor = JumpAmountCorrFactor(lookback_days=JUMP_LOOKBACK_DAYS) + spec = factor.spec + eval_cfg = _build_eval_config(cfg) + logger.info("eval config: %s rebalance=daily oos_split=%s", eval_cfg.universe, eval_cfg.oos_split) + + cache = _build_cache(cfg) + universe, symbols = _build_universe(cfg, logger, cache) + panel = _load_panel(cfg, symbols, logger, cache) + + # Book (value_ep / value_bp / volatility_20): enrich, compute, process. The + # value factors need daily_basic pe/pb; volatility_20 needs close. + book_factors = _build_book_factors() + panel = _maybe_enrich_value(cfg, panel, symbols, book_factors, logger, cache) + panel = _maybe_enrich_covariates(cfg, panel, symbols, logger, cache) + _log_run_cache_stats(cache, logger) # daily/universe/covariate gap-fetches (warm -> 0) + + panel_dates = pd.Index( + pd.unique(panel.index.get_level_values(DATE_LEVEL)), name=DATE_LEVEL + ) + book_raw = pd.concat( + [f.compute(panel).rename(f.name) for f in book_factors], axis=1 + ) + book_processed = _process_factors(cfg, book_raw, panel) + + # Jump factor: cache-only per-symbol minute aggregation -> raw -> process. + load = _load_jump_factor_panel( + cfg, symbols, spec, logger, + lookback_days=JUMP_LOOKBACK_DAYS, min_pairs=JUMP_MIN_PAIRS, + ) + # A minute date with no daily bar cannot have a forward return; keep the factor + # on the daily trading grid so the analytics boundary has a price for every date. + jump_raw = load.factor[ + load.factor.index.get_level_values(DATE_LEVEL).isin(panel_dates) + ] + jump_processed = _process_factors(cfg, jump_raw.to_frame(spec.factor_id), panel) + factor_series = jump_processed[spec.factor_id] + + price_panel = panel[CORE_COLUMNS] + reports = evaluate_two_runs( + factor_series, + spec, + eval_cfg, + price_panel, + book_processed, + universe_symbols=tuple(symbols), + fee_rate=float(cfg.cost.fee_rate), + report_dir=Path(cfg.output.report_dir), + ) + + no_book_metrics = extract_metrics(reports.no_book) + with_book_metrics = extract_metrics(reports.with_book) + logger.info( + "verdict no-book: %s (predictive=%s); with-book: %s (incremental=%s)", + no_book_metrics["deployment"], no_book_metrics["predictive"], + with_book_metrics["deployment"], with_book_metrics["incremental"], + ) + + return JumpEvalResult( + config=cfg, + spec=spec, + requested_symbols=load.requested, + covered_symbols=len(load.covered), + empty_symbols=len(load.empty_symbols), + factor_rows=int(len(factor_series)), + minute_raw_rows=load.raw_rows, + minute_live_calls=load.live_calls, + no_book_metrics=no_book_metrics, + with_book_metrics=with_book_metrics, + reports=reports, + log_path=log_path, + elapsed=time.monotonic() - started, + ) + + +def _build_book_factors() -> list: + """Instantiate the confirmed book (value_ep / value_bp / volatility_20).""" + from factors.compute.candidates import ValueFactor, VolatilityFactor + + return [ + ValueFactor("value_ep"), + ValueFactor("value_bp"), + VolatilityFactor(window=20), + ] + + +__all__ = [ + "JumpEvalResult", + "evaluate_two_runs", + "extract_metrics", + "run_eval_jump_amount_corr", +] diff --git a/tests/test_eval_figures.py b/tests/test_eval_figures.py new file mode 100644 index 0000000..f4785ce --- /dev/null +++ b/tests/test_eval_figures.py @@ -0,0 +1,166 @@ +"""Tests for the factor-evaluation dashboard renderer (analytics/eval/figures.py). + +Pure-rendering checks (a PNG is written, no display, no crash on degenerate data) +plus the two contract guarantees: + * ``evaluate_with_ir`` returns a report byte-identical to ``evaluate``'s; + * the MANDATORY factor-definition text carries how the factor is computed. +""" +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from analytics.eval import EvalConfig, EvalContext, StandardFactorEvaluator +from analytics.eval.figures import ( + DashboardData, + _definition_block_line, + _definition_meta_line, + render_factor_dashboard, +) +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL +from factors.spec import FactorSpec + + +# --------------------------------------------------------------------------- # +# toy builders (small, self-contained) +# --------------------------------------------------------------------------- # +def _panel(dates, symbols, rng, signal=None): + n_d, n_s = len(dates), len(symbols) + noise = rng.normal(0.0, 0.02, size=(n_d, n_s)) + returns = noise if signal is None else noise + signal + close = pd.DataFrame(np.exp(np.log(100.0) + np.cumsum(returns, axis=0)), + index=dates, columns=symbols) + stacked = close.stack().sort_index() + stacked.index.names = [DATE_LEVEL, SYMBOL_LEVEL] + return pd.DataFrame({ + "open": stacked, "high": stacked * 1.01, "low": stacked * 0.99, + "close": stacked, "volume": 1_000_000.0, "amount": 100_000_000.0, + "adj_factor": 1.0, + }) + + +def _spec(**over): + kw = dict(factor_id="synth_factor", version="1.0", + description="a synthetic factor computed as the 20d mean of X", + expected_ic_sign=1, is_intraday=False, forward_return_horizon=1, + return_basis="close_to_close", input_fields=("close", "amount"), + family="microstructure") + kw.update(over) + return FactorSpec(**kw) + + +def _cfg(**over): + kw = dict(universe="TEST500", universe_is_pit=True, start="2024-01-01", + end="2025-01-01", is_exploratory=True, post_hoc_selected=False, + rebalance="daily", n_quantiles=5) + kw.update(over) + return EvalConfig(**kw) + + +@pytest.fixture +def rng(): + return np.random.default_rng(20260718) + + +def _report_and_ir(rng, *, oos=False): + dates = pd.bdate_range("2024-01-02", periods=60, name=DATE_LEVEL) + symbols = [f"{i:06d}.SZ" for i in range(15)] + # a planted cross-sectional signal so IC / quantiles are non-degenerate + strength = np.linspace(-0.004, 0.004, len(symbols)) + panel = _panel(dates, symbols, rng, signal=strength) + factor = pd.Series( + np.repeat(strength[None, :], len(dates), axis=0).reshape(-1), + index=panel.index, name="synth_factor", + ) + rng.normal(0, 1e-4, size=len(panel)) + cfg = _cfg(oos_split="2024-02-15") if oos else _cfg() + ctx = EvalContext(price_panel=panel[["open", "high", "low", "close", + "volume", "amount", "adj_factor"]], + universe_symbols=tuple(symbols)) + return StandardFactorEvaluator().evaluate_with_ir(factor, _spec(), cfg, ctx) + + +# --------------------------------------------------------------------------- # +# evaluate_with_ir mirrors evaluate +# --------------------------------------------------------------------------- # +def test_evaluate_with_ir_report_is_identical_to_evaluate(rng): + dates = pd.bdate_range("2024-01-02", periods=50, name=DATE_LEVEL) + symbols = [f"{i:06d}.SZ" for i in range(12)] + panel = _panel(dates, symbols, rng) + factor = pd.Series(rng.normal(size=len(panel)), index=panel.index, + name="synth_factor") + ctx = EvalContext(price_panel=panel) + ev = StandardFactorEvaluator() + report_only = ev.evaluate(factor, _spec(), _cfg(), ctx) + report_wi, ir = ev.evaluate_with_ir(factor, _spec(), _cfg(), ctx) + assert report_wi.to_json() == report_only.to_json() + assert isinstance(ir.ic, pd.Series) + assert isinstance(ir.quantile_returns, pd.DataFrame) + + +# --------------------------------------------------------------------------- # +# the dashboard renders to a PNG +# --------------------------------------------------------------------------- # +def test_render_writes_a_nontrivial_png(rng, tmp_path): + report, ir = _report_and_ir(rng, oos=True) + out = render_factor_dashboard(report, ir, tmp_path / "dash.png") + assert out.exists() + assert out.stat().st_size > 20_000 # a real multi-panel figure, not an empty axes + with open(out, "rb") as fh: + assert fh.read(8) == b"\x89PNG\r\n\x1a\n" # PNG magic + + +def test_render_survives_empty_series_and_missing_payloads(tmp_path): + """Degenerate input (no IC, no quantiles, no sections) must not crash.""" + from analytics.eval.verdict import REJECT, VerdictResult + + data = DashboardData( + spec=_spec(), verdict=VerdictResult(REJECT, ("hand-built",)), + payloads={}, ic=pd.Series(dtype=float), quantile_returns=pd.DataFrame(), + ) + from analytics.eval.figures import _render + + out = _render(data, tmp_path / "empty.png") + assert out.exists() and out.stat().st_size > 5_000 + + +def test_from_report_pulls_spec_verdict_payloads_and_series(rng): + report, ir = _report_and_ir(rng) + data = DashboardData.from_report(report, ir) + assert data.spec.factor_id == "synth_factor" + assert data.verdict is report.require_verdict() + assert "predictive_power" in data.payloads + assert data.ic.equals(ir.ic) + + +# --------------------------------------------------------------------------- # +# MANDATORY factor-definition content +# --------------------------------------------------------------------------- # +def test_definition_meta_line_states_how_the_factor_is_computed(): + spec = _spec() + line = _definition_meta_line(spec) + for field in spec.input_fields: + assert field in line + assert "expected sign: +1" in line + assert f"horizon: {spec.forward_return_horizon}" in line + assert spec.return_basis in line + assert spec.family in line + assert f"min-history: {spec.min_history_bars}" in line + assert spec.price_adjust in line + + +def test_definition_block_is_none_for_a_daily_factor(): + assert _definition_block_line(_spec(is_intraday=False)) is None + + +def test_definition_block_states_the_minute_contract_for_an_intraday_factor(): + spec = _spec( + is_intraday=True, return_basis="exec_to_exec", + decision_cutoff="14:50:00", data_lag="1min", session_open="09:30:00", + execution_model="next_minute_close", execution_window="[14:51,14:56:59]", + ) + block = _definition_block_line(spec) + assert block is not None + for token in ("14:50:00", "1min", "09:30:00", "next_minute_close", + "[14:51,14:56:59]"): + assert token in block diff --git a/tests/test_eval_jump_amount_corr_runner.py b/tests/test_eval_jump_amount_corr_runner.py new file mode 100644 index 0000000..270f75c --- /dev/null +++ b/tests/test_eval_jump_amount_corr_runner.py @@ -0,0 +1,232 @@ +"""PR-C runner: cache-only minute loader + the two-run evaluation core (no network).""" + +from __future__ import annotations + +import logging + +import numpy as np +import pandas as pd +import pytest + +from analytics.eval import EvalConfig, MANDATORY_SECTIONS, Section, Skipped +from analytics.eval.verdict import AXIS_NOT_ASSESSED, AXIS_VERDICTS +from data.cache.intraday_cache import ENDPOINT as INTRADAY_ENDPOINT +from data.cache.intraday_parquet_store import KEY_COLS, IntradayParquetStore +from factors.compute.intraday_derived import JumpAmountCorrFactor +from qt.config import ( + AlphaCfg, + BacktestCfg, + CacheCfg, + CostCfg, + DataCfg, + FactorCfg, + OutputCfg, + PortfolioCfg, + RootConfig, + UniverseCfg, +) +from qt.eval_jump_amount_corr import ( + _load_jump_factor_panel, + evaluate_two_runs, + extract_metrics, +) + + +# --------------------------------------------------------------------------- # +# Minute cache-only loader +# --------------------------------------------------------------------------- # +def _stored_rows(sym, day, amps, amounts): + """Build STORED_COLUMNS-shaped 1min rows (open=100, amplitude a) for one session.""" + base = pd.Timestamp(day) + pd.Timedelta("09:31:00") + rows = [] + for i, (a, amt) in enumerate(zip(amps, amounts)): + be = base + pd.Timedelta(minutes=i) + rows.append( + { + "symbol": sym, + "bar_end": be, + "source_trade_time": be, + "open": 100.0, + "high": 100.0 + a * 50.0, + "low": 100.0 - a * 50.0, + "close": 100.0, + "volume": 1.0, + "amount": amt, + "freq": "1min", + } + ) + return pd.DataFrame(rows) + + +def _min_config(root, start, end): + return RootConfig( + data=DataCfg( + source="tushare", + start=start, + end=end, + external_secret_file="/nonexistent.json", + cache=CacheCfg(enabled=True, root_dir=str(root)), + ), + universe=UniverseCfg(type="static", symbols=["A.SZ"]), + factors=[FactorCfg(name="momentum_20")], + alpha=AlphaCfg(), + portfolio=PortfolioCfg(top_n=1), + backtest=BacktestCfg(), + cost=CostCfg(), + output=OutputCfg(), + ) + + +def test_minute_loader_is_cache_only_and_discloses_empty(tmp_path): + root = tmp_path / "cache" + store = IntradayParquetStore(str(root)) + amps = [0.001, 0.02, 0.001, 0.001, 0.02, 0.001] + amts = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0] + # Two symbols with cached minute bars across two trading days; a third symbol + # has NO cached minute at all (must be disclosed as empty, never fetched). + for sym in ("AAA.SZ", "BBB.SZ"): + for day in ("2021-07-01", "2021-07-02"): + store.upsert(INTRADAY_ENDPOINT, sym, "1min", + _stored_rows(sym, day, amps, amts), KEY_COLS) + + cfg = _min_config(root, "2021-07-01", "2021-07-02") + spec = JumpAmountCorrFactor().spec + logger = logging.getLogger("test.jac.loader") + load = _load_jump_factor_panel( + cfg, ["AAA.SZ", "BBB.SZ", "CCC.SZ"], spec, logger, + lookback_days=20, min_pairs=2, + ) + assert load.live_calls == 0 # store read has no fetch closure + assert set(load.covered) == {"AAA.SZ", "BBB.SZ"} # both produced a value + assert load.empty_symbols == ("CCC.SZ",) # uncovered disclosed, not fetched + assert load.factor.name == spec.factor_id + assert load.factor.notna().any() + # values live only on the covered symbols + assert set(load.factor.index.get_level_values("symbol")) == {"AAA.SZ", "BBB.SZ"} + + +def test_minute_loader_blocks_when_nothing_cached(tmp_path): + cfg = _min_config(tmp_path / "empty", "2021-07-01", "2021-07-02") + spec = JumpAmountCorrFactor().spec + with pytest.raises(ValueError, match="no requested symbol produced"): + _load_jump_factor_panel( + cfg, ["ZZZ.SZ"], spec, logging.getLogger("test.jac.block"), + lookback_days=20, min_pairs=2, + ) + + +# --------------------------------------------------------------------------- # +# Evaluation core (two runs) — synthetic processed panels, no network +# --------------------------------------------------------------------------- # +def _synthetic_panels(n_days=90, n_symbols=15, seed=7): + rng = np.random.default_rng(seed) + dates = pd.bdate_range("2022-01-03", periods=n_days) + symbols = [f"{i:06d}.SZ" for i in range(n_symbols)] + idx = pd.MultiIndex.from_product([dates, symbols], names=["date", "symbol"]) + + # processed (z-scored-ish) subject factor + a 3-column book on the same grid + factor = pd.Series(rng.standard_normal(len(idx)), index=idx, name="jump_amount_corr_20") + book = pd.DataFrame( + { + "value_ep": rng.standard_normal(len(idx)), + "value_bp": rng.standard_normal(len(idx)), + "volatility_20": rng.standard_normal(len(idx)), + }, + index=idx, + ) + # a qfq-style price panel with the CORE_COLUMNS the forward-return boundary needs + close = pd.Series( + 100.0 * np.exp(np.cumsum(0.001 * rng.standard_normal(len(idx)))), index=idx + ) + price = pd.DataFrame( + { + "open": close.to_numpy(), + "high": close.to_numpy() * 1.01, + "low": close.to_numpy() * 0.99, + "close": close.to_numpy(), + "volume": 1_000.0, + "amount": 100_000.0, + "adj_factor": 1.0, + }, + index=idx, + ) + return factor, book, price + + +def _eval_cfg(): + return EvalConfig( + universe="000905.SH", + universe_is_pit=True, + start="2022-01-03", + end="2022-05-10", + is_exploratory=True, + post_hoc_selected=False, + rebalance="daily", + n_quantiles=5, + oos_split="2022-03-15", + winsorize=None, + standardize="zscore", + neutralization=("industry", "size"), + industry_level="L1", + ) + + +def test_evaluate_two_runs_produces_full_reports_and_incremental_axis(tmp_path): + factor, book, price = _synthetic_panels() + spec = JumpAmountCorrFactor().spec + reports = evaluate_two_runs( + factor, spec, _eval_cfg(), price, book, + universe_symbols=tuple(sorted(set(book.index.get_level_values("symbol")))), + fee_rate=0.001, + report_dir=tmp_path, + ) + + for report in (reports.no_book, reports.with_book): + by = report.by_name() + assert set(by) == set(MANDATORY_SECTIONS) # all 8 present + assert all(isinstance(s, (Section, Skipped)) for s in by.values()) + assert report.verdict is not None # a verdict was produced + + # no-book: purity Skipped (no book) -> Incremental NOT_ASSESSED + assert isinstance(reports.no_book.by_name()["purity"], Skipped) + assert reports.no_book.verdict.incremental.verdict == AXIS_NOT_ASSESSED + + # with-book: purity is a real Section that populated the Incremental facts + purity = reports.with_book.by_name()["purity"] + assert isinstance(purity, Section) + assert purity.payload["known_factors_supplied"] is True + assert "incremental_ic_ir" in purity.payload + # the axis is now assessed (not NOT_ASSESSED) and is a valid axis state + incr = reports.with_book.verdict.incremental.verdict + assert incr in AXIS_VERDICTS and incr != AXIS_NOT_ASSESSED + + # reports were written to disk (md + json), both runs + for p in (reports.no_book_md, reports.no_book_json, + reports.with_book_md, reports.with_book_json): + assert p.exists() and p.stat().st_size > 0 + + # the research-style dashboard PNG is emitted for both runs (mandatory report + # artifact alongside md/json) + for p in (reports.no_book_dashboard, reports.with_book_dashboard): + assert p.exists() and p.stat().st_size > 20_000 + with open(p, "rb") as fh: + assert fh.read(8) == b"\x89PNG\r\n\x1a\n" + + # metrics extraction surfaces the gated fields + m = extract_metrics(reports.no_book) + assert m["deployment"] in {"Adopt", "Watch", "Reject", "INSUFFICIENT-DATA"} + assert "effective_samples" in m and "ic_ir" in m + + +def test_no_book_run_never_reaches_adopt(tmp_path): + # Structural guarantee (design §6): with no book + no execution facts, at most + # Watch — regardless of the signal (exploratory cap + NOT_ASSESSED axes). + factor, book, price = _synthetic_panels(seed=3) + reports = evaluate_two_runs( + factor, JumpAmountCorrFactor().spec, _eval_cfg(), price, book, + universe_symbols=(), + fee_rate=0.001, + report_dir=tmp_path, + ) + assert reports.no_book.verdict.verdict != "Adopt" + assert reports.no_book.verdict.tradable.verdict == AXIS_NOT_ASSESSED diff --git a/tests/test_factor_eval_contract.py b/tests/test_factor_eval_contract.py new file mode 100644 index 0000000..093b663 --- /dev/null +++ b/tests/test_factor_eval_contract.py @@ -0,0 +1,2209 @@ +"""Factor-evaluation contract tests (PR-A): the CI enforcement layer (design §7). + +Network-free. These lock the contract itself — the validators, the mandatory-spec +enforcement, the migration of every real factor, the mandatory sections, the +verdict rule table, and the determinism/secret-safety of the report. + +``StandardFactorEvaluator`` and the vectorized eval-IR are PR-B; nothing here +computes a metric. +""" + +from __future__ import annotations + +import functools +import json + +import pytest + +from analytics.eval import ( + ADOPT, + AXIS_FAIL, + AXIS_INSUFFICIENT_DATA, + AXIS_NOT_ASSESSED, + AXIS_PASS, + INSUFFICIENT_DATA, + MANDATORY_SECTIONS, + REJECT, + WATCH, + AxisVerdict, + EvalConfig, + FactorEvalReport, + FactorEvaluator, + Section, + Skipped, + VerdictInputs, + VerdictResult, + VerdictThresholds, + decide_verdict, +) +from analytics.eval.render import MAX_VALUE_CHARS +from factors.base import Factor +from factors.compute.candidates import ( + VALUE_FIELDS, + LiquidityFactor, + OvernightMomentumFactor, + ReversalFactor, + ValueFactor, + VolatilityFactor, +) +from factors.compute.financial import SUPPORTED_FIELDS, FinancialFactor +from factors.compute.momentum import MomentumFactor +from factors.spec import ( + INTRADAY_FIELDS, + INTRADAY_RETURN_BASIS, + RETURN_BASES, + FactorSpec, +) + + +def _spec(**overrides) -> FactorSpec: + """A valid daily FactorSpec; override one field per test.""" + base = dict( + factor_id="unit_test_factor", + version="1.0", + description="A test factor.", + expected_ic_sign=+1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + input_fields=("close",), + ) + base.update(overrides) + return FactorSpec(**base) + + +def _intraday_block() -> dict: + """The project's minute conventions (I5a/I5b).""" + return { + "decision_cutoff": "14:50:00", + "data_lag": "1min", + "session_open": "09:30:00", + "execution_model": "next_minute_close", + "execution_window": "[14:51,14:56:59]", + } + + +def _cfg(**overrides) -> EvalConfig: + base = dict( + universe="000016.SH", + universe_is_pit=True, + start="2023-07-01", + end="2024-06-30", + is_exploratory=True, + post_hoc_selected=False, + ) + base.update(overrides) + return EvalConfig(**base) + + +# -------------------------------------------------------------------------- +# FactorSpec validators (enforcement layer #1) +# -------------------------------------------------------------------------- + + +def test_valid_spec_is_frozen_and_normalizes_input_fields(): + spec = _spec(input_fields=["close", "amount"]) + assert spec.input_fields == ("close", "amount") # list -> tuple, hashable + with pytest.raises(Exception): + spec.factor_id = "mutated" # frozen + + +@pytest.mark.parametrize("horizon", [0, -1, -20]) +def test_non_positive_forward_horizon_rejected(horizon): + with pytest.raises(ValueError, match="forward_return_horizon"): + _spec(forward_return_horizon=horizon) + + +@pytest.mark.parametrize("sign", [None, 0, 2, -2, "+1", 1.0, True]) +def test_expected_ic_sign_must_be_plus_or_minus_one(sign): + """None is explicitly forbidden: the direction is a pre-run commitment.""" + with pytest.raises(ValueError, match="expected_ic_sign"): + _spec(expected_ic_sign=sign) + + +@pytest.mark.parametrize("sign", [+1, -1]) +def test_expected_ic_sign_accepts_both_directions(sign): + assert _spec(expected_ic_sign=sign).expected_ic_sign == sign + + +def test_intraday_true_requires_the_whole_minute_block(): + for missing in INTRADAY_FIELDS: + block = _intraday_block() + block.pop(missing) + with pytest.raises(ValueError, match="minute block"): + _spec(is_intraday=True, return_basis="exec_to_exec", **block) + + +def test_intraday_true_with_full_block_is_valid(): + spec = _spec(is_intraday=True, return_basis="exec_to_exec", **_intraday_block()) + assert spec.decision_cutoff == "14:50:00" + assert spec.execution_model == "next_minute_close" + + +@pytest.mark.parametrize("field_name", INTRADAY_FIELDS) +def test_daily_factor_may_not_set_any_intraday_field(field_name): + with pytest.raises(ValueError, match="is_intraday=False"): + _spec(is_intraday=False, **{field_name: "14:50:00"}) + + +def test_intraday_block_of_empty_strings_is_rejected(): + """Present-but-blank is the same half-declared contract as missing. + + An all-empty block passes an ``is not None`` check while declaring nothing — + exactly what the guard claims to reject. + """ + block = dict.fromkeys(INTRADAY_FIELDS, "") + with pytest.raises(ValueError, match="non-empty strings"): + _spec(is_intraday=True, return_basis="exec_to_exec", **block) + + +@pytest.mark.parametrize("blank", ["", " ", 1450]) +@pytest.mark.parametrize("field_name", INTRADAY_FIELDS) +def test_intraday_block_rejects_any_blank_or_non_string_entry(field_name, blank): + block = _intraday_block() + block[field_name] = blank + with pytest.raises(ValueError, match="non-empty strings"): + _spec(is_intraday=True, return_basis="exec_to_exec", **block) + + +def test_intraday_factor_may_not_claim_close_to_close_returns(): + """I5a: an intraday holding period is exec(T)->exec(T_next), never close-to-close.""" + with pytest.raises(ValueError, match="exec_to_exec"): + _spec(is_intraday=True, return_basis="close_to_close", **_intraday_block()) + + +def test_intraday_return_basis_constant_matches_the_supported_bases(): + assert INTRADAY_RETURN_BASIS in RETURN_BASES + + +@pytest.mark.parametrize("field_name", ["factor_id", "version", "description"]) +@pytest.mark.parametrize("bad", ["", " ", None]) +def test_identity_fields_must_be_non_empty(field_name, bad): + with pytest.raises(ValueError, match=field_name): + _spec(**{field_name: bad}) + + +@pytest.mark.parametrize("bad", [(), [], None]) +def test_input_fields_must_be_non_empty(bad): + with pytest.raises(ValueError, match="input_fields"): + _spec(input_fields=bad) + + +def test_input_fields_rejects_a_bare_string(): + """'close' must not silently become ('c','l','o','s','e').""" + with pytest.raises(ValueError, match="not a bare string"): + _spec(input_fields="close") + + +@pytest.mark.parametrize("basis", ["close_to_open", "", "exec"]) +def test_return_basis_must_be_supported(basis): + with pytest.raises(ValueError, match="return_basis"): + _spec(return_basis=basis) + + +@pytest.mark.parametrize("adjust", ["hfq", "none", "raw"]) +def test_price_adjust_only_supports_qfq_today(adjust): + with pytest.raises(ValueError, match="price_adjust"): + _spec(price_adjust=adjust) + + +def test_min_history_bars_must_be_non_negative(): + with pytest.raises(ValueError, match="min_history_bars"): + _spec(min_history_bars=-1) + + +# -------------------------------------------------------------------------- +# Factor.__init_subclass__ enforcement (spec is mandatory) +# -------------------------------------------------------------------------- + + +def test_factor_subclass_without_spec_raises_at_class_definition(): + with pytest.raises(TypeError, match="must declare a FactorSpec"): + + class NoSpecFactor(Factor): + name = "no_spec" + + def compute(self, panel): # pragma: no cover - never defined + return panel + + +def test_factor_subclass_with_classattr_spec_is_accepted(): + class ClassAttrFactor(Factor): + name = "class_attr_factor" + spec = _spec(factor_id="class_attr_factor") + + def compute(self, panel): + return panel + + assert ClassAttrFactor().spec.factor_id == "class_attr_factor" + + +def test_factor_subclass_with_property_spec_is_accepted(): + class PropertyFactor(Factor): + name = "property_factor" + + @property + def spec(self): + return _spec(factor_id="property_factor") + + def compute(self, panel): + return panel + + assert PropertyFactor().spec.factor_id == "property_factor" + + +def test_factor_subclass_with_cached_property_spec_is_accepted(): + """cached_property is an explicitly allowed deferred form (it needs an instance).""" + + class CachedPropertyFactor(Factor): + name = "cached_property_factor" + + @functools.cached_property + def spec(self): + return _spec(factor_id="cached_property_factor") + + def compute(self, panel): + return panel + + assert CachedPropertyFactor().spec.factor_id == "cached_property_factor" + + +def test_factor_subclass_with_non_spec_classattr_raises_at_class_definition(): + with pytest.raises(TypeError, match="neither a FactorSpec"): + + class StringSpecFactor(Factor): + name = "string_spec" + spec = "momentum_20" + + def compute(self, panel): # pragma: no cover - never defined + return panel + + +def test_spec_as_a_bare_method_is_rejected_at_class_definition(): + """The classic slip: @property forgotten. + + A plain function IS a descriptor, so a ``hasattr(type(x), '__get__')`` check + waves it through and the contract only fails much later, at instantiation, + with a generic message. The message must name the actual mistake. + """ + with pytest.raises(TypeError, match="did you forget @property"): + + class BareMethodFactor(Factor): + name = "bare_method" + + def spec(self): # pragma: no cover - never defined + return _spec(factor_id="bare_method") + + def compute(self, panel): # pragma: no cover - never defined + return panel + + +def test_spec_as_a_staticmethod_is_rejected_at_class_definition(): + with pytest.raises(TypeError, match="did you forget @property"): + + class StaticMethodFactor(Factor): + name = "static_method" + + @staticmethod + def spec(): # pragma: no cover - never defined + return _spec(factor_id="static_method") + + def compute(self, panel): # pragma: no cover - never defined + return panel + + +def test_spec_as_a_classmethod_is_rejected_at_class_definition(): + with pytest.raises(TypeError, match="did you forget @property"): + + class ClassMethodFactor(Factor): + name = "class_method" + + @classmethod + def spec(cls): # pragma: no cover - never defined + return _spec(factor_id="class_method") + + def compute(self, panel): # pragma: no cover - never defined + return panel + + +# -------------------------------------------------------------------------- +# The spec requirement fires on CONCRETE classes only (the factor layer will be +# refactored; a shared family base must not have to invent a bogus spec). +# -------------------------------------------------------------------------- + + +def test_abstract_intermediate_base_needs_no_spec(): + """An intermediate that leaves ``compute`` abstract is not a factor yet.""" + + class AbstractFamilyBase(Factor): + """A shared base for a factor family: no compute, hence still abstract.""" + + def _shared_helper(self) -> int: + return 1 + + assert AbstractFamilyBase.__abstractmethods__ == frozenset({"compute"}) + with pytest.raises(TypeError, match="abstract"): + AbstractFamilyBase() # still not instantiable — it is not a factor + + +def test_concrete_subclass_of_an_abstract_intermediate_still_needs_a_spec(): + class AbstractFamilyBase(Factor): + def _shared_helper(self) -> int: + return 1 + + with pytest.raises(TypeError, match="must declare a FactorSpec"): + + class ConcreteChild(AbstractFamilyBase): + name = "concrete_child" + + def compute(self, panel): # pragma: no cover - never defined + return panel + + +def test_concrete_subclass_of_an_abstract_intermediate_works_with_a_spec(): + class AbstractFamilyBase(Factor): + def _shared_helper(self) -> int: + return 1 + + class ConcreteChild(AbstractFamilyBase): + name = "concrete_child" + spec = _spec(factor_id="concrete_child") + + def compute(self, panel): + return panel + + assert ConcreteChild().spec.factor_id == "concrete_child" + assert ConcreteChild()._shared_helper() == 1 + + +def test_instance_spec_is_validated_at_construction(): + """A property returning a non-FactorSpec cannot even be instantiated.""" + + class LyingFactor(Factor): + name = "lying" + + @property + def spec(self): + return {"factor_id": "lying"} # a dict is not a FactorSpec + + def compute(self, panel): # pragma: no cover - never reached + return panel + + with pytest.raises(TypeError, match="must be a FactorSpec instance"): + LyingFactor() + + +# -------------------------------------------------------------------------- +# Migration lock: every real factor declares a valid spec +# -------------------------------------------------------------------------- + +REAL_FACTORS = [ + MomentumFactor(), + MomentumFactor(window=60), + ReversalFactor(), + ReversalFactor(window=5), + VolatilityFactor(), + LiquidityFactor(), + OvernightMomentumFactor(), + *[FinancialFactor(f) for f in SUPPORTED_FIELDS], + *[ValueFactor(f) for f in VALUE_FIELDS], +] + + +@pytest.mark.parametrize("factor", REAL_FACTORS, ids=lambda f: f.name) +def test_every_real_factor_exposes_a_valid_spec(factor): + spec = factor.spec + assert isinstance(spec, FactorSpec) + # factor_id IS the panel column: a mismatch would misjoin the whole report. + assert spec.factor_id == factor.name + assert spec.expected_ic_sign in (+1, -1) + assert spec.forward_return_horizon > 0 + assert spec.input_fields + assert not spec.is_intraday # every factor today is daily + assert all(getattr(spec, f) is None for f in INTRADAY_FIELDS) + + +def _all_subclasses(cls) -> set: + found = set(cls.__subclasses__()) + for sub in list(found): + found |= _all_subclasses(sub) + return found + + +def test_every_shipped_factor_subclass_declares_a_spec(): + """Drift guard: a NEW factor class cannot be merged without a spec. + + Scoped to classes defined under ``factors.`` on purpose — this module and + other tests define deliberately-broken subclasses, and a class whose + ``__init_subclass__`` raised still lingers in ``__subclasses__()``. + """ + shipped = { + cls for cls in _all_subclasses(Factor) if cls.__module__.startswith("factors.") + } + assert len(shipped) >= 7 # never pass vacuously + for cls in shipped: + assert "spec" in cls.__dict__, f"{cls.__name__} declares no spec" + + +@pytest.mark.parametrize("window", [5, 20, 60]) +def test_parameterized_factor_id_tracks_the_window(window): + """A property (not a classattr) is why momentum_60 is not labelled momentum_20.""" + factor = MomentumFactor(window=window) + assert factor.spec.factor_id == f"momentum_{window}" + assert factor.spec.min_history_bars == window + assert ReversalFactor(window=window).spec.factor_id == f"reversal_{window}" + + +def test_reversal_sign_is_the_exact_negation_of_momentum(): + """Reversal = -momentum by construction, so the hypotheses cannot drift.""" + assert ReversalFactor(window=5).spec.expected_ic_sign == -( + MomentumFactor(window=5).spec.expected_ic_sign + ) + + +def test_independently_confirmed_signs_match_the_project_evidence(): + """P3-5/P3-7/P3-8: value_ep/value_bp +1, volatility_20 -1 (low-vol).""" + assert ValueFactor("value_ep").spec.expected_ic_sign == +1 + assert ValueFactor("value_bp").spec.expected_ic_sign == +1 + assert VolatilityFactor().spec.expected_ic_sign == -1 + + +# -------------------------------------------------------------------------- +# EvalConfig validators +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", [1, 0, -5]) +def test_n_quantiles_below_two_rejected(n): + with pytest.raises(ValueError, match="n_quantiles"): + _cfg(n_quantiles=n) + + +@pytest.mark.parametrize("scenarios", [(2.0, 4.0), (0.5,), ()]) +def test_cost_scenarios_must_contain_the_base_anchor(scenarios): + with pytest.raises(ValueError, match="base anchor 1.0"): + _cfg(cost_scenarios=scenarios) + + +def test_cost_scenarios_with_base_anchor_accepted(): + assert _cfg(cost_scenarios=[1.0, 2.0, 4.0]).cost_scenarios == (1.0, 2.0, 4.0) + + +def test_post_hoc_selection_requires_exploratory(): + with pytest.raises(ValueError, match="post_hoc_selected=True requires"): + _cfg(is_exploratory=False, post_hoc_selected=True) + + +def test_post_hoc_with_exploratory_accepted(): + assert _cfg(is_exploratory=True, post_hoc_selected=True).post_hoc_selected + + +def test_long_short_defaults_to_top_and_bottom_bucket(): + assert _cfg(n_quantiles=5).long_short == (5, 1) + assert _cfg(n_quantiles=10).long_short == (10, 1) + + +def test_long_short_must_stay_in_natural_orientation(): + """Flipping the legs here would double-flip a -1 factor in the verdict.""" + with pytest.raises(ValueError, match="top > bottom"): + _cfg(n_quantiles=5, long_short=(1, 5)) + + +@pytest.mark.parametrize("pair", [(6, 1), (5, 0), (5, 5)]) +def test_long_short_out_of_range_rejected(pair): + with pytest.raises(ValueError, match="long_short"): + _cfg(n_quantiles=5, long_short=pair) + + +@pytest.mark.parametrize("field_name", ["universe", "start", "end"]) +def test_universe_and_window_must_be_non_empty(field_name): + with pytest.raises(ValueError, match=field_name): + _cfg(**{field_name: ""}) + + +@pytest.mark.parametrize("rate", [0.0, -0.1, 1.5]) +def test_max_participation_rate_must_be_a_fraction(rate): + with pytest.raises(ValueError, match="max_participation_rate"): + _cfg(max_participation_rate=rate) + + +@pytest.mark.parametrize("notional", [0, -1, -1e6, "10000000", []]) +def test_capacity_notional_must_be_none_or_positive(notional): + with pytest.raises(ValueError, match="capacity_notional"): + _cfg(capacity_notional=notional) + + +def test_capacity_notional_accepts_none_or_a_positive_number(): + assert _cfg().capacity_notional is None # None = capacity not assessed + assert _cfg(capacity_notional=10_000_000).capacity_notional == 10_000_000 + + +# -------------------------------------------------------------------------- +# bool is an int subclass: True must never sneak into a numeric field (the same +# gotcha as expected_ic_sign=1.0). FactorSpec already guards every numeric +# validator; these lock the EvalConfig ones that did not. +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("value", [True, False]) +def test_capacity_notional_rejects_bool(value): + with pytest.raises(ValueError, match="capacity_notional"): + _cfg(capacity_notional=value) + + +@pytest.mark.parametrize("value", [True, False]) +def test_max_participation_rate_rejects_bool(value): + with pytest.raises(ValueError, match="max_participation_rate"): + _cfg(max_participation_rate=value) + + +@pytest.mark.parametrize("value", [True, False]) +def test_cost_scenarios_reject_bool_entries(value): + """float(True) is 1.0 — a stray True would masquerade as the base anchor.""" + with pytest.raises(ValueError, match="cost_scenarios"): + _cfg(cost_scenarios=(value, 2.0, 4.0)) + + +@pytest.mark.parametrize("value", [True, False]) +def test_n_quantiles_rejects_bool(value): + with pytest.raises(ValueError, match="n_quantiles"): + _cfg(n_quantiles=value) + + +@pytest.mark.parametrize("value", [True, False]) +def test_n_factors_screened_rejects_bool(value): + with pytest.raises(ValueError, match="n_factors_screened"): + _cfg(n_factors_screened=value) + + +@pytest.mark.parametrize("value", [True, False]) +def test_forward_return_horizon_rejects_bool(value): + with pytest.raises(ValueError, match="forward_return_horizon"): + _spec(forward_return_horizon=value) + + +@pytest.mark.parametrize("value", [True, False]) +def test_min_history_bars_rejects_bool(value): + with pytest.raises(ValueError, match="min_history_bars"): + _spec(min_history_bars=value) + + +@pytest.mark.parametrize("value", [True, False]) +def test_long_short_rejects_bool_entries(value): + with pytest.raises(ValueError, match="long_short"): + _cfg(n_quantiles=5, long_short=(value, 1)) + + +@pytest.mark.parametrize("n", [0, -1, 1.5, True, "3"]) +def test_n_factors_screened_must_be_none_or_a_positive_int(n): + with pytest.raises(ValueError, match="n_factors_screened"): + _cfg(n_factors_screened=n) + + +def test_n_factors_screened_accepts_none_or_a_positive_int(): + assert _cfg().n_factors_screened is None + assert _cfg(n_factors_screened=11).n_factors_screened == 11 + + +def test_declared_sequences_are_coerced_to_tuples_and_stay_immutable(): + """frozen=True must actually MEAN immutable + hashable. + + A caller handing in a list otherwise keeps a live reference into the config + (mutate the list -> mutate the "frozen" provenance record) and hash() raises. + """ + cells = [("SSE50", "2024-2026")] + neutralization = ["industry", "size"] + cfg = _cfg(independent_cells=cells, neutralization=neutralization) + assert cfg.independent_cells == (("SSE50", "2024-2026"),) + assert cfg.neutralization == ("industry", "size") + assert isinstance(cfg.independent_cells, tuple) + assert isinstance(cfg.neutralization, tuple) + hash(cfg) # would raise on a list field + cells.append(("CSI300", "2024-2026")) + neutralization.append("beta") + assert cfg.independent_cells == (("SSE50", "2024-2026"),) + assert cfg.neutralization == ("industry", "size") + + +def test_declared_sequence_defaults_are_already_tuples(): + cfg = _cfg() + assert cfg.independent_cells == () + assert cfg.neutralization == ("industry", "size") + hash(cfg) + + +@pytest.mark.parametrize("field_name", ["independent_cells", "neutralization"]) +def test_declared_sequences_reject_a_bare_string(field_name): + """'industry' must not silently become ('i','n','d','u','s','t','r','y').""" + with pytest.raises(ValueError, match=field_name): + _cfg(**{field_name: "industry"}) + + +@pytest.mark.parametrize("field_name", ["independent_cells", "neutralization"]) +def test_declared_sequences_reject_a_non_sequence(field_name): + with pytest.raises(ValueError, match=field_name): + _cfg(**{field_name: 3}) + + +# -------------------------------------------------------------------------- +# Section / Skipped / report assembly (enforcement layer #2) +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("reason", ["", " "]) +def test_skipped_without_a_reason_rejected(reason): + with pytest.raises(ValueError, match="reason"): + Skipped("purity", reason) + + +def test_skipped_with_reason_accepted(): + assert Skipped("purity", "no known-factor anchor set configured").reason + + +#: A data_coverage payload that clears all three parts of the §6 v0.3 sample +#: gate (raw floor / effective samples / calendar span), so a test about +#: something else is not silently gated into INSUFFICIENT-DATA. +_COVERAGE_OK: dict = { + "settled_rebalances": 36, + "effective_samples": 36.0, + "span_days": 400.0, +} + + +def _full_sections(**payloads) -> list: + return [Section(name, payloads.get(name, {})) for name in MANDATORY_SECTIONS] + + +def test_missing_mandatory_section_raises(): + sections = _full_sections()[:-1] # drop 'caveats' + report = FactorEvalReport.assemble(_spec(), _cfg(), sections) + with pytest.raises(ValueError, match="caveats"): + report.validate_all_mandatory_present() + + +def test_skipped_with_reason_counts_as_present(): + sections = _full_sections()[:-1] + [Skipped("caveats", "nothing to disclose")] + report = FactorEvalReport.assemble(_spec(), _cfg(), sections) + report.validate_all_mandatory_present() # must not raise + + +def test_duplicate_sections_rejected(): + sections = _full_sections() + [Section("purity", {})] + with pytest.raises(ValueError, match="duplicate"): + FactorEvalReport.assemble(_spec(), _cfg(), sections) + + +def test_non_section_object_rejected(): + with pytest.raises(TypeError, match="Section or an explicit"): + FactorEvalReport.assemble(_spec(), _cfg(), [*_full_sections(), None]) + + +def test_extra_custom_section_is_allowed(): + sections = [*_full_sections(), Section("crowding", {"note": 1})] + report = FactorEvalReport.assemble(_spec(), _cfg(), sections) + report.validate_all_mandatory_present() + assert "crowding" in report.by_name() + + +# -------------------------------------------------------------------------- +# Verdict rules: three axes + a derived deployment label (design §6, v0.5) +# -------------------------------------------------------------------------- + +#: A strong signal whose ICIR point clears AND whose N_eff-based 95% CI LOWER +#: bound (design §6, v0.6) also clears min_abs_icir=0.30 — a PASS-grade estimate. +_STRONG = dict( + ic_ir=0.8, + ic_ir_ci_low=0.55, + ic_ir_ci_high=1.05, + ic_win_rate=0.7, + ic_nw_t=3.5, + monotonicity_spearman=0.9, + net_long_short_by_cost=((1.0, 0.05), (2.0, 0.04), (4.0, 0.02)), +) +_TRADABLE = dict(tradable=True, capacity_sufficient=True) +_OOS_OK = dict(oos_available=True, oos_sign_consistent=True) +#: a known-factor book the factor is genuinely INCREMENTAL to (orthogonalized +#: ICIR point AND lower CI bound both clear the bar in the expected direction). +_INCREMENTAL_OK = dict( + known_factors_supplied=True, + incremental_ic_ir=0.6, + incremental_ic_mean=0.04, + incremental_ic_ir_ci_low=0.42, + incremental_ic_ir_ci_high=0.78, +) +#: everything all three axes need to PASS — the ONLY route to Adopt (v0.5). +_ADOPT_FACTS = {**_STRONG, **_TRADABLE, **_OOS_OK, **_INCREMENTAL_OK} + + +#: A sample that CLEARS all three gate parts, so a rule test exercises the RULE +#: it is about instead of tripping over the gate. +_SAMPLE_OK = dict(settled_rebalances=36, effective_samples=36.0, span_days=400.0) + +#: The project's signature failure SHAPE: plenty of raw periods over a comfortable +#: calendar span, but a regime-flipping IC series is a STEP FUNCTION whose rho-hat +#: stays positive for ~100 lags, so N_eff collapses to ~3 out of ~300. +_THIN_SAMPLE = dict(settled_rebalances=300, effective_samples=3.0, span_days=420.0) + + +def _inputs(**overrides) -> VerdictInputs: + base = dict(expected_ic_sign=+1, **_SAMPLE_OK) + base.update(overrides) + return VerdictInputs(**base) + + +# -- the new data model ---------------------------------------------------- + + +def test_axis_verdict_rejects_an_unknown_state(): + with pytest.raises(ValueError, match="axis verdict"): + AxisVerdict("MAYBE") + + +def test_verdict_result_exposes_the_three_axes_and_the_derived_label(): + r = decide_verdict(_inputs(**_ADOPT_FACTS, is_exploratory=False)) + assert set(r.axes()) == {"predictive", "incremental", "tradable"} + assert all(isinstance(a, AxisVerdict) for a in r.axes().values()) + assert r.verdict in (ADOPT, WATCH, REJECT, INSUFFICIENT_DATA) + + +def test_verdict_result_defaults_axes_to_not_assessed_when_hand_built(): + """A placeholder verdict (e.g. on an incomplete report) stays constructible.""" + hand = VerdictResult(REJECT, ("hand-built",)) + assert all(a.verdict == AXIS_NOT_ASSESSED for a in hand.axes().values()) + + +# -- Axis A: Predictive ---------------------------------------------------- + + +def test_predictive_axis_passes_with_oos_and_metrics(): + r = decide_verdict(_inputs(**_STRONG, **_OOS_OK)) + assert r.predictive.verdict == AXIS_PASS + assert any("both out-of-sample subperiods" in x for x in r.predictive.reasons) + + +def test_predictive_axis_insufficient_on_a_thin_sample(): + r = decide_verdict(_inputs(effective_samples=5.0, **_STRONG, **_OOS_OK)) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any("effective samples (A)" in x for x in r.predictive.reasons) + + +def test_predictive_axis_insufficient_without_oos(): + """No holdout -> no predictive claim; in-sample metrics are reported, not a PASS.""" + r = decide_verdict(_inputs(**_STRONG, oos_available=False)) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any("no out-of-sample split" in x for x in r.predictive.reasons) + + +def test_predictive_axis_fails_on_a_known_oos_sign_flip(): + r = decide_verdict( + _inputs(**_STRONG, oos_available=True, oos_sign_flipped=True) + ) + assert r.predictive.verdict == AXIS_FAIL + assert any("sign flipped" in x for x in r.predictive.reasons) + + +def test_predictive_axis_fails_on_an_independent_cell_reversal(): + r = decide_verdict( + _inputs(**_STRONG, oos_available=True, oos_monotonicity_reversed=True) + ) + assert r.predictive.verdict == AXIS_FAIL + assert any("independent cell" in x for x in r.predictive.reasons) + + +def test_predictive_axis_fails_when_the_signal_is_absent_despite_oos(): + """Sufficient data + OOS available, but nothing convincing -> a NEGATIVE finding.""" + r = decide_verdict( + _inputs( + oos_available=True, + oos_sign_consistent=False, + ic_ir=0.05, + ic_win_rate=0.5, + ic_nw_t=0.3, + monotonicity_spearman=0.1, + ) + ) + assert r.predictive.verdict == AXIS_FAIL + + +def test_predictive_axis_passes_on_weak_but_aligned_monotonicity(): + """v0.7 direction gate: monotonicity is a DIRECTION check, not a strength bar. + + A tail-concentrated real factor (e.g. jump-amount-corr: strong ICIR/NW-t, yet a + hump-shaped quantile profile whose aligned Spearman is only ~0.3) used to FAIL + the old 0.80 STRENGTH bar despite a genuine out-of-sample signal. Under the + default 0.0 direction gate a strictly-positive aligned monotonicity clears, so + the axis PASSes on its ICIR/NW-t/win-rate/OOS evidence. + """ + r = decide_verdict( + _inputs(**{**_STRONG, "monotonicity_spearman": 0.3}, **_OOS_OK) + ) + assert r.predictive.verdict == AXIS_PASS + + +def test_predictive_axis_fails_on_reversed_monotonicity_despite_strong_ic(): + """v0.7 direction gate still bites the WRONG way: aligned monotonicity <= 0. + + Strong ICIR/NW-t/win-rate/OOS, but the quantile buckets are ordered AGAINST the + hypothesis (aligned Spearman -0.4). The direction gate is not cleared, so the + point signal is absent and the axis is a NEGATIVE finding, not a PASS. + """ + r = decide_verdict( + _inputs(**{**_STRONG, "monotonicity_spearman": -0.4}, **_OOS_OK) + ) + assert r.predictive.verdict == AXIS_FAIL + assert any("monotonicity" in x for x in r.predictive.reasons) + + +# -- Axis B: Incremental --------------------------------------------------- + + +def test_incremental_axis_not_assessed_without_a_book(): + """The DEFAULT: no known_factors supplied -> the axis cannot be assessed.""" + r = decide_verdict(_inputs(**_STRONG, **_OOS_OK)) + assert r.incremental.verdict == AXIS_NOT_ASSESSED + assert any("no known-factor book" in x for x in r.incremental.reasons) + + +def test_incremental_axis_passes_on_a_genuinely_additive_factor(): + r = decide_verdict(_inputs(**_STRONG, **_OOS_OK, **_INCREMENTAL_OK)) + assert r.incremental.verdict == AXIS_PASS + assert any("adds a signal" in x for x in r.incremental.reasons) + + +def test_incremental_axis_fails_on_a_redundant_factor(): + """Book supplied, sufficient sample, orthogonalized IC ~ 0 -> redundant.""" + r = decide_verdict( + _inputs( + **_STRONG, + **_OOS_OK, + known_factors_supplied=True, + incremental_ic_ir=0.02, + incremental_ic_mean=0.001, + ) + ) + assert r.incremental.verdict == AXIS_FAIL + assert any("redundant" in x for x in r.incremental.reasons) + + +def test_incremental_axis_uses_its_own_lower_bar_not_the_raw_predictive_bar(): + """v0.7: the Incremental axis gates on min_incremental_abs_icir (0.15), a bar + SEPARATE from — and lower than — the raw predictive min_abs_icir (0.30). + + Orthogonalization removes variance, so a genuinely-additive residual lives on a + smaller scale. An orthogonalized ICIR of 0.20 (point AND lower CI bound above + 0.15) is a PASS under the incremental bar, though it would have missed the raw + 0.30 bar the axis used before v0.7. + """ + r = decide_verdict( + _inputs( + **_STRONG, + **_OOS_OK, + known_factors_supplied=True, + incremental_ic_ir=0.20, + incremental_ic_mean=0.012, + incremental_ic_ir_ci_low=0.17, + incremental_ic_ir_ci_high=0.23, + ) + ) + assert r.incremental.verdict == AXIS_PASS + + +def test_predictive_bar_is_unchanged_by_the_separate_incremental_bar(): + """The v0.7 split must NOT lower the raw predictive bar: an ICIR of 0.20 clears + the incremental 0.15 bar but still fails the predictive 0.30 point bar. + """ + r = decide_verdict( + _inputs( + **_OOS_OK, + ic_ir=0.20, + ic_ir_ci_low=0.17, + ic_ir_ci_high=0.23, + ic_win_rate=0.7, + ic_nw_t=3.5, + monotonicity_spearman=0.9, + ) + ) + assert r.predictive.verdict == AXIS_FAIL + + +def test_incremental_axis_fails_on_the_wrong_direction(): + r = decide_verdict( + _inputs( + **_STRONG, + **_OOS_OK, + known_factors_supplied=True, + incremental_ic_ir=-0.6, + incremental_ic_mean=-0.04, + ) + ) + assert r.incremental.verdict == AXIS_FAIL + assert any("anti-incremental" in x for x in r.incremental.reasons) + + +def test_incremental_axis_insufficient_when_a_book_is_supplied_but_thin(): + r = decide_verdict( + _inputs(effective_samples=5.0, **_STRONG, **_OOS_OK, **_INCREMENTAL_OK) + ) + assert r.incremental.verdict == AXIS_INSUFFICIENT_DATA + + +def test_incremental_axis_insufficient_when_the_orthogonalized_ic_is_unknown(): + """An unmeasurable orthogonalized IC is UNKNOWN, never a FAIL (unknown never + convicts) — distinct from a measured ~ 0, which IS a redundant FAIL.""" + r = decide_verdict( + _inputs( + **_STRONG, + **_OOS_OK, + known_factors_supplied=True, + incremental_ic_ir=float("nan"), + ) + ) + assert r.incremental.verdict == AXIS_INSUFFICIENT_DATA + assert any("UNKNOWN" in x for x in r.incremental.reasons) + + +# -- Axis C: Tradable ------------------------------------------------------ + + +def test_tradable_axis_not_assessed_by_default(): + """Execution facts (I5b/I5f) are measured elsewhere, so the default is unset.""" + r = decide_verdict(_inputs(**_STRONG, **_OOS_OK)) + assert r.tradable.verdict == AXIS_NOT_ASSESSED + assert any("not assessed" in x for x in r.tradable.reasons) + + +def test_tradable_axis_passes_with_facts_and_a_positive_base_spread(): + r = decide_verdict(_inputs(**_STRONG, **_OOS_OK, **_TRADABLE)) + assert r.tradable.verdict == AXIS_PASS + + +def test_tradable_axis_fails_when_not_tradable(): + r = decide_verdict( + _inputs(**_STRONG, **_OOS_OK, tradable=False, capacity_sufficient=True) + ) + assert r.tradable.verdict == AXIS_FAIL + assert any("not tradable" in x for x in r.tradable.reasons) + + +def test_tradable_axis_fails_on_insufficient_capacity(): + r = decide_verdict( + _inputs(**_STRONG, **_OOS_OK, tradable=True, capacity_sufficient=False) + ) + assert r.tradable.verdict == AXIS_FAIL + assert any("capacity insufficient" in x for x in r.tradable.reasons) + + +def test_tradable_axis_fails_when_every_cost_scenario_is_negative(): + kwargs = {**_STRONG, **_OOS_OK, **_TRADABLE} + kwargs["net_long_short_by_cost"] = ((1.0, -0.01), (2.0, -0.03), (4.0, -0.08)) + r = decide_verdict(_inputs(**kwargs)) + assert r.tradable.verdict == AXIS_FAIL + assert any("EVERY cost scenario" in x for x in r.tradable.reasons) + + +def test_tradable_axis_insufficient_on_partial_facts(): + """tradable known but capacity not: established neither way — not PASS, not FAIL.""" + r = decide_verdict( + _inputs(**_STRONG, **_OOS_OK, tradable=True, capacity_sufficient=None) + ) + assert r.tradable.verdict == AXIS_INSUFFICIENT_DATA + + +# -- the deployment label derivation (design §6 table) --------------------- + + +def test_deployment_reject_on_any_axis_fail(): + r = decide_verdict(_inputs(**{**_ADOPT_FACTS, "tradable": False})) + assert r.verdict == REJECT + assert r.tradable.verdict == AXIS_FAIL + assert any("tradable axis" in x for x in r.reasons) + + +def test_deployment_adopt_when_all_three_pass(): + r = decide_verdict(_inputs(**_ADOPT_FACTS, is_exploratory=False)) + assert r.verdict == ADOPT + assert r.predictive.verdict == AXIS_PASS + assert r.incremental.verdict == AXIS_PASS + assert r.tradable.verdict == AXIS_PASS + + +def test_deployment_watch_when_one_pass_and_the_rest_unresolved(): + """>=1 PASS with the rest INSUFFICIENT/NOT_ASSESSED -> WATCH, naming the gaps.""" + r = decide_verdict(_inputs(**_STRONG, **_OOS_OK)) # only predictive can PASS + assert r.verdict == WATCH + assert r.predictive.verdict == AXIS_PASS + assert r.incremental.verdict == AXIS_NOT_ASSESSED + assert r.tradable.verdict == AXIS_NOT_ASSESSED + assert any("unresolved" in x for x in r.reasons) + + +def test_deployment_insufficient_when_no_pass_and_no_fail(): + """Nothing demonstrated and nothing refuted -> INSUFFICIENT-DATA (NOT the old + fallback Reject: with no OOS, no book and no execution facts we cannot tell).""" + r = decide_verdict(_inputs()) # sufficient sample, but no OOS/book/exec facts + assert r.verdict == INSUFFICIENT_DATA + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert r.incremental.verdict == AXIS_NOT_ASSESSED + assert r.tradable.verdict == AXIS_NOT_ASSESSED + + +def test_default_run_tops_out_at_watch(): + """⚠️ INTENDED: with no known_factors and no execution facts the Incremental + and Tradable axes are NOT_ASSESSED, so a default run can never have all three + PASS — it maxes at WATCH, even with a flawless out-of-sample predictive signal. + This is the multi-factor point: a factor judged in isolation has not earned an + Adopt.""" + r = decide_verdict(_inputs(**_STRONG, **_OOS_OK, is_exploratory=False)) + assert r.verdict == WATCH + assert r.incremental.verdict == AXIS_NOT_ASSESSED + assert r.tradable.verdict == AXIS_NOT_ASSESSED + + +def test_full_facts_make_adopt_reachable(): + """The companion to the ceiling: a clean book + execution facts + OOS -> Adopt.""" + assert decide_verdict(_inputs(**_ADOPT_FACTS, is_exploratory=False)).verdict == ADOPT + + +# -- unknown never convicts, under the new axis structure ------------------ + + +@pytest.mark.parametrize( + "unknown", + [ + dict(oos_sign_flipped=False, oos_monotonicity_reversed=False), # no reversal + dict(tradable=None, capacity_sufficient=None), # tradability unknown + dict(known_factors_supplied=True, incremental_ic_ir=float("nan")), # ortho unknown + dict(net_long_short_by_cost=()), # no cost measured + dict(net_long_short_by_cost=((1.0, float("nan")), (2.0, float("nan")))), + ], +) +def test_no_unknown_ever_produces_a_fail_on_any_axis(unknown): + """⚠️ THE safety property: with the FAILs decided first (bypassing the gate), an + unknown that COULD convict would convict immediately and at any sample size. It + must not — every axis FAIL tests for an explicitly KNOWN failure, so each + unknown below yields INSUFFICIENT_DATA / NOT_ASSESSED, never a FAIL.""" + r = decide_verdict( + _inputs(**{**_THIN_SAMPLE, **_STRONG, **_TRADABLE, **_OOS_OK, **unknown}) + ) + for name, axis in r.axes().items(): + assert axis.verdict != AXIS_FAIL, name + assert r.verdict != REJECT + + +# -- the asymmetric gate: a FAIL bypasses the sample gate ------------------ + + +def test_a_regime_flip_on_a_thin_sample_rejects_rather_than_gating(): + """THE headline: a regime-flipping factor is the project's signature failure + mode (I5e / P3-3 / P3-4). Its N_eff ~ 3 is CORRECT — two regimes really are + about two observations — but the MEASURED flip is a NEGATIVE finding, so the + Predictive axis FAILs it BEFORE the sample gate and the label is REJECT.""" + r = decide_verdict( + _inputs( + **_THIN_SAMPLE, + **_STRONG, + **_TRADABLE, + oos_available=True, + oos_sign_flipped=True, + ) + ) + assert r.verdict == REJECT + assert r.predictive.verdict == AXIS_FAIL + assert any("sign flipped" in x for x in r.predictive.reasons) + # ... and it rejects ON THE FLIP, not with a mumbled word about the sample. + assert not any("effective samples" in x for x in r.predictive.reasons) + + +def test_not_tradable_on_a_thin_sample_still_rejects(): + """"It cannot be executed" is not a statistical claim: it needs no sample.""" + r = decide_verdict( + _inputs( + **_THIN_SAMPLE, + **_STRONG, + **_OOS_OK, + tradable=False, + capacity_sufficient=True, + ) + ) + assert r.verdict == REJECT + assert r.tradable.verdict == AXIS_FAIL + + +def test_all_cost_negative_on_a_thin_sample_still_rejects(): + kwargs = {**_STRONG, **_OOS_OK, **_TRADABLE} + kwargs["net_long_short_by_cost"] = ((1.0, -0.01), (2.0, -0.03), (4.0, -0.08)) + r = decide_verdict(_inputs(**_THIN_SAMPLE, **kwargs)) + assert r.verdict == REJECT + assert any("EVERY cost scenario" in x for x in r.tradable.reasons) + + +def test_a_measured_failure_rejects_even_when_the_sample_facts_are_unknown(): + """A Predictive FAIL reads none of the gate's facts, so an unmeasured N_eff / + span cannot rescue a factor that visibly flipped.""" + r = decide_verdict( + _inputs( + settled_rebalances=2, + effective_samples=float("nan"), + span_days=float("nan"), + **_STRONG, + **_TRADABLE, + oos_available=True, + oos_sign_flipped=True, + ) + ) + assert r.verdict == REJECT + + +def test_a_thin_sample_with_nothing_measured_reads_cannot_tell_not_bad(): + """The subtle half of the asymmetry: no measured failure on a thin sample must + read INSUFFICIENT-DATA ("cannot tell"), never a manufactured Reject.""" + assert decide_verdict(_inputs(**_THIN_SAMPLE)).verdict == INSUFFICIENT_DATA + + +def test_a_thin_sample_still_blocks_both_positive_claims(): + """The gate did not get weaker, it got NARROWER: a thin sample buys neither a + Predictive nor an Incremental PASS. Only a MEASURED failure is let past.""" + predictive = _inputs(**_THIN_SAMPLE, **_STRONG, **_OOS_OK) + assert predictive.effective_samples < 24 # sanity + assert decide_verdict(predictive).predictive.verdict == AXIS_INSUFFICIENT_DATA + incremental = _inputs(**_THIN_SAMPLE, **_STRONG, **_OOS_OK, **_INCREMENTAL_OK) + assert decide_verdict(incremental).incremental.verdict == AXIS_INSUFFICIENT_DATA + + +# -- the three-part sample gate governs the statistical axes (design §6) ---- + + +def test_each_gate_part_fires_independently_on_the_predictive_axis(): + good = {**_STRONG, **_OOS_OK} + only_floor = VerdictThresholds( + min_rebalances=30, min_effective_samples=1.0, min_span_days=0 + ) + r = decide_verdict( + _inputs(settled_rebalances=20, effective_samples=20.0, **good), only_floor + ) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any(x.startswith("raw floor") for x in r.predictive.reasons) + assert not any( + "effective samples" in x or "calendar span" in x for x in r.predictive.reasons + ) + + r = decide_verdict(_inputs(effective_samples=5.0, **good)) + assert any(x.startswith("effective samples (A)") for x in r.predictive.reasons) + assert not any( + x.startswith("raw floor") or "calendar span" in x for x in r.predictive.reasons + ) + + r = decide_verdict(_inputs(span_days=30.0, **good)) + assert any(x.startswith("calendar span (B)") for x in r.predictive.reasons) + assert not any( + x.startswith("raw floor") or "effective samples" in x + for x in r.predictive.reasons + ) + + +def test_the_gate_names_every_part_that_failed_not_just_the_first(): + r = decide_verdict( + _inputs( + settled_rebalances=4, effective_samples=2.0, span_days=10.0, + **_STRONG, **_OOS_OK, + ) + ) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + reasons = r.predictive.reasons + assert any(x.startswith("raw floor") for x in reasons) + assert any(x.startswith("effective samples (A)") for x in reasons) + assert any(x.startswith("calendar span (B)") for x in reasons) + joined = " ".join(reasons) + for actual, required in (("4", "12"), ("2.00", "24.0"), ("10", "365")): + assert actual in joined and required in joined + + +def test_a_dense_but_short_window_is_gated_on_span_despite_many_periods(): + r = decide_verdict( + _inputs( + settled_rebalances=500, effective_samples=480.0, span_days=35.0, + **_STRONG, **_OOS_OK, + ) + ) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any(x.startswith("calendar span (B)") for x in r.predictive.reasons) + + +@pytest.mark.parametrize("unknown", ["effective_samples", "span_days"]) +def test_an_unknown_gate_fact_fails_the_gate_and_never_passes_it(unknown): + r = decide_verdict(_inputs(**{unknown: float("nan")}, **_STRONG, **_OOS_OK)) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any("UNKNOWN" in x for x in r.predictive.reasons) + + +def test_effective_samples_boundary_is_inclusive(): + """Exactly min_effective_samples lets the Predictive axis PASS (>=, not >).""" + ok = decide_verdict( + _inputs(settled_rebalances=24, effective_samples=24.0, **_STRONG, **_OOS_OK) + ) + assert ok.predictive.verdict == AXIS_PASS + below = decide_verdict( + _inputs(settled_rebalances=23, effective_samples=23.0, **_STRONG, **_OOS_OK) + ) + assert below.predictive.verdict == AXIS_INSUFFICIENT_DATA + + +def test_span_days_boundary_is_inclusive(): + ok = decide_verdict(_inputs(span_days=365.0, **_STRONG, **_OOS_OK)) + assert ok.predictive.verdict == AXIS_PASS + short = decide_verdict(_inputs(span_days=364.0, **_STRONG, **_OOS_OK)) + assert short.predictive.verdict == AXIS_INSUFFICIENT_DATA + + +def test_project_holdout_sample_size_hits_the_default_gate(): + """⚠️ Calibration flag, and the headline consequence of the v0.3 gate. + + The real P3-7/P3-8 holdouts settled ~21 monthly rebalances spanning ~670 + calendar days. Under the defaults the STATISTICAL axes are gated on part A + (N_eff <= N, so 21 raw periods can never reach 24), while the Tradable axis — + which is not a statistical claim — still PASSes, so the label is WATCH. Locked + as a REMINDER that these defaults are UNCALIBRATED, not an endorsement.""" + real = _inputs( + settled_rebalances=21, effective_samples=21.0, span_days=670.0, **_ADOPT_FACTS + ) + r = decide_verdict(real) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any("effective samples (A)" in x for x in r.predictive.reasons) + assert not any("calendar span" in x for x in r.predictive.reasons) # 670 >= 365 + assert r.incremental.verdict == AXIS_INSUFFICIENT_DATA + assert r.tradable.verdict == AXIS_PASS + assert r.verdict == WATCH + # ... and it takes lowering part A (not just the raw floor) to let it through. + assert decide_verdict(real, VerdictThresholds(min_rebalances=8)).verdict == WATCH + lenient = VerdictThresholds( + min_rebalances=8, min_effective_samples=8.0, min_span_days=365 + ) + non_exploratory = _inputs( + settled_rebalances=21, effective_samples=21.0, span_days=670.0, + **_ADOPT_FACTS, is_exploratory=False, + ) + assert decide_verdict(non_exploratory, lenient).verdict == ADOPT + + +def test_negative_sign_factor_is_judged_in_its_own_direction(): + """A low-vol factor (sign -1): negative IC / spread / orthogonalized IC ARE the + hypothesis, on EVERY axis.""" + kwargs = dict( + expected_ic_sign=-1, + **_SAMPLE_OK, + ic_ir=-0.8, + ic_ir_ci_low=-1.05, # CI brackets the negative point; the + ic_ir_ci_high=-0.55, # aligned lower bound is +0.55 > 0.30 + ic_win_rate=0.7, + ic_nw_t=-3.5, + monotonicity_spearman=-0.9, # high-vol bucket underperforms + net_long_short_by_cost=((1.0, -0.05),), # QN - Q1 negative: as predicted + known_factors_supplied=True, + incremental_ic_ir=-0.6, # residual IC negative: as predicted + incremental_ic_mean=-0.04, + incremental_ic_ir_ci_low=-0.78, + incremental_ic_ir_ci_high=-0.42, # aligned lower bound +0.42 > 0.30 + tradable=True, + capacity_sufficient=True, + oos_available=True, + oos_sign_consistent=True, + is_exploratory=False, + ) + r = decide_verdict(VerdictInputs(**kwargs)) + assert r.verdict == ADOPT + assert r.incremental.verdict == AXIS_PASS + + +def test_thresholds_are_configurable_but_the_rule_structure_is_not(): + weak = dict( + ic_ir=0.1, ic_ir_ci_low=0.06, ic_ir_ci_high=0.14, + ic_win_rate=0.52, ic_nw_t=1.0, monotonicity_spearman=0.5, + net_long_short_by_cost=((1.0, 0.01),), + ) + # default thresholds: the weak POINT (0.1 < 0.30) does not clear -> Predictive + # FAILs (point-based negative finding). + assert decide_verdict(_inputs(**weak, **_OOS_OK)).predictive.verdict == AXIS_FAIL + # lenient thresholds: the point clears (0.1 > 0.05) AND the lower CI bound + # (0.06 > 0.05) clears too -> PASS. + lenient = VerdictThresholds( + min_abs_icir=0.05, min_ic_win_rate=0.5, min_abs_nw_t=0.5, + min_monotonicity_spearman=0.4, + ) + assert ( + decide_verdict(_inputs(**weak, **_OOS_OK), lenient).predictive.verdict + == AXIS_PASS + ) + + +# -------------------------------------------------------------------------- +# The exploratory cap (design §6, v0.2, preserved): is_exploratory=True caps the +# DEPLOYMENT LABEL (an all-PASS Adopt -> Watch). Adopt IS a claim; an exploratory +# run declares it is not making one. +# -------------------------------------------------------------------------- + + +def test_exploratory_run_is_capped_at_watch_instead_of_adopt(): + r = decide_verdict(_inputs(**_ADOPT_FACTS, is_exploratory=True)) + assert r.verdict == WATCH + assert any("is_exploratory=True" in x for x in r.reasons) + assert any("CAPPED AT WATCH" in x for x in r.reasons) + + +def test_the_same_facts_without_the_exploratory_flag_still_adopt(): + assert decide_verdict(_inputs(**_ADOPT_FACTS, is_exploratory=False)).verdict == ADOPT + # ... and the VerdictInputs default (is_exploratory=False) is the Adopt path. + assert decide_verdict(_inputs(**_ADOPT_FACTS)).verdict == ADOPT + + +def test_the_capped_watch_still_reports_the_qualifying_evidence(): + """Capping withholds the LABEL, not the facts: the axes are identical either + way, and the capped Watch prepends exactly the cap reason to the Adopt evidence.""" + capped = decide_verdict(_inputs(**_ADOPT_FACTS, is_exploratory=True)) + adopted = decide_verdict(_inputs(**_ADOPT_FACTS, is_exploratory=False)) + assert capped.axes() == adopted.axes() # only the label moved + assert set(adopted.reasons).issubset(set(capped.reasons)) + assert len(capped.reasons) == len(adopted.reasons) + 1 + + +def test_the_cap_lands_on_watch_never_reject_or_insufficient(): + """The cap only downgrades the LABEL — it never converts a PASS into a FAIL.""" + r = decide_verdict(_inputs(**_ADOPT_FACTS, is_exploratory=True)) + assert r.verdict == WATCH + # a genuine failure under the same flag still Rejects: the cap does not rescue it. + failed = decide_verdict( + _inputs(**{**_ADOPT_FACTS, "is_exploratory": True, "tradable": False}) + ) + assert failed.verdict == REJECT + + +@pytest.mark.parametrize( + "failure", + [ + dict(oos_sign_flipped=True), + dict(oos_monotonicity_reversed=True), + dict(tradable=False), + dict(net_long_short_by_cost=((1.0, -0.01), (2.0, -0.03), (4.0, -0.08))), + ], +) +def test_an_exploratory_factor_that_fails_is_still_rejected(failure): + """A negative finding is a legitimate exploratory outcome (I5e); the cap must + not rescue a failure.""" + r = decide_verdict(_inputs(**{**_ADOPT_FACTS, "is_exploratory": True, **failure})) + assert r.verdict == REJECT + + +def test_an_exploratory_run_with_no_evidence_reads_insufficient_not_reject(): + """No OOS, no book, no execution facts -> no PASS, no FAIL -> INSUFFICIENT-DATA.""" + assert decide_verdict(_inputs(is_exploratory=True)).verdict == INSUFFICIENT_DATA + + +def test_the_sample_gate_still_precedes_the_exploratory_cap(): + """A too-small sample gates the statistical axes regardless of the flag.""" + r = decide_verdict( + _inputs(settled_rebalances=23, effective_samples=23.0, **_ADOPT_FACTS, is_exploratory=True) + ) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert r.incremental.verdict == AXIS_INSUFFICIENT_DATA + assert r.verdict == WATCH # tradable PASS carries it; the cap never applied + + +# -------------------------------------------------------------------------- +# The deployment derivation end-to-end THROUGH THE REPORT (extract -> decide). +# -------------------------------------------------------------------------- + + +def _adopt_grade_sections(**payloads) -> list: + """Section payloads that satisfy every axis of the §6 Adopt rule. + + A caller may OVERRIDE any section payload (e.g. purity= to test a redundant + book), so the overrides are merged over the defaults rather than passed + alongside them (which would be a duplicate keyword). + """ + defaults = dict( + data_coverage=_COVERAGE_OK, + predictive_power={ + "ic_ir": 0.8, + "ic_ir_ci_low": 0.55, + "ic_ir_ci_high": 1.05, + "ic_win_rate": 0.7, + "ic_nw_t": 3.5, + }, + return_risk={ + "monotonicity_spearman": 0.9, + "net_long_short_by_cost": {1.0: 0.05, 2.0: 0.04}, + }, + purity={ + "known_factors_supplied": True, + "incremental_ic_ir": 0.6, + "incremental_ic_mean": 0.04, + "incremental_ic_ir_ci_low": 0.42, + "incremental_ic_ir_ci_high": 0.78, + }, + oos_generalization={"oos_available": True, "sign_consistent": True}, + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ) + defaults.update(payloads) + return _full_sections(**defaults) + + +def test_post_hoc_selection_cannot_reach_adopt_through_the_report(): + """§4 forces post_hoc_selected=True => is_exploratory=True, so the §6 cap + closes the post-hoc -> Adopt hole end to end.""" + cfg = _cfg(is_exploratory=True, post_hoc_selected=True) + report = FactorEvalReport.assemble(_spec(), cfg, _adopt_grade_sections()).with_verdict() + assert report.verdict.verdict == WATCH + assert any("is_exploratory=True" in r for r in report.verdict.reasons) + + +def test_the_report_reads_the_cap_off_the_config_not_the_caveats_payload(): + cfg = _cfg(is_exploratory=True) + lying = _adopt_grade_sections(caveats={"is_exploratory": False}) + assert ( + FactorEvalReport.assemble(_spec(), cfg, lying).with_verdict().verdict.verdict + == WATCH + ) + without = _adopt_grade_sections()[:-1] + [Skipped("caveats", "not produced")] + assert ( + FactorEvalReport.assemble(_spec(), cfg, without).with_verdict().verdict.verdict + == WATCH + ) + + +def test_a_non_exploratory_report_can_still_reach_adopt(): + report = FactorEvalReport.assemble( + _spec(), _cfg(is_exploratory=False), _adopt_grade_sections() + ).with_verdict() + assert report.verdict.verdict == ADOPT + assert report.verdict.incremental.verdict == AXIS_PASS + + +def test_a_report_without_a_book_tops_out_at_watch(): + """The multi-factor point through the report: a purity section with no book flag + leaves the Incremental axis NOT_ASSESSED, so the label maxes at WATCH.""" + sections = _adopt_grade_sections(purity={}) # purity present, but no book + report = FactorEvalReport.assemble( + _spec(), _cfg(is_exploratory=False), sections + ).with_verdict() + assert report.verdict.verdict == WATCH + assert report.verdict.incremental.verdict == AXIS_NOT_ASSESSED + + +def test_a_report_with_a_redundant_factor_is_rejected_through_the_report(): + """THE headline new capability: a factor redundant with the supplied book gets + REJECTED even when its raw predictive signal and tradability are perfect.""" + redundant = _adopt_grade_sections( + purity={ + "known_factors_supplied": True, + "incremental_ic_ir": 0.01, + "incremental_ic_mean": 0.0, + } + ) + report = FactorEvalReport.assemble( + _spec(), _cfg(is_exploratory=False), redundant + ).with_verdict() + assert report.verdict.verdict == REJECT + assert report.verdict.incremental.verdict == AXIS_FAIL + + +def test_verdict_thresholds_validated(): + with pytest.raises(ValueError, match="min_rebalances"): + VerdictThresholds(min_rebalances=0) + with pytest.raises(ValueError, match="min_ic_win_rate"): + VerdictThresholds(min_ic_win_rate=1.5) + + +# -------------------------------------------------------------------------- +# VerdictThresholds type + range validation. +# These 5 numbers decide Adopt/Watch/Reject, so a garbage threshold yields a +# garbage VERDICT. Structural only: passing says NOTHING about calibration. +# -------------------------------------------------------------------------- + +_THRESHOLD_FIELDS = ( + "min_rebalances", + "min_effective_samples", + "min_span_days", + "min_abs_icir", + "min_incremental_abs_icir", + "min_ic_win_rate", + "min_abs_nw_t", + "min_monotonicity_spearman", +) + + +def test_every_threshold_field_is_validated(): + """Drift guard: a NEW threshold cannot be added without validation.""" + import dataclasses + + declared = {f.name for f in dataclasses.fields(VerdictThresholds)} + assert declared == set(_THRESHOLD_FIELDS), ( + "VerdictThresholds gained/lost a field: add it to the validators in " + "verdict.py AND to _THRESHOLD_FIELDS here." + ) + + +@pytest.mark.parametrize("field_name", _THRESHOLD_FIELDS) +@pytest.mark.parametrize("value", [True, False]) +def test_verdict_thresholds_reject_bool(field_name, value): + with pytest.raises(ValueError, match=field_name): + VerdictThresholds(**{field_name: value}) + + +@pytest.mark.parametrize("field_name", _THRESHOLD_FIELDS) +def test_verdict_thresholds_reject_a_string_with_a_readable_error(field_name): + """A bare TypeError leaking out of '"3" < 1' is not a readable error.""" + with pytest.raises(ValueError, match=field_name): + VerdictThresholds(**{field_name: "3"}) + + +@pytest.mark.parametrize("field_name", _THRESHOLD_FIELDS) +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_verdict_thresholds_reject_non_finite(field_name, value): + """NaN/inf silently disable a rule: every '>' against NaN is False.""" + with pytest.raises(ValueError, match=field_name): + VerdictThresholds(**{field_name: value}) + + +@pytest.mark.parametrize( + "field_name,value", + [ + ("min_rebalances", 0), + ("min_rebalances", -5), + ("min_abs_icir", -0.1), + ("min_abs_nw_t", -1.0), + ("min_ic_win_rate", 1.5), + ("min_ic_win_rate", -0.1), + ("min_monotonicity_spearman", 1.5), + ("min_monotonicity_spearman", -0.1), + ], +) +def test_verdict_thresholds_reject_out_of_range(field_name, value): + with pytest.raises(ValueError, match=field_name): + VerdictThresholds(**{field_name: value}) + + +def test_verdict_thresholds_accept_their_inclusive_lower_bounds(): + """Lower bounds are inclusive; the upper bounds are EXCLUSIVE (see below).""" + thr = VerdictThresholds( + min_rebalances=1, + min_abs_icir=0.0, + min_ic_win_rate=0.0, + min_abs_nw_t=0.0, + min_monotonicity_spearman=0.0, + ) + assert thr.min_rebalances == 1 + assert thr.min_monotonicity_spearman == 0.0 + + +@pytest.mark.parametrize( + "field_name", ["min_ic_win_rate", "min_monotonicity_spearman"] +) +def test_bounded_thresholds_reject_exactly_one_point_zero(field_name): + """1.0 is unreachable: both metrics max out at 1.0 and the rules use '>'. + + Same class of silent breakage as NaN/inf — a threshold that quietly makes the + rule it gates unsatisfiable — it just fails STRICT (the rule can never fire) + instead of permissive. + """ + with pytest.raises(ValueError, match=field_name) as excinfo: + VerdictThresholds(**{field_name: 1.0}) + message = str(excinfo.value) + assert "[0.0, 1.0)" in message # the exclusive bound is stated + assert "strict" in message # and WHY 1.0 can never be exceeded + + +@pytest.mark.parametrize( + "field_name", ["min_ic_win_rate", "min_monotonicity_spearman"] +) +def test_bounded_thresholds_accept_just_under_one(field_name): + assert getattr(VerdictThresholds(**{field_name: 0.999}), field_name) == 0.999 + + +def test_a_threshold_just_under_one_still_flows_through_decide_verdict(): + """0.999 is legal AND has teeth: it really gates the Predictive PASS rule.""" + near_perfect = VerdictThresholds( + min_ic_win_rate=0.999, min_monotonicity_spearman=0.999 + ) + facts = _inputs(**_STRONG, **_TRADABLE, **_OOS_OK) + # defaults: OOS + strong metrics -> Predictive PASS (deployment WATCH, no book). + assert decide_verdict(facts).predictive.verdict == AXIS_PASS + assert decide_verdict(facts).verdict == WATCH + # demanding a >0.999 win rate: 0.7 no longer clears -> the Predictive axis fails. + assert decide_verdict(facts, near_perfect).predictive.verdict == AXIS_FAIL + + +def test_valid_custom_thresholds_still_flow_through_decide_verdict(): + """Validation must not break the configurability the axis rules depend on.""" + lenient = VerdictThresholds( + min_rebalances=8, + min_effective_samples=8.0, + min_span_days=180, + min_abs_icir=0.05, + min_ic_win_rate=0.5, + min_abs_nw_t=0.5, + min_monotonicity_spearman=0.4, + ) + real = _inputs( + settled_rebalances=21, effective_samples=21.0, span_days=200.0, + **_ADOPT_FACTS, is_exploratory=False, + ) + assert decide_verdict(real, lenient).verdict == ADOPT + # defaults gate the statistical axes; the Tradable axis still PASSes -> WATCH. + assert decide_verdict(real).verdict == WATCH + + +def test_int_is_accepted_for_a_float_threshold(): + """1 (int) is a perfectly good |ICIR| threshold; only bool is excluded.""" + assert VerdictThresholds(min_abs_icir=1).min_abs_icir == 1 + + +# -------------------------------------------------------------------------- +# #3: pre-registered criteria + lower-CI gating (design §6, v0.6) +# -------------------------------------------------------------------------- + + +def test_predictive_pass_gates_on_the_lower_ci_bound_not_the_point(): + """THE CI-gating headline: a POINT that clears the bar but a LOWER CI that does + NOT -> the axis is NOT a PASS. The sample gate passed (sufficient N_eff), so this + is the 'promising, unconfirmed' INSUFFICIENT ABOVE the gate — not a gate failure + below it, and not a FAIL.""" + wide = decide_verdict( + _inputs( + ic_ir=0.8, ic_ir_ci_low=0.10, ic_ir_ci_high=1.50, # lower 0.10 < 0.30 + ic_win_rate=0.7, ic_nw_t=3.5, monotonicity_spearman=0.9, **_OOS_OK, + ) + ) + assert wide.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert wide.predictive.verdict != AXIS_PASS + assert any("LOWER CI" in r for r in wide.predictive.reasons) + + +def test_predictive_pass_when_the_lower_ci_clears(): + """Regression: a tight estimate whose lower CI clears -> PASS (Adopt reachable).""" + assert decide_verdict(_inputs(**_STRONG, **_OOS_OK)).predictive.verdict == AXIS_PASS + + +def test_incremental_pass_gates_on_the_lower_ci_bound(): + wide = decide_verdict( + _inputs( + **_STRONG, **_OOS_OK, + known_factors_supplied=True, incremental_ic_ir=0.6, incremental_ic_mean=0.04, + incremental_ic_ir_ci_low=0.05, incremental_ic_ir_ci_high=1.15, # lower < 0.30 + ) + ) + assert wide.incremental.verdict == AXIS_INSUFFICIENT_DATA + assert wide.incremental.verdict != AXIS_PASS + + +def test_a_nan_ci_bound_never_produces_a_fail_and_never_passes(): + """Unknown never convicts, under CI gating: a point that clears in the expected + direction with an UNKNOWN CI is NOT a PASS (can't confirm) and NOT a FAIL.""" + pred = decide_verdict( + _inputs(ic_ir=0.8, ic_win_rate=0.7, ic_nw_t=3.5, monotonicity_spearman=0.9, **_OOS_OK) + ) # no ic_ir_ci_* -> NaN + assert pred.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert pred.predictive.verdict != AXIS_FAIL + incr = decide_verdict( + _inputs( + **_STRONG, **_OOS_OK, + known_factors_supplied=True, incremental_ic_ir=0.6, incremental_ic_mean=0.04, + ) + ) # no incremental_ic_ir_ci_* -> NaN + assert incr.incremental.verdict == AXIS_INSUFFICIENT_DATA + assert incr.incremental.verdict != AXIS_FAIL + + +def test_a_redundant_point_still_fails_regardless_of_its_ci(): + """FAIL stays POINT-based: a measured ~ 0 orthogonalized IC is redundant even + with a (wide) CI attached — the CI is only consulted for the PASS test.""" + r = decide_verdict( + _inputs( + **_STRONG, **_OOS_OK, + known_factors_supplied=True, incremental_ic_ir=0.01, incremental_ic_mean=0.0, + incremental_ic_ir_ci_low=-0.40, incremental_ic_ir_ci_high=0.42, + ) + ) + assert r.incremental.verdict == AXIS_FAIL + + +def test_default_criteria_source_is_stamped_default(): + report = _verdicted_report() # _cfg declares no success_criteria + assert report.criteria_source == "default" + assert json.loads(report.to_json())["criteria_source"] == "default" + assert "DEFAULT global bar" in report.render() + + +def test_declared_success_criteria_are_stamped_and_used(): + """A pre-registered bar declared on the frozen EvalConfig is USED and DISCLOSED.""" + criteria = VerdictThresholds(min_abs_icir=0.05) + cfg = _cfg(is_exploratory=False, success_criteria=criteria) + report = FactorEvalReport.assemble( + _spec(), cfg, _adopt_grade_sections() + ).with_verdict() + assert report.criteria_source == "declared" + assert report.thresholds is criteria # the declared object, not a default + exported = json.loads(report.to_json()) + assert exported["criteria_source"] == "declared" + assert exported["thresholds"]["min_abs_icir"] == 0.05 + assert "PRE-REGISTERED" in report.render() + + +def test_a_stricter_declared_bar_flips_a_borderline_pass_through_decide_verdict(): + """The load-bearing point of pre-registration: the threshold is a run-declared + input now, so a stricter declared bar changes the verdict end to end. _STRONG's + ICIR lower CI bound is 0.55: a 0.50 bar clears, a 0.70 bar does not.""" + lenient = _cfg( + is_exploratory=False, success_criteria=VerdictThresholds(min_abs_icir=0.50) + ) + strict = _cfg( + is_exploratory=False, success_criteria=VerdictThresholds(min_abs_icir=0.70) + ) + passed = FactorEvalReport.assemble( + _spec(), lenient, _adopt_grade_sections() + ).with_verdict() + blocked = FactorEvalReport.assemble( + _spec(), strict, _adopt_grade_sections() + ).with_verdict() + assert passed.verdict.predictive.verdict == AXIS_PASS + assert blocked.verdict.predictive.verdict != AXIS_PASS + + +def test_success_criteria_must_be_a_verdict_thresholds_not_a_dict(): + """A dict would be silently ignored (fall back to default) — a pre-registration + that never took effect. Rejected loudly instead.""" + with pytest.raises(ValueError, match="success_criteria"): + _cfg(success_criteria={"min_abs_icir": 0.3}) + + +def test_a_declared_bar_is_not_weakened_for_being_per_run(): + """The per-run object passes exactly the same VerdictThresholds validation.""" + with pytest.raises(ValueError, match="min_ic_win_rate"): + _cfg(success_criteria=VerdictThresholds(min_ic_win_rate=1.0)) # 1.0 unreachable + cfg = _cfg(success_criteria=VerdictThresholds(min_abs_icir=0.4)) + assert cfg.success_criteria.min_abs_icir == 0.4 + + +# -------------------------------------------------------------------------- +# Report render / JSON: deterministic + secret-free +# -------------------------------------------------------------------------- + + +def _verdicted_report(**payloads) -> FactorEvalReport: + defaults = dict( + data_coverage=_COVERAGE_OK, + predictive_power={ + "ic_ir": 0.8, + "ic_ir_ci_low": 0.55, + "ic_ir_ci_high": 1.05, + "ic_win_rate": 0.7, + "ic_nw_t": 3.5, + }, + return_risk={ + "monotonicity_spearman": 0.9, + "net_long_short_by_cost": {1.0: 0.05, 2.0: 0.04}, + }, + purity={ + "known_factors_supplied": True, + "incremental_ic_ir": 0.6, + "incremental_ic_mean": 0.04, + "incremental_ic_ir_ci_low": 0.42, + "incremental_ic_ir_ci_high": 0.78, + }, + oos_generalization={"oos_available": True, "sign_consistent": True}, + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ) + defaults.update(payloads) + sections = _full_sections(**defaults) + # is_exploratory=False on purpose: these render/export tests want a report + # that actually reaches Adopt, and the §6 cap denies Adopt to an exploratory + # run (which _cfg() declares by default). + report = FactorEvalReport.assemble(_spec(), _cfg(is_exploratory=False), sections) + report.validate_all_mandatory_present() + return report.with_verdict() + + +def test_render_requires_a_verdict(): + report = FactorEvalReport.assemble(_spec(), _cfg(), _full_sections()) + with pytest.raises(ValueError, match="no verdict"): + report.render() + with pytest.raises(ValueError, match="no verdict"): + report.to_dict() + + +def test_render_is_deterministic_and_contains_all_ten_sections(): + report = _verdicted_report() + first, second = report.render(), report.render() + assert first == second + assert first.startswith("# Factor Evaluation — unit_test_factor (v1.0)") + assert "## 0. Header & Provenance" in first + assert "## 1. Verdict & Scorecard" in first + for number, name in enumerate(MANDATORY_SECTIONS, start=2): + assert f"## {number}." in first, name + assert "Adopt" in first + + +def test_render_discloses_a_skipped_section_with_its_reason(): + sections = _full_sections()[:-1] + [Skipped("caveats", "no caveats configured")] + report = FactorEvalReport.assemble(_spec(), _cfg(), sections).with_verdict() + rendered = report.render() + assert "_Skipped: no caveats configured_" in rendered + + +def test_payload_key_order_does_not_change_the_render(): + """Insertion order must not leak into the artifact (sorted keys).""" + a = Section("purity", {"vif": 1.2, "corr_value_ep": 0.3}) + b = Section("purity", {"corr_value_ep": 0.3, "vif": 1.2}) + spec, cfg = _spec(), _cfg() + ra = FactorEvalReport.assemble(spec, cfg, _full_sections()[:3] + [a] + _full_sections()[4:]) + rb = FactorEvalReport.assemble(spec, cfg, _full_sections()[:3] + [b] + _full_sections()[4:]) + assert ra.with_verdict().render() == rb.with_verdict().render() + + +def test_json_export_is_deterministic_and_machine_readable(): + report = _verdicted_report() + payload = json.loads(report.to_json()) + assert payload["schema_version"] == "0.1" + assert payload["spec"]["factor_id"] == "unit_test_factor" + assert payload["verdict"]["verdict"] == ADOPT + assert {s["name"] for s in payload["sections"]} == set(MANDATORY_SECTIONS) + assert report.to_json() == report.to_json() + + +def test_report_is_secret_free(): + """A token/path smuggled into a payload or reason never reaches the artifact.""" + leaky = "/home/u/.config.json token=abcdef123456" + sections = _full_sections()[:-1] + [Skipped("caveats", f"loaded {leaky}")] + sections[3] = Section("purity", {"src": leaky}, note=f"note {leaky}") + report = FactorEvalReport.assemble(_spec(), _cfg(), sections).with_verdict() + for text in (report.render(), report.to_json()): + assert "abcdef123456" not in text + assert ".config.json" not in text + assert "[REDACTED]" in text + + +def test_report_redacts_a_secret_looking_payload_key(): + """Keys reach the artifact too, not just values. + + D3's own _format_examples redacts example KEYS; leaving them raw here was + both leaky and inconsistent with the layer this reuses. + """ + report = _verdicted_report( + purity={"token=abcdef123456": 1}, + stability_cost={"/home/u/.config.json": 2}, + ) + for text in (report.render(), report.to_json()): + assert "abcdef123456" not in text + assert ".config.json" not in text + assert "[REDACTED]" in text + + +def test_a_nested_secret_looking_payload_key_is_redacted_too(): + """Redaction recurses: a key nested inside a payload value is redacted too. + + NOTE the assertion is scoped to what the D3 layer actually promises — it + matches secret-SHAPED patterns (``tushare.token`` / ``token=...`` / + ``*.config.json``), it is not a secrets detector. A bare 'abcdef123456' with + no such marker is left alone by design, so asserting otherwise here would + lock in a guarantee the layer does not make. + """ + report = _verdicted_report( + purity={"sources": {"tushare.token": "token=abcdef123456"}} + ) + for text in (report.render(), report.to_json()): + assert "tushare.token" not in text # the nested KEY + assert "abcdef123456" not in text # the pattern-shaped VALUE + assert "[REDACTED]" in text + + +def test_redacting_keys_keeps_the_render_deterministic(): + """Two secret-shaped keys can collapse onto one marker; it must not flap.""" + report = _verdicted_report(purity={"token=aaa": 1, "token=bbb": 2}) + assert report.render() == report.render() + assert report.to_json() == report.to_json() + + +def test_section_payload_is_copied_not_aliased(): + payload = {"ic_ir": 0.5} + section = Section("predictive_power", payload) + payload["ic_ir"] = 999.0 # mutating the caller's dict must not touch the section + assert section.payload["ic_ir"] == 0.5 + + +def test_section_payload_nested_values_are_deep_copied(): + """A shallow dict() left the NESTED map aliased to the caller's object. + + ``net_long_short_by_cost`` is the exact key the verdict reads, so a post-hoc + nested mutation could rewrite a finished report's facts. + """ + payload = {"net_long_short_by_cost": {1.0: 0.05, 2.0: 0.04}, "tags": ["base"]} + section = Section("return_risk", payload) + payload["net_long_short_by_cost"][1.0] = -999.0 + payload["tags"].append("mutated") + assert section.payload["net_long_short_by_cost"] == {1.0: 0.05, 2.0: 0.04} + assert section.payload["tags"] == ["base"] + + +def test_nested_mutation_after_the_fact_cannot_rewrite_the_verdict(): + """End-to-end of the same hole: an Adopt must not be mutable into a Reject.""" + costs = {1.0: 0.05, 2.0: 0.04} + sections = _full_sections( + data_coverage=dict(_COVERAGE_OK), + predictive_power={ + "ic_ir": 0.8, + "ic_ir_ci_low": 0.55, + "ic_ir_ci_high": 1.05, + "ic_win_rate": 0.7, + "ic_nw_t": 3.5, + }, + return_risk={"monotonicity_spearman": 0.9, "net_long_short_by_cost": costs}, + purity={ + "known_factors_supplied": True, + "incremental_ic_ir": 0.6, + "incremental_ic_mean": 0.04, + "incremental_ic_ir_ci_low": 0.42, + "incremental_ic_ir_ci_high": 0.78, + }, + oos_generalization={"oos_available": True, "sign_consistent": True}, + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ) + report = FactorEvalReport.assemble( + _spec(), _cfg(is_exploratory=False), sections + ).with_verdict() + assert report.verdict.verdict == ADOPT + costs[1.0] = -1.0 # flip the base spread AFTER the report was built + assert report.with_verdict().verdict.verdict == ADOPT + + +# -------------------------------------------------------------------------- +# A verdicted report is ALWAYS a complete report (enforcement layer #2 made +# unbypassable), and no export may silently omit a mandatory section. +# -------------------------------------------------------------------------- + + +def test_with_verdict_validates_even_when_the_caller_skips_the_explicit_check(): + """The contract's central promise must not depend on caller discipline. + + ``assemble().with_verdict()`` is public; without an internal validation a + 3-of-8 report gets a real verdict and renders as a corrupt record. + """ + report = FactorEvalReport.assemble(_spec(), _cfg(), _full_sections()[:3]) + with pytest.raises(ValueError, match="missing mandatory section"): + report.with_verdict() # no explicit validate_all_mandatory_present() call + + +def test_with_verdict_names_every_missing_section(): + report = FactorEvalReport.assemble(_spec(), _cfg(), _full_sections()[:3]) + with pytest.raises(ValueError) as excinfo: + report.with_verdict() + for name in MANDATORY_SECTIONS[3:]: + assert name in str(excinfo.value) + + +def test_with_verdict_still_accepts_a_complete_report_of_skipped_sections(): + """Disclosure, not results: Skipped-with-reason is a complete report.""" + sections = [Skipped(name, "not configured") for name in MANDATORY_SECTIONS] + report = FactorEvalReport.assemble(_spec(), _cfg(), sections).with_verdict() + assert report.verdict is not None + + +def _hand_built_incomplete_report() -> FactorEvalReport: + """Bypass assemble+with_verdict entirely — the only route left to an export.""" + return FactorEvalReport( + spec=_spec(), + cfg=_cfg(), + sections=tuple(_full_sections()[:3]), + verdict=VerdictResult(REJECT, ("hand-built",)), + ) + + +def test_json_marks_a_missing_mandatory_section_instead_of_dropping_it(): + """The actual hole: render() printed _MISSING_ 5x while to_json() shipped 3/8. + + A short record is structurally valid and would silently poison the cross-run + factor library, so the JSON states the hole explicitly. + """ + payload = json.loads(_hand_built_incomplete_report().to_json()) + names = [s["name"] for s in payload["sections"]] + assert names[: len(MANDATORY_SECTIONS)] == list(MANDATORY_SECTIONS) + by_name = {s["name"]: s for s in payload["sections"]} + for name in MANDATORY_SECTIONS[:3]: + assert by_name[name]["status"] == "ok" + for name in MANDATORY_SECTIONS[3:]: + assert by_name[name]["status"] == "missing" + assert "MISSING" in by_name[name]["reason"] + + +def test_markdown_and_json_agree_about_a_hole(): + report = _hand_built_incomplete_report() + rendered = report.render() + assert rendered.count("_MISSING — the contract requires this section._") == 5 + exported = report.to_dict() + n_missing = sum(1 for s in exported["sections"] if s["status"] == "missing") + assert n_missing == 5 + + +def test_json_section_order_is_canonical_not_assembly_order(): + """Two equivalent reports assembled in different order must be byte-identical. + + sort_keys=True sorts dict KEYS, not list elements — the sections list has to + be canonicalized itself, or a cross-run diff is pure noise. + """ + spec, cfg = _spec(), _cfg() + payloads = dict( + data_coverage=_COVERAGE_OK, + predictive_power={"ic_ir": 0.8, "ic_win_rate": 0.7, "ic_nw_t": 3.5}, + return_risk={ + "monotonicity_spearman": 0.9, + "net_long_short_by_cost": {1.0: 0.05}, + }, + oos_generalization={"oos_available": True, "sign_consistent": True}, + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ) + forward = _full_sections(**payloads) + backward = list(reversed(forward)) + a = FactorEvalReport.assemble(spec, cfg, forward).with_verdict() + b = FactorEvalReport.assemble(spec, cfg, backward).with_verdict() + assert a.to_json() == b.to_json() + assert a.render() == b.render() + names = [s["name"] for s in json.loads(a.to_json())["sections"]] + assert names == list(MANDATORY_SECTIONS) + + +def test_extra_sections_are_canonicalized_by_name_too(): + spec, cfg = _spec(), _cfg() + extras = [Section("crowding", {"x": 1}), Section("alt_data", {"y": 2})] + a = FactorEvalReport.assemble(spec, cfg, [*_full_sections(), *extras]) + b = FactorEvalReport.assemble(spec, cfg, [*_full_sections(), *reversed(extras)]) + assert a.with_verdict().to_json() == b.with_verdict().to_json() + names = [s["name"] for s in json.loads(a.with_verdict().to_json())["sections"]] + assert names == [*MANDATORY_SECTIONS, "alt_data", "crowding"] + + +def test_a_huge_payload_value_is_truncated_in_the_artifact(): + """Bounded like the D3 quality layer's examples: a report is a summary.""" + report = _verdicted_report(purity={"dump": "x" * 5000}) + rendered = report.render() + assert "...[truncated]" in rendered + assert "x" * 5000 not in rendered + exported = report.to_dict() + dump = next(s for s in exported["sections"] if s["name"] == "purity")["payload"] + assert len(dump["dump"]) == MAX_VALUE_CHARS + len("...[truncated]") + + +# -------------------------------------------------------------------------- +# FactorEvaluator ABC +# -------------------------------------------------------------------------- + + +class _SpyEvaluator(FactorEvaluator): + """Minimal evaluator: records call order, computes nothing (PR-B does that).""" + + def __init__(self) -> None: + self.calls: list[str] = [] + + def build_ir(self, factor_panel, spec, cfg, ctx=None): + self.calls.append("build_ir") + return object() + + def _record(self, name: str) -> Section: + self.calls.append(name) + return Section(name, {}) + + def predictive_power(self, ir): + self.calls.append("predictive_power") + return Section("predictive_power", {"ic_ir": 0.8, "ic_win_rate": 0.7, "ic_nw_t": 3.5}) + + def return_risk(self, ir): + self.calls.append("return_risk") + return Section( + "return_risk", + {"monotonicity_spearman": 0.9, "net_long_short_by_cost": {1.0: 0.05}}, + ) + + def stability_cost(self, ir): + return self._record("stability_cost") + + def purity(self, ir): + return self._record("purity") + + def oos_generalization(self, ir): + self.calls.append("oos_generalization") + return Skipped("oos_generalization", "no oos_split configured") + + def execution_capacity(self, ir): + return self._record("execution_capacity") + + def data_coverage(self, ir): + self.calls.append("data_coverage") + # clears the three-part gate, so the ORDER/verdict assertions below are + # about the template method rather than the sample gate. + return Section( + "data_coverage", + {"settled_rebalances": 30, "effective_samples": 30.0, "span_days": 400.0}, + ) + + def caveats(self, ir): + return self._record("caveats") + + +def test_evaluator_missing_a_mandatory_section_cannot_be_instantiated(): + """The ABC is lock #1: you cannot drop a mandatory section, only Skip it.""" + + class MissingCaveats(FactorEvaluator): + def build_ir(self, factor_panel, spec, cfg, ctx=None): + return object() + + def predictive_power(self, ir): + return Section("predictive_power", {}) + + def return_risk(self, ir): + return Section("return_risk", {}) + + def stability_cost(self, ir): + return Section("stability_cost", {}) + + def purity(self, ir): + return Section("purity", {}) + + def oos_generalization(self, ir): + return Section("oos_generalization", {}) + + def execution_capacity(self, ir): + return Section("execution_capacity", {}) + + def data_coverage(self, ir): + return Section("data_coverage", {}) + + # 'caveats' deliberately not implemented + + with pytest.raises(TypeError, match="abstract"): + MissingCaveats() + + +def test_evaluator_missing_the_ir_builder_cannot_be_instantiated(): + class NoIR(_SpyEvaluator): + build_ir = FactorEvaluator.build_ir + + with pytest.raises(TypeError, match="abstract"): + NoIR() + + +def test_template_method_calls_sections_in_the_fixed_order(): + spy = _SpyEvaluator() + report = spy.evaluate(factor_panel=None, spec=_spec(), cfg=_cfg()) + assert spy.calls == ["build_ir", *MANDATORY_SECTIONS] + report.validate_all_mandatory_present() + assert report.verdict is not None + + +def test_template_method_verdict_reflects_a_skipped_oos_section(): + """Skipped OOS -> Predictive INSUFFICIENT_DATA; no book / no execution facts + -> Incremental & Tradable NOT_ASSESSED -> no PASS, no FAIL -> INSUFFICIENT-DATA. + + (Under the scalar verdict this was Watch; the three-axis structure reads it as + "we cannot tell" — there is no positive claim on any axis.) + """ + report = _SpyEvaluator().evaluate(factor_panel=None, spec=_spec(), cfg=_cfg()) + assert report.verdict.verdict == INSUFFICIENT_DATA + assert report.verdict.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any("no out-of-sample split" in r for r in report.verdict.reasons) + assert "_Skipped: no oos_split configured_" in report.render() diff --git a/tests/test_factor_eval_standard.py b/tests/test_factor_eval_standard.py new file mode 100644 index 0000000..499e604 --- /dev/null +++ b/tests/test_factor_eval_standard.py @@ -0,0 +1,1898 @@ +"""PR-B: the vectorized eval-IR + ``StandardFactorEvaluator``. + +The load-bearing tests here are the EQUIVALENCE ones. ``analytics/eval/ir.py`` +replaces a per-rebalance Python loop with grouped whole-panel reductions, and the +only way that is a speed-up rather than a rewrite of the semantics is if it +produces the SAME numbers. So this file carries a deliberately naive per-period +loop (:func:`naive_ic`, :func:`naive_quantile_returns`, written in the shape +``analytics/factor.py`` actually uses today) and asserts the vectorized objects +match it — on ragged cross-sections, NaNs, ties, single-symbol dates and all-NaN +dates, not just on a tidy rectangle. + +Everything is synthetic: no network, no cache, no tushare. +""" + +from __future__ import annotations + +import math + +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal, assert_series_equal + +from analytics.eval import ( + ADOPT, + AXIS_FAIL, + AXIS_INSUFFICIENT_DATA, + AXIS_NOT_ASSESSED, + AXIS_PASS, + INSUFFICIENT_DATA, + MANDATORY_SECTIONS, + REJECT, + WATCH, + EvalConfig, + EvalContext, + FactorEvalReport, + Section, + Skipped, + StandardFactorEvaluator, + VerdictInputs, + VerdictThresholds, + build_eval_ir, + decide_verdict, +) +from analytics.eval.ir import assign_quantile_buckets, quantile_turnover +from analytics.eval.stats import ( + effective_sample_size, + half_life, + hypothesis_win_rate, + information_ratio_ci, + mean_ci, + newey_west_lag, + newey_west_t, + sortino, +) +from analytics.factor import compute_ic +from factors.spec import FactorSpec +from qt.intraday_groups import assign_quantile_buckets as canonical_buckets + +DATE = "date" +SYMBOL = "symbol" + + +# ========================================================================== +# the naive per-period reference (what the vectorized IR must reproduce) +# ========================================================================== + + +def naive_ic(factor: pd.Series, fwd: pd.Series, dates: pd.Index) -> pd.Series: + """Rank IC, ONE PYTHON ITERATION PER DATE. Deliberately naive. + + Mirrors ``analytics/factor.py::compute_ic`` (drop non-finite pairs, NaN when + fewer than 2 survive or either side is constant, Spearman otherwise). + """ + aligned = pd.DataFrame({"f": factor, "r": fwd}) + date_values = aligned.index.get_level_values(DATE) + out: dict[object, float] = {} + for date, block in aligned.groupby(date_values, sort=True): + pair = block.replace([np.inf, -np.inf], np.nan).dropna() + if len(pair) < 2 or pair["f"].nunique() < 2 or pair["r"].nunique() < 2: + out[date] = float("nan") + else: + out[date] = float(pair["f"].corr(pair["r"], method="spearman")) + return pd.Series(out, dtype=float).reindex(dates) + + +def naive_quantile_returns( + factor: pd.Series, fwd: pd.Series, n_quantiles: int, dates: pd.Index +) -> pd.DataFrame: + """Mean forward return per (date, bucket), ONE PYTHON ITERATION PER DATE. + + Same semantics the IR documents: buckets come from the FACTOR cross-section + alone, ordered by ``(factor, symbol)`` and split by position exactly as + ``qt.intraday_groups.assign_quantile_buckets`` does. + """ + rows: dict[object, dict[int, float]] = {} + factor_by_date = factor.groupby(factor.index.get_level_values(DATE), sort=True) + for date, block in factor_by_date: + values = ( + block.droplevel(DATE).replace([np.inf, -np.inf], np.nan).dropna() + ) + returns = fwd.xs(date, level=DATE) if len(values) else pd.Series(dtype=float) + order = sorted(values.index, key=lambda s: (float(values.loc[s]), str(s))) + chunks = np.array_split(np.array(order, dtype=object), n_quantiles) + row: dict[int, float] = {} + for i, chunk in enumerate(chunks, start=1): + got = pd.Series( + [returns.get(sym, float("nan")) for sym in chunk], dtype=float + ) + got = got.replace([np.inf, -np.inf], np.nan).dropna() + row[i] = float(got.mean()) if len(got) else float("nan") + rows[date] = row + frame = pd.DataFrame(rows).T if rows else pd.DataFrame() + return frame.reindex(index=dates, columns=list(range(1, n_quantiles + 1))) + + +# ========================================================================== +# synthetic fixtures +# ========================================================================== + + +def make_price_panel(dates: pd.Index, symbols: list[str], rng, signal=None) -> pd.DataFrame: + """A CANONICAL market panel whose returns optionally follow a planted signal.""" + n_d, n_s = len(dates), len(symbols) + noise = rng.normal(0.0, 0.02, size=(n_d, n_s)) + returns = noise if signal is None else noise + signal + log_price = np.log(100.0) + np.cumsum(returns, axis=0) + close = pd.DataFrame(np.exp(log_price), index=dates, columns=symbols) + stacked = close.stack() + stacked.index.names = [DATE, SYMBOL] + stacked = stacked.sort_index() + return pd.DataFrame( + { + "open": stacked, + "high": stacked * 1.01, + "low": stacked * 0.99, + "close": stacked, + "volume": 1_000_000.0, + "amount": 100_000_000.0, + "adj_factor": 1.0, + } + ) + + +def make_spec(**overrides) -> FactorSpec: + kwargs = dict( + factor_id="synth_factor", + version="1.0", + description="a synthetic factor for tests", + expected_ic_sign=1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + input_fields=("close",), + ) + kwargs.update(overrides) + return FactorSpec(**kwargs) + + +def make_cfg(**overrides) -> EvalConfig: + kwargs = dict( + universe="TEST500", + universe_is_pit=True, + start="2024-01-01", + end="2025-01-01", + is_exploratory=True, + post_hoc_selected=False, + rebalance="daily", + n_quantiles=5, + ) + kwargs.update(overrides) + return EvalConfig(**kwargs) + + +def panel_dates(obj) -> pd.DatetimeIndex: + """The sorted unique dates of a panel, as real Timestamps.""" + return pd.DatetimeIndex( + pd.unique(obj.index.get_level_values(DATE)), name=DATE + ).sort_values() + + +@pytest.fixture +def rng(): + return np.random.default_rng(20260716) + + +def ragged_panel(rng, n_dates=40, n_symbols=23): + """A panel with ragged cross-sections, NaNs, ties and degenerate dates. + + The point is that the vectorized path must survive exactly what real A-share + panels do: names appearing/leaving, warm-up NaNs, tied factor values, a date + with one name and a date with nothing at all. + """ + dates = pd.bdate_range("2024-01-02", periods=n_dates, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(n_symbols)] + panel = make_price_panel(dates, symbols, rng) + factor = pd.Series(rng.normal(size=len(panel)), index=panel.index, name="synth_factor") + + # ties: a whole block of identical values (rank tie-break must decide) + factor.loc[(dates[3], symbols[0]):(dates[3], symbols[9])] = 0.5 + # ragged: drop a chunk of names on some dates + for i, date in enumerate(dates[5:15]): + for sym in symbols[: (i % 7) + 3]: + factor.loc[(date, sym)] = np.nan + # a single-symbol cross-section + for sym in symbols[1:]: + factor.loc[(dates[20], sym)] = np.nan + # an all-NaN cross-section + factor.loc[dates[21]] = np.nan + # a constant (zero-variance) cross-section + factor.loc[dates[22]] = 7.0 + # an infinity, which must be treated as missing rather than as an extreme + factor.loc[(dates[25], symbols[2])] = np.inf + return panel, factor + + +# ========================================================================== +# 1. EQUIVALENCE: vectorized IR vs the naive per-period loop +# ========================================================================== + + +def test_ic_matches_naive_loop_on_ragged_panel(rng): + panel, factor = ragged_panel(rng) + ir = build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + expected = naive_ic(ir.factor, ir.forward_returns, ir.dates) + assert_series_equal( + ir.ic.rename(None), expected.rename(None), check_names=False, rtol=1e-12, atol=1e-12 + ) + # the fixture must actually exercise the degenerate paths, or this proves nothing + assert ir.ic.isna().sum() >= 3 + assert ir.ic.notna().sum() >= 20 + + +def test_quantile_returns_match_naive_loop_on_ragged_panel(rng): + panel, factor = ragged_panel(rng) + cfg = make_cfg(n_quantiles=5) + ir = build_eval_ir(factor, make_spec(), cfg, EvalContext(price_panel=panel)) + expected = naive_quantile_returns(ir.factor, ir.forward_returns, 5, ir.dates) + got = ir.quantile_returns.copy() + got.columns = list(got.columns) + expected.columns = list(expected.columns) + assert_frame_equal( + got, expected, check_names=False, check_column_type=False, rtol=1e-12, atol=1e-12 + ) + + +@pytest.mark.parametrize("n_quantiles", [2, 3, 5, 7]) +def test_quantile_returns_match_naive_loop_across_bucket_counts(rng, n_quantiles): + panel, factor = ragged_panel(rng) + ir = build_eval_ir( + factor, make_spec(), make_cfg(n_quantiles=n_quantiles), EvalContext(price_panel=panel) + ) + expected = naive_quantile_returns(ir.factor, ir.forward_returns, n_quantiles, ir.dates) + got = ir.quantile_returns.copy() + got.columns = list(got.columns) + expected.columns = list(expected.columns) + assert_frame_equal( + got, expected, check_names=False, check_column_type=False, rtol=1e-12, atol=1e-12 + ) + + +def test_ic_matches_the_projects_authoritative_compute_ic(rng): + """The vectorized rank IC must equal ``analytics.factor.compute_ic`` exactly. + + Not merely 'equal to my own reference': the eval layer must not silently + disagree with the IC the rest of the framework (P3-x runners, ic_weight alpha) + has been reporting. + """ + panel, factor = ragged_panel(rng) + ir = build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + expected = compute_ic(ir.factor, ir.forward_returns, method="spearman") + assert_series_equal( + ir.ic.rename(None), expected.rename(None), check_names=False, rtol=1e-12, atol=1e-12 + ) + + +def test_pearson_ic_matches_compute_ic(rng): + panel, factor = ragged_panel(rng) + ir = build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + expected = compute_ic(ir.factor, ir.forward_returns, method="pearson") + assert_series_equal( + ir.ic_pearson.rename(None), expected.rename(None), check_names=False, + rtol=1e-10, atol=1e-10, + ) + + +def test_bucketing_matches_the_projects_canonical_equal_count_rule(rng): + """Vectorized buckets == ``qt.intraday_groups.assign_quantile_buckets`` (I5d/I5e). + + That function is the project's canonical equal-count rank rule and it works + per cross-section with ``sorted() + np.array_split``. The IR reproduces it with + integer arithmetic on grouped ranks; if the two ever diverge, an eval report's + quintiles would stop meaning what the grouped backtest's quintiles mean. + """ + _, factor = ragged_panel(rng) + for n_groups in (2, 3, 5, 7): + labels = assign_quantile_buckets(factor, n_groups) + for date in pd.unique(factor.index.get_level_values(DATE)): + cross_section = factor.xs(date, level=DATE).replace( + [np.inf, -np.inf], np.nan + ) + expected = canonical_buckets(cross_section, n_groups) + got = ( + labels.xs(date, level=DATE).to_dict() + if date in labels.index.get_level_values(DATE) + else {} + ) + assert got == expected, f"bucket mismatch on {date} with {n_groups} groups" + + +def test_bucket_sizes_are_equal_count_with_extras_in_the_low_buckets(): + """13 names into 5 buckets -> 3,3,3,2,2 (the np.array_split convention).""" + dates = pd.bdate_range("2024-01-02", periods=1, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(13)] + index = pd.MultiIndex.from_product([dates, symbols], names=[DATE, SYMBOL]) + factor = pd.Series(np.arange(13, dtype=float), index=index) + labels = assign_quantile_buckets(factor, 5) + assert labels.value_counts().sort_index().tolist() == [3, 3, 3, 2, 2] + # bucket 1 must hold the LOWEST factor values + assert labels.xs(dates[0], level=DATE).loc[symbols[0]] == 1 + assert labels.xs(dates[0], level=DATE).loc[symbols[12]] == 5 + + +def test_bucket_arithmetic_matches_array_split_for_every_size_and_count(): + """EXHAUSTIVE: the integer formula vs ``np.array_split``, every (n, q). + + ``assign_quantile_buckets`` inverts ``np.array_split`` with integer arithmetic + on grouped ranks instead of materializing the chunks. That inversion is the + single fiddliest expression in the PR (the ``base == 0`` guard, the ``n % q`` + remainder head), and an off-by-one there would silently misplace names at a + bucket boundary on every date. So check it everywhere, not on a lucky example. + """ + dates = pd.bdate_range("2024-01-02", periods=1, name=DATE) + mismatches = [] + for n in range(1, 41): + symbols = [f"{i:06d}.SZ" for i in range(n)] + index = pd.MultiIndex.from_product([dates, symbols], names=[DATE, SYMBOL]) + factor = pd.Series(np.arange(n, dtype=float), index=index) + for n_quantiles in range(2, 9): + got = dict(assign_quantile_buckets(factor, n_quantiles).xs(dates[0], level=DATE)) + expected = { + sym: i + for i, chunk in enumerate( + np.array_split(np.array(symbols, dtype=object), n_quantiles), start=1 + ) + for sym in chunk + } + if got != expected: + mismatches.append((n, n_quantiles)) + assert mismatches == [] + + +def test_fewer_names_than_buckets_leaves_high_buckets_empty(): + dates = pd.bdate_range("2024-01-02", periods=1, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(3)] + index = pd.MultiIndex.from_product([dates, symbols], names=[DATE, SYMBOL]) + factor = pd.Series([1.0, 2.0, 3.0], index=index) + labels = assign_quantile_buckets(factor, 5) + assert sorted(labels.tolist()) == [1, 2, 3] + + +def test_ties_are_broken_by_symbol_ascending(): + """Every factor value identical -> buckets follow symbol order, deterministically.""" + dates = pd.bdate_range("2024-01-02", periods=1, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(10)] + index = pd.MultiIndex.from_product([dates, symbols], names=[DATE, SYMBOL]) + factor = pd.Series(np.full(10, 3.3), index=index) + labels = assign_quantile_buckets(factor, 5).xs(dates[0], level=DATE) + assert [labels.loc[s] for s in symbols] == [1, 1, 2, 2, 3, 3, 4, 4, 5, 5] + + +def test_all_nan_and_empty_cross_sections_are_nan_not_zero(rng): + panel, factor = ragged_panel(rng) + dates = panel_dates(factor) + ir = build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + all_nan_date = dates[21] + assert math.isnan(ir.ic.loc[all_nan_date]) + assert ir.quantile_returns.loc[all_nan_date].isna().all() + assert ir.cross_section_size.loc[all_nan_date] == 0 + # a constant cross-section has no rank variance -> undefined, not 0.0 + assert math.isnan(ir.ic.loc[dates[22]]) + # a single-symbol cross-section cannot be correlated + assert math.isnan(ir.ic.loc[dates[20]]) + + +def test_infinity_is_treated_as_missing_not_as_an_extreme(rng): + panel, factor = ragged_panel(rng) + dates = panel_dates(factor) + ir = build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + labels = ir.quantile_labels + assert (dates[25], "000002.SZ") not in labels.index + + +def test_ir_build_is_deterministic(rng): + panel, factor = ragged_panel(rng) + ctx = EvalContext(price_panel=panel) + first = build_eval_ir(factor, make_spec(), make_cfg(), ctx) + second = build_eval_ir(factor, make_spec(), make_cfg(), ctx) + assert_series_equal(first.ic, second.ic) + assert_frame_equal(first.quantile_returns, second.quantile_returns) + assert_frame_equal(first.quantile_turnover, second.quantile_turnover) + + +def test_build_does_not_mutate_the_caller_s_panels(rng): + panel, factor = ragged_panel(rng) + factor_before, panel_before = factor.copy(), panel.copy() + build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + assert_series_equal(factor, factor_before) + assert_frame_equal(panel, panel_before) + + +# ========================================================================== +# 2. the IR's PIT contract +# ========================================================================== + + +def test_forward_returns_are_exactly_close_t_plus_h_over_close_t(rng): + dates = pd.bdate_range("2024-01-02", periods=12, name=DATE) + symbols = ["000001.SZ", "000002.SZ"] + panel = make_price_panel(dates, symbols, rng) + factor = pd.Series(1.0, index=panel.index) + ir = build_eval_ir( + factor, make_spec(forward_return_horizon=3), make_cfg(), EvalContext(price_panel=panel) + ) + close = panel["close"] + for i in range(len(dates) - 3): + expected = ( + close.loc[(dates[i + 3], "000001.SZ")] / close.loc[(dates[i], "000001.SZ")] - 1.0 + ) + assert ir.forward_returns.loc[(dates[i], "000001.SZ")] == pytest.approx(expected) + # the last h dates cannot have realized anything + for date in dates[-3:]: + assert ir.forward_returns.loc[(date, "000001.SZ")] != ir.forward_returns.loc[ + (date, "000001.SZ") + ] + assert pd.isna(ir.realized_date.loc[date]) + + +def test_realized_date_is_t_plus_h_on_the_return_grid(rng): + dates = pd.bdate_range("2024-01-02", periods=10, name=DATE) + panel = make_price_panel(dates, ["000001.SZ", "000002.SZ"], rng) + factor = pd.Series(1.0, index=panel.index) + ir = build_eval_ir( + factor, make_spec(forward_return_horizon=2), make_cfg(), EvalContext(price_panel=panel) + ) + assert ir.realized_date.loc[dates[0]] == dates[2] + assert ir.realized_date.loc[dates[7]] == dates[9] + assert pd.isna(ir.realized_date.loc[dates[8]]) + + +def test_settled_rebalances_excludes_the_unrealized_tail(rng): + panel, factor = ragged_panel(rng, n_dates=30, n_symbols=12) + ir = build_eval_ir( + factor, make_spec(forward_return_horizon=1), make_cfg(), EvalContext(price_panel=panel) + ) + assert ir.settled_rebalances == int(ir.ic.notna().sum()) + assert ir.settled_rebalances < ir.n_rebalances # the tail never settles + + +def test_poisoning_prices_beyond_t_plus_h_cannot_move_an_earlier_ic(rng): + """Lookahead lock by perturbation — the project's standard technique. + + Scramble every close from grid position 20 onward. No period whose forward + return realizes at or before position 19 may notice. + + Two traps this test had to avoid, both of which would make it vacuous: + * a UNIFORM scale factor leaves the cross-sectional RANKING untouched, so + rank IC is invariant to it and nothing would move even with a real leak. + Hence a per-symbol multiplier. + * for t >= 20 BOTH close[t] and close[t+h] are poisoned and the multiplier + cancels in the ratio — so only the boundary periods (t = 18, 19) can + legitimately move. The test asserts they DO, or the poison proved nothing. + """ + dates = pd.bdate_range("2024-01-02", periods=30, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(15)] + panel = make_price_panel(dates, symbols, rng) + factor = pd.Series(rng.normal(size=len(panel)), index=panel.index) + spec = make_spec(forward_return_horizon=2) + base = build_eval_ir(factor, spec, make_cfg(), EvalContext(price_panel=panel)) + + poisoned = panel.copy() + mask = poisoned.index.get_level_values(DATE) >= dates[20] + multiplier = pd.Series(rng.uniform(0.2, 5.0, len(symbols)), index=symbols) + poisoned.loc[mask, "close"] = poisoned.loc[mask, "close"] * poisoned.loc[ + mask + ].index.get_level_values(SYMBOL).map(multiplier) + after = build_eval_ir(factor, spec, make_cfg(), EvalContext(price_panel=poisoned)) + + moved_early = [ + d for d in dates[:18] + if not np.allclose(base.ic[d], after.ic[d], equal_nan=True) + ] + assert moved_early == [], "LOOKAHEAD: an IC changed from data strictly after t+h" + moved_late = sum( + 1 for d in dates[18:28] + if not np.allclose(base.ic[d], after.ic[d], equal_nan=True) + ) + assert moved_late > 0, "the poison was impotent, so the assertion above proves nothing" + + +def test_exec_to_exec_without_supplied_returns_is_a_readable_error(rng): + dates = pd.bdate_range("2024-01-02", periods=5, name=DATE) + panel = make_price_panel(dates, ["000001.SZ", "000002.SZ"], rng) + factor = pd.Series(1.0, index=panel.index) + spec = make_spec( + is_intraday=True, + return_basis="exec_to_exec", + decision_cutoff="14:50:00", + data_lag="1min", + session_open="09:30:00", + execution_model="next_minute_close", + execution_window="[14:51,14:56:59]", + ) + with pytest.raises(ValueError, match="execution-anchored"): + build_eval_ir(factor, spec, make_cfg(), EvalContext(price_panel=panel)) + + +def test_no_returns_at_all_is_a_readable_error(rng): + dates = pd.bdate_range("2024-01-02", periods=5, name=DATE) + panel = make_price_panel(dates, ["000001.SZ", "000002.SZ"], rng) + factor = pd.Series(1.0, index=panel.index) + with pytest.raises(ValueError, match="no forward returns available"): + build_eval_ir(factor, make_spec(), make_cfg(), EvalContext()) + + +def test_factor_dates_absent_from_the_price_panel_are_rejected(rng): + dates = pd.bdate_range("2024-01-02", periods=8, name=DATE) + panel = make_price_panel(dates, ["000001.SZ", "000002.SZ"], rng) + stray = pd.MultiIndex.from_tuples( + [(pd.Timestamp("2030-01-01"), "000001.SZ")], names=[DATE, SYMBOL] + ) + factor = pd.concat([pd.Series(1.0, index=panel.index), pd.Series(1.0, index=stray)]) + with pytest.raises(ValueError, match="absent from the price panel"): + build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + + +def monthly_grid_setup(rng, n_days=200): + """A factor on the FIRST trading day of each month, over a DAILY price panel.""" + dates = pd.bdate_range("2024-01-02", periods=n_days, name=DATE) + symbols = ["000001.SZ", "000002.SZ", "000003.SZ", "000004.SZ"] + panel = make_price_panel(dates, symbols, rng) + first_of_month = pd.DatetimeIndex( + sorted(pd.Series(dates).groupby([dates.year, dates.month]).min()), name=DATE + ) + index = pd.MultiIndex.from_product([first_of_month, symbols], names=[DATE, SYMBOL]) + factor = pd.Series(rng.normal(size=len(index)), index=index) + return panel, factor, dates, first_of_month, symbols + + +def test_horizon_counts_evaluation_periods_not_price_panel_rows(rng): + """``FactorSpec.forward_return_horizon`` is "in evaluation periods" (frozen contract). + + A monthly factor over a daily price panel with h=1 must be scored on its + next-PERIOD return. Delegating ``periods=(1,)`` to the whole price panel would + silently score it on the next TRADING DAY instead — a completely different + (and much easier) claim, on a horizon nobody asked about. + """ + panel, factor, dates, months, symbols = monthly_grid_setup(rng) + ir = build_eval_ir( + factor, make_spec(forward_return_horizon=1), make_cfg(rebalance="monthly"), + EvalContext(price_panel=panel), + ) + assert ir.median_period_gap_days > 25 # the grid really is sparse + close = panel["close"] + first, second = ir.dates[0], ir.dates[1] + + next_period = close.loc[(second, symbols[0])] / close.loc[(first, symbols[0])] - 1.0 + next_day = ( + close.loc[(dates[dates.get_loc(first) + 1], symbols[0])] + / close.loc[(first, symbols[0])] + - 1.0 + ) + assert ir.forward_returns.loc[(first, symbols[0])] == pytest.approx(next_period) + assert ir.forward_returns.loc[(first, symbols[0])] != pytest.approx(next_day) + # the realized date is the next EVALUATION PERIOD, not the next trading day + assert ir.realized_date.loc[first] == second + assert "evaluation periods" in ir.forward_return_source + assert "restricted" in ir.forward_return_source + + +def test_dense_grid_is_unaffected_by_the_evaluation_period_resolution(rng): + """When F's grid IS the price grid, h=trading days=periods — nothing changed.""" + dates = pd.bdate_range("2024-01-02", periods=20, name=DATE) + panel = make_price_panel(dates, ["000001.SZ", "000002.SZ"], rng) + factor = pd.Series(rng.normal(size=len(panel)), index=panel.index) + ir = build_eval_ir( + factor, make_spec(forward_return_horizon=3), make_cfg(), + EvalContext(price_panel=panel), + ) + close = panel["close"] + expected = close.loc[(dates[3], "000001.SZ")] / close.loc[(dates[0], "000001.SZ")] - 1.0 + assert ir.forward_returns.loc[(dates[0], "000001.SZ")] == pytest.approx(expected) + assert "restricted" not in ir.forward_return_source + + +def test_duplicate_panel_rows_are_rejected(rng): + dates = pd.bdate_range("2024-01-02", periods=4, name=DATE) + panel = make_price_panel(dates, ["000001.SZ"], rng) + factor = pd.concat([pd.Series(1.0, index=panel.index)] * 2) + with pytest.raises(ValueError, match="duplicate"): + build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + + +def test_ambiguous_multi_column_frame_is_rejected(rng): + dates = pd.bdate_range("2024-01-02", periods=4, name=DATE) + panel = make_price_panel(dates, ["000001.SZ", "000002.SZ"], rng) + frame = pd.DataFrame({"a": 1.0, "b": 2.0}, index=panel.index) + with pytest.raises(ValueError, match="none is named"): + build_eval_ir(frame, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + + +# ========================================================================== +# 3. turnover / stats primitives +# ========================================================================== + + +def test_turnover_charges_the_first_period_and_a_full_rotation(): + dates = pd.bdate_range("2024-01-02", periods=2, name=DATE) + symbols = ["000001.SZ", "000002.SZ", "000003.SZ", "000004.SZ"] + index = pd.MultiIndex.from_product([dates, symbols], names=[DATE, SYMBOL]) + # day 1 ranks 1,2,3,4 -> day 2 exactly reversed: bucket 1 and 2 fully swap + factor = pd.Series([1.0, 2.0, 3.0, 4.0, 4.0, 3.0, 2.0, 1.0], index=index) + labels = assign_quantile_buckets(factor, 2) + turnover = quantile_turnover(labels, 2, pd.Index(dates, name=DATE)) + # establishing an equal-weight book of 2 names is sum|w| = 1.0 + assert turnover.loc[dates[0], 1] == pytest.approx(1.0) + # a complete rotation sells 1.0 and buys 1.0 + assert turnover.loc[dates[1], 1] == pytest.approx(2.0) + + +def test_turnover_is_zero_when_membership_does_not_change(): + dates = pd.bdate_range("2024-01-02", periods=3, name=DATE) + symbols = ["000001.SZ", "000002.SZ", "000003.SZ", "000004.SZ"] + index = pd.MultiIndex.from_product([dates, symbols], names=[DATE, SYMBOL]) + factor = pd.Series([1.0, 2.0, 3.0, 4.0] * 3, index=index) + turnover = quantile_turnover( + assign_quantile_buckets(factor, 2), 2, pd.Index(dates, name=DATE) + ) + assert turnover.loc[dates[1], 1] == pytest.approx(0.0) + assert turnover.loc[dates[2], 2] == pytest.approx(0.0) + + +def test_newey_west_t_is_smaller_than_the_naive_t_on_an_autocorrelated_series(): + """The whole reason design §A demands it: the IID t overstates significance.""" + rng = np.random.default_rng(7) + n = 400 + x = np.zeros(n) + for i in range(1, n): + x[i] = 0.85 * x[i - 1] + rng.normal(0, 0.02) + series = pd.Series(x + 0.05, index=pd.RangeIndex(n)) + result = newey_west_t(series) + assert result["lags"] >= 1 + assert abs(result["t"]) < abs(result["t_iid"]) + # not a rounding difference: the naive t is materially inflated here + assert abs(result["t"]) < 0.6 * abs(result["t_iid"]) + + +def test_newey_west_t_on_white_noise_is_close_to_the_naive_t(): + rng = np.random.default_rng(11) + series = pd.Series(rng.normal(0.01, 0.1, 500)) + result = newey_west_t(series) + assert result["t"] == pytest.approx(result["t_iid"], rel=0.35) + + +def reference_newey_west_t(series: pd.Series, lags: int) -> float: + """Deliberately dumb NW-t: an explicit double loop over (t, j) pairs. + + A pair contributes ONLY when both endpoints are observed — the definition of + "the gap breaks the lag pair". Obviously correct, obviously slow. + """ + values = pd.Series(series, dtype=float).to_numpy(dtype=float) + total = len(values) + valid = np.isfinite(values) + n = int(valid.sum()) + mean = float(values[valid].mean()) + gamma = [] + for j in range(lags + 1): + acc = 0.0 + for t in range(j, total): + if valid[t] and valid[t - j]: + acc += (values[t] - mean) * (values[t - j] - mean) + gamma.append(acc / n) + variance = gamma[0] + 2.0 * sum( + (1.0 - j / (lags + 1.0)) * gamma[j] for j in range(1, lags + 1) + ) + return mean / math.sqrt(variance / n) + + +def test_newey_west_t_breaks_lag_pairs_across_holes_instead_of_bridging_them(): + """An IC series HAS holes (degenerate cross-sections yield NaN). + + Dropping them and then lagging pairs observations that are j SURVIVING ROWS + apart while calling them j PERIODS apart — a bridged gap, i.e. an + autocovariance that does not exist. The statistic whose entire job is to stop + the IID t from overstating significance must not itself be quietly wrong. + """ + rng = np.random.default_rng(23) + n = 300 + x = np.zeros(n) + for i in range(1, n): + x[i] = 0.8 * x[i - 1] + rng.normal(0, 0.02) + series = pd.Series(x + 0.04) + # punch interior holes (not just a tail): exactly the degenerate-cross-section + # pattern a real IC series shows. + for i in (17, 18, 60, 61, 62, 140, 201, 202, 250): + series.iloc[i] = np.nan + assert series.iloc[1:-1].isna().sum() == 9 # holes really are INTERIOR + + got = newey_west_t(series, lags=4) + assert got["t"] == pytest.approx(reference_newey_west_t(series, 4), rel=1e-12) + assert got["n"] == 300 - 9 + assert got["n_dropped"] == 9 + + # ... and the drop-then-lag version really does differ, or this proves nothing + bridged = newey_west_t(series.dropna().reset_index(drop=True), lags=4) + assert not np.isclose(got["t"], bridged["t"], rtol=1e-6) + + +def test_newey_west_t_without_holes_is_unchanged_by_the_gap_handling(): + """A hole-free series must give exactly what it always gave.""" + rng = np.random.default_rng(29) + n = 200 + x = np.zeros(n) + for i in range(1, n): + x[i] = 0.7 * x[i - 1] + rng.normal(0, 0.03) + series = pd.Series(x + 0.05) + got = newey_west_t(series, lags=3) + assert got["t"] == pytest.approx(reference_newey_west_t(series, 3), rel=1e-12) + assert got["n_dropped"] == 0 + # with no holes, zero-filling is a no-op: identical to the dropna() path + assert got["t"] == pytest.approx(newey_west_t(series.dropna(), lags=3)["t"], rel=1e-12) + + +def test_newey_west_t_ignores_a_trailing_nan_tail_like_a_settled_ic_series(): + """The last h periods of a real IC series never settle -> a NaN TAIL. + + A tail has no interior pair to break, so tail-only NaN must not change the + estimate at all relative to simply trimming it. + """ + rng = np.random.default_rng(31) + values = pd.Series(rng.normal(0.02, 0.1, 120)) + with_tail = pd.concat([values, pd.Series([np.nan] * 5)], ignore_index=True) + assert newey_west_t(with_tail, lags=3)["t"] == pytest.approx( + newey_west_t(values, lags=3)["t"], rel=1e-12 + ) + + +def test_newey_west_t_degenerate_inputs_are_nan_not_zero(): + assert math.isnan(newey_west_t(pd.Series([], dtype=float))["t"]) + assert math.isnan(newey_west_t(pd.Series([1.0]))["t"]) + constant = newey_west_t(pd.Series([2.0] * 30)) + assert math.isnan(constant["t"]) # zero variance -> undefined, never +inf + + +# -------------------------------------------------------------------------- +# effective_sample_size — gate part A (design §6, v0.3) +# -------------------------------------------------------------------------- + + +def _ar1(rho: float, n: int, seed: int) -> pd.Series: + rng = np.random.default_rng(seed) + x = np.zeros(n) + noise = rng.normal(size=n) + for t in range(1, n): + x[t] = rho * x[t - 1] + noise[t] + return pd.Series(x) + + +def test_a_long_but_heavily_autocorrelated_daily_ic_series_has_few_effective_samples(): + """⭐ THE test the three-part gate exists for. + + ~500 daily IC points with rho_1 ~ 0.95 are NOT 500 pieces of evidence: the + AR(1) truth is N_eff = N*(1-rho)/(1+rho) ~ 12.8. The raw count sails past any + raw floor; the effective count does not. + + ⚠️ ASSERTED OVER MANY DRAWS ON PURPOSE — the estimator is NOISY at this rho, + and a single seed would be cherry-picking. Measured over 200 draws: + median 17.9 (biased UP from the 12.8 truth — rho-hat is downward-biased and + the positive-sequence truncation stops early), range 5.9..35.6, and + P(N_eff < 24) = 80%. So the DEFAULT gate catches the typical such series but + NOT every draw; what is invariant is the order of magnitude (P(N_eff<100) = + 100%, i.e. always a >5x cut from 500). Both facts are asserted; neither is + tuned away. At rho=0.99 it is decisive (see the next test). + """ + values = np.array( + [float(effective_sample_size(_ar1(0.95, 500, seed=s))["n_eff"]) for s in range(40)] + ) + # invariant: never anywhere near the raw count. + assert values.max() < 100.0 + # the typical draw is gated by the default min_effective_samples=24 ... + assert np.median(values) < 24.0 + # ... and most draws are, though NOT all (~80% at this rho). + assert (values < 24.0).mean() >= 0.6 + + got = effective_sample_size(_ar1(0.95, 500, seed=0)) + assert got["n"] == 500 + # it looked PAST the Newey-West floor, which alone truncates at ~5 lags and + # reports a ~4x too permissive N_eff (~53 vs a ~13 truth). + assert got["lags"] > got["lags_nw"] + assert got["sum_rho"] > 1.0 + + # end to end: the TYPICAL such series gates the Predictive axis on part A + # (no OOS/book/exec facts either -> the deployment label is INSUFFICIENT-DATA). + result = decide_verdict( + VerdictInputs( + expected_ic_sign=+1, + settled_rebalances=500, + effective_samples=float(np.median(values)), + span_days=700.0, + ) + ) + assert result.verdict == INSUFFICIENT_DATA + assert result.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any(r.startswith("effective samples (A)") for r in result.predictive.reasons) + + +def test_a_near_unit_root_ic_series_is_gated_on_every_draw(): + """rho=0.99: the truth is N_eff ~ 2.5 and the gate catches it 99.5% of the + time (200 draws). The noise that makes rho=0.95 a ~80% catch does not rescue + a series this persistent.""" + values = np.array( + [float(effective_sample_size(_ar1(0.99, 500, seed=s))["n_eff"]) for s in range(40)] + ) + assert values.max() < 24.0 + + +def test_an_iid_series_keeps_essentially_all_of_its_samples(): + """The other end: independent observations must NOT be penalized.""" + for seed in range(4): + series = pd.Series(np.random.default_rng(seed).normal(size=500)) + got = effective_sample_size(series) + assert got["n_eff"] > 0.6 * 500 # ~N, up to rho-hat noise + assert got["n_eff"] <= 500 # never MORE than N + + +def test_a_perfectly_correlated_series_carries_one_sample(): + """A constant series holds exactly one distinct value: N_eff = 1, not 0/0.""" + got = effective_sample_size(pd.Series(np.ones(100))) + assert got["n_eff"] == 1.0 + assert "constant" in str(got["status"]) + assert math.isfinite(got["n_eff"]) + + +def test_effective_sample_size_is_monotone_in_the_autocorrelation(): + """More persistence must never buy MORE effective samples.""" + values = [effective_sample_size(_ar1(r, 400, seed=5))["n_eff"] for r in (0.0, 0.5, 0.9, 0.99)] + assert values == sorted(values, reverse=True) + + +def test_effective_sample_size_never_emits_nan_inf_or_a_negative(): + """Every guarded path still yields a finite, non-negative, <= N number.""" + cases = { + "empty": pd.Series([], dtype=float), + "one": pd.Series([0.4]), + "all_nan": pd.Series([np.nan, np.nan, np.nan]), + "constant": pd.Series(np.ones(50)), + "two_points": pd.Series([0.1, -0.1]), + "alternating": pd.Series([1.0, -1.0] * 50), # extreme anti-correlation + "with_inf": pd.Series([1.0, np.inf, -1.0, 2.0, 0.5] * 10), + "ar1_099": _ar1(0.99, 300, seed=3), + } + for name, series in cases.items(): + got = effective_sample_size(series) + value = got["n_eff"] + assert isinstance(value, float), name + assert math.isfinite(value), name + assert value >= 0.0, name + assert value <= got["n"] or got["n"] == 0, name + + +def test_an_anti_correlated_series_is_clamped_to_n_never_credited_beyond_it(): + """A negative 1+2*sum(rho) must not become a negative or an inflated N_eff. + + An anti-correlated series is 'at least as informative as i.i.d.'; we credit + exactly i.i.d. and no more, because a noisy rho-hat is precisely what would + otherwise manufacture the evidence a short run does not have. + """ + got = effective_sample_size(pd.Series([1.0, -1.0] * 50)) + assert got["denominator"] <= 0 or got["n_eff"] == got["n"] + assert got["n_eff"] == 100.0 # == N, clamped + assert math.isfinite(got["n_eff"]) + + +def test_effective_sample_size_breaks_lag_pairs_across_holes_like_the_nw_t(): + """Same gap rule as newey_west_t: a hole DROPS the pair, never bridges it.""" + series = _ar1(0.9, 300, seed=7) + holed = series.copy() + for i in (17, 18, 60, 61, 140, 201, 250): + holed.iloc[i] = np.nan + got = effective_sample_size(holed) + assert got["n"] == 300 - 7 # holes are not counted as observations + # dropping-then-lagging (the bridge) genuinely differs -> the rule has teeth + bridged = effective_sample_size(holed.dropna().reset_index(drop=True)) + assert not np.isclose(got["n_eff"], bridged["n_eff"], rtol=1e-6) + + +def test_effective_sample_size_shares_the_newey_west_lag_floor(): + """The two statistics must not disagree about the autocorrelation structure: + the ESS window is anchored at the NW-t's bandwidth and only ever EXTENDS it.""" + for n in (30, 120, 500): + got = effective_sample_size(pd.Series(np.random.default_rng(1).normal(size=n))) + assert got["lags_nw"] == newey_west_lag(n) + assert got["lags"] >= got["lags_nw"] + # an explicit lags= disables the extension: the caller chose the truncation. + fixed = effective_sample_size(_ar1(0.95, 500, seed=11), lags=5) + assert fixed["lags"] == 5.0 and fixed["lags_nw"] == 5.0 + + +def test_hypothesis_win_rate_is_relative_to_the_expected_sign(): + series = pd.Series([0.1, 0.2, -0.3, float("nan"), 0.0]) + assert hypothesis_win_rate(series, 1) == pytest.approx(0.5) # 2 of 4 finite + assert hypothesis_win_rate(series, -1) == pytest.approx(0.25) + assert math.isnan(hypothesis_win_rate(pd.Series([], dtype=float), 1)) + + +def test_half_life_is_defined_only_for_a_decaying_signal(): + assert half_life(0.5) == pytest.approx(1.0) + assert math.isnan(half_life(-0.4)) # flips rather than decays + assert math.isnan(half_life(1.0)) # never decays + assert math.isnan(half_life(float("nan"))) + + +def test_sortino_is_nan_without_downside(): + # no negative period -> the downside deviation is 0 and the ratio is undefined. + # +inf would render as a spectacular fake, so NaN it is. + assert math.isnan(sortino(pd.Series([0.01, 0.02, 0.03]), 252)) + assert math.isnan(sortino(pd.Series([0.01]), 252)) # too short to measure + + +def test_sortino_penalizes_only_the_downside(): + """Same mean, same vol, but one series has its dispersion on the upside.""" + downside_heavy = pd.Series([0.02, -0.04, 0.02, 0.04, -0.04, 0.04] * 4) + upside_heavy = pd.Series([0.01, 0.01, 0.01, 0.01, -0.005, 0.06] * 4) + assert sortino(downside_heavy, 252) < sortino(upside_heavy, 252) + assert sortino(pd.Series([-0.01, -0.02, 0.005, -0.03] * 4), 252) < 0 + + +# ========================================================================== +# 4. the sections +# ========================================================================== + + +def planted_signal_setup(rng, n_dates=180, n_symbols=40, strength=0.02, sign=1): + """A panel where tomorrow's return genuinely follows today's factor.""" + dates = pd.bdate_range("2024-01-02", periods=n_dates, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(n_symbols)] + raw = rng.normal(size=(n_dates, n_symbols)) + # the planted edge: return[t] is driven by factor[t-1] + signal = np.zeros_like(raw) + signal[1:] = sign * strength * raw[:-1] + panel = make_price_panel(dates, symbols, rng, signal=signal) + factor = pd.DataFrame(raw, index=dates, columns=symbols).stack() + factor.index.names = [DATE, SYMBOL] + return panel, factor.sort_index().rename("synth_factor") + + +def incremental_book(rng, factor): + """A pure-NOISE known-factor book the factor is genuinely incremental to. + + Independent of the factor, so residualizing the factor on it leaves the signal + essentially intact -> the Incremental axis PASSes. (Contrast ``redundant_setup``, + which reconstructs the factor -> the residual is ~ 0 -> the axis FAILs.) + """ + return pd.DataFrame( + {"noise_anchor": pd.Series(rng.normal(size=len(factor)), index=factor.index)} + ) + + +def redundant_setup(rng, n_dates=300, n_symbols=40, strength=0.05, noise_scale=0.5): + """A factor that DUPLICATES a supplied known factor: strong RAW IC, ~0 residual. + + Returns follow the ANCHOR, and the factor is ``anchor + noise`` — so the factor + predicts returns (strong raw IC, sign-consistent out-of-sample) yet residualizing + it on the anchor book leaves ONLY the noise, which is independent of returns. Its + incremental IC is therefore ~ 0: the Incremental axis FAILs and the factor is + rejected DESPITE its perfect predictive/tradable story. This is the value_ep-twin + scenario the three-axis verdict exists to catch.""" + dates = pd.bdate_range("2024-01-02", periods=n_dates, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(n_symbols)] + anchor_raw = rng.normal(size=(n_dates, n_symbols)) + signal = np.zeros_like(anchor_raw) + signal[1:] = strength * anchor_raw[:-1] # returns follow the ANCHOR, not the noise + panel = make_price_panel(dates, symbols, rng, signal=signal) + noise = rng.normal(size=(n_dates, n_symbols)) * noise_scale + factor = pd.DataFrame(anchor_raw + noise, index=dates, columns=symbols).stack() + factor.index.names = [DATE, SYMBOL] + anchor = pd.DataFrame(anchor_raw, index=dates, columns=symbols).stack() + anchor.index.names = [DATE, SYMBOL] + book = pd.DataFrame({"anchor": anchor.sort_index()}) + return panel, factor.sort_index().rename("synth_factor"), book + + +def test_all_eight_sections_are_present_and_ordered(rng): + panel, factor = planted_signal_setup(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + assert [s.name for s in report.sections] == list(MANDATORY_SECTIONS) + report.validate_all_mandatory_present() + + +def test_planted_signal_is_detected_with_the_expected_sign(rng): + panel, factor = planted_signal_setup(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + predictive = report.by_name()["predictive_power"] + assert isinstance(predictive, Section) + assert predictive.payload["ic_mean"] > 0.10 + assert predictive.payload["ic_win_rate"] > 0.70 + assert predictive.payload["ic_ir"] > 0.30 + returns = report.by_name()["return_risk"] + assert returns.payload["monotonicity_spearman"] == pytest.approx(1.0) + assert returns.payload["net_long_short_by_cost"][1.0] > 0 + + +def test_negative_hypothesis_factor_reads_in_its_own_direction(rng): + """A -1 factor (low-vol style) must not be punished for a negative Spearman.""" + panel, factor = planted_signal_setup(rng, sign=-1) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(expected_ic_sign=-1), make_cfg(), EvalContext(price_panel=panel) + ) + predictive = report.by_name()["predictive_power"] + returns = report.by_name()["return_risk"] + assert predictive.payload["ic_mean"] < 0 # raw IC is negative... + assert predictive.payload["ic_win_rate"] > 0.70 # ...but that IS the hypothesis + assert returns.payload["monotonicity_spearman"] == pytest.approx(-1.0) # RAW + assert returns.payload["net_long_short_by_cost"][1.0] < 0 # RAW + + +def test_cost_scenarios_only_move_the_cost_line(rng): + panel, factor = planted_signal_setup(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + returns = report.by_name()["return_risk"] + by_cost = returns.payload["net_long_short_by_cost"] + # the SAME trades at k x the fee: strictly monotone degradation, gross fixed + assert by_cost[1.0] > by_cost[2.0] > by_cost[4.0] + stability = report.by_name()["stability_cost"] + drag = stability.payload["mean_cost_per_period_by_scenario"] + assert drag[2.0] == pytest.approx(2 * drag[1.0]) + assert drag[4.0] == pytest.approx(4 * drag[1.0]) + assert stability.payload["extra_cost_vs_base_by_scenario"][1.0] == pytest.approx(0.0) + + +def test_return_risk_note_labels_the_synthetic_leg_difference(rng): + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + note = report.by_name()["return_risk"].note + assert "SYNTHETIC LONG-ONLY LEG DIFFERENCE" in note + assert "NOT a dollar-neutral executed portfolio" in note + + +def test_purity_is_skipped_without_anchors_and_says_why(rng): + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + purity = report.by_name()["purity"] + assert isinstance(purity, Skipped) + assert "known_factors was not supplied" in purity.reason + + +def test_purity_reports_correlation_and_orthogonalized_ic_when_anchored(rng): + panel, factor = planted_signal_setup(rng, n_dates=90, n_symbols=30) + # "near_twin" explains almost all of the factor -> the orthogonalized IC must + # collapse. "noise" explains none of it -> the signal must survive. + anchors = pd.DataFrame( + { + "near_twin": factor + pd.Series( + rng.normal(scale=0.05, size=len(factor)), index=factor.index + ), + "noise": pd.Series(rng.normal(size=len(factor)), index=factor.index), + } + ) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), + EvalContext(price_panel=panel, known_factors=anchors), + ) + purity = report.by_name()["purity"] + assert isinstance(purity, Section) + raw_ic = purity.payload["ic_mean_raw"] + assert purity.payload["mean_rank_corr_with_anchor"]["near_twin"] > 0.95 + assert abs(purity.payload["mean_rank_corr_with_anchor"]["noise"]) < 0.1 + # removing the near-twin removes the signal; removing noise does not + assert abs(purity.payload["ic_mean_orthogonalized_vs_anchor"]["near_twin"]) < 0.3 * raw_ic + assert purity.payload["ic_mean_orthogonalized_vs_anchor"]["noise"] > 0.7 * raw_ic + assert purity.payload["vif"] is None + assert "NOT COMPUTED" in purity.payload["vif_status"] + + +def test_orthogonalizing_against_a_perfect_twin_is_nan_not_zero(rng): + """A residual of exactly zero has no rank variance: the IC is UNDEFINED. + + NaN is the honest answer — a 0.0 would read as "the signal survived + orthogonalization and turned out to be worthless", which is a different claim. + """ + panel, factor = planted_signal_setup(rng, n_dates=60, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), + EvalContext(price_panel=panel, known_factors=pd.DataFrame({"twin": factor})), + ) + purity = report.by_name()["purity"] + assert purity.payload["mean_rank_corr_with_anchor"]["twin"] == pytest.approx(1.0) + assert math.isnan(purity.payload["ic_mean_orthogonalized_vs_anchor"]["twin"]) + + +def test_execution_capacity_is_skipped_with_a_precise_reason(rng): + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + section = report.by_name()["execution_capacity"] + assert isinstance(section, Skipped) + assert "NOT WIRED" in section.reason + assert "cannot reach the Adopt verdict" in section.reason + + +def test_execution_capacity_passes_through_measured_facts(rng): + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), + EvalContext( + price_panel=panel, + execution_capacity={"tradable": True, "capacity_sufficient": False, + "capacity_ratio_median": 1.104}, + ), + ) + section = report.by_name()["execution_capacity"] + assert isinstance(section, Section) + assert section.payload["tradable"] is True + assert section.payload["capacity_sufficient"] is False + assert "MEASURED OUTSIDE this evaluator" in section.payload["source"] + + +def test_execution_capacity_marks_a_missing_flag_as_unknown(rng): + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), + EvalContext(price_panel=panel, execution_capacity={"tradable": True}), + ) + section = report.by_name()["execution_capacity"] + assert "UNKNOWN" in section.payload["capacity_sufficient_status"] + + +def test_data_coverage_reports_the_sample_and_the_dropped_names(rng): + panel, factor = ragged_panel(rng) + declared = [f"{i:06d}.SZ" for i in range(30)] # 7 names never reach the panel + report = StandardFactorEvaluator().evaluate( + factor, make_spec(min_history_bars=2), make_cfg(), + EvalContext(price_panel=panel, universe_symbols=tuple(declared)), + ) + coverage = report.by_name()["data_coverage"] + assert coverage.payload["settled_rebalances"] == int( + report.by_name()["predictive_power"].payload["ic_periods_finite"] + ) + assert coverage.payload["symbols_evaluated"] == 23 + assert coverage.payload["dropped_symbols_count"] == 7 + assert coverage.payload["universe_symbols_declared"] == 30 + assert coverage.payload["factor_nan_rate"] > 0 + assert coverage.payload["warmup_rows_excluded"] == 2 * 23 + + +def test_data_coverage_says_when_the_universe_was_not_declared(rng): + panel, factor = ragged_panel(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + coverage = report.by_name()["data_coverage"] + assert coverage.payload["dropped_symbols_count"] is None + assert "NOT ASSESSED" in coverage.payload["dropped_symbols_status"] + + +def test_daily_panel_declared_monthly_is_rejected_not_silently_counted(rng): + """The sample gate must not be satisfiable by a sample that does not exist. + + A daily panel declared 'monthly' would otherwise report ~250 settled + rebalances and walk straight through min_rebalances=24 — the gate whose entire + job is deciding INSUFFICIENT-DATA. Disclosure is not enough for a GATE. + """ + panel, factor = planted_signal_setup(rng, n_dates=250, n_symbols=20) + with pytest.raises(ValueError, match="declares evaluation periods") as excinfo: + StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(rebalance="monthly"), + EvalContext(price_panel=panel), + ) + message = str(excinfo.value) + assert "'monthly'" in message # names the declared frequency + assert "median 1.00 days apart" in message # names the DETECTED one + assert "min_rebalances" in message # names the gate it would defeat + assert "does not resample" in message + + +def test_min_rebalances_gates_on_the_real_count_of_a_matching_grid(rng): + """A truly monthly grid yields ~monthly periods -> INSUFFICIENT-DATA, honestly.""" + panel, factor, _, months, _ = monthly_grid_setup(rng, n_days=250) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(rebalance="monthly"), EvalContext(price_panel=panel) + ) + coverage = report.by_name()["data_coverage"] + # ~12 months of data, NOT the ~250 rows the daily price panel carries + assert coverage.payload["evaluation_periods"] == len(months) + assert coverage.payload["settled_rebalances"] < 24 + assert coverage.payload["rebalance_grid_check"].startswith("OK") + # and the gate now bites on the REAL sample size + assert report.require_verdict().verdict == INSUFFICIENT_DATA + + +def test_a_matching_daily_grid_passes_the_check(rng): + panel, factor = planted_signal_setup(rng, n_dates=60, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(rebalance="daily"), EvalContext(price_panel=panel) + ) + coverage = report.by_name()["data_coverage"] + assert coverage.payload["rebalance_grid_check"].startswith("OK") + assert coverage.payload["median_period_gap_days"] == pytest.approx(1.0) + + +@pytest.mark.parametrize( + ("label", "should_raise"), + [("daily", True), ("weekly", True), ("monthly", False), ("quarterly", True)], +) +def test_rebalance_grid_check_accepts_only_the_matching_declaration(rng, label, should_raise): + """A ~monthly grid is accepted as 'monthly' and rejected as anything else.""" + panel, factor, _, _, _ = monthly_grid_setup(rng, n_days=250) + ctx = EvalContext(price_panel=panel) + if should_raise: + with pytest.raises(ValueError, match="declares evaluation periods"): + build_eval_ir(factor, make_spec(), make_cfg(rebalance=label), ctx) + else: + ir = build_eval_ir(factor, make_spec(), make_cfg(rebalance=label), ctx) + assert ir.rebalance_grid_check.startswith("OK") + + +def test_an_unrecognized_rebalance_label_is_disclosed_as_unverified(rng): + """A minute-frequency study has no expected spacing here — say so, don't guess. + + The declaration cannot be checked, so the report must not imply it was. + """ + panel, factor = planted_signal_setup(rng, n_dates=60, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(rebalance="5min"), EvalContext(price_panel=panel) + ) + coverage = report.by_name()["data_coverage"] + check = coverage.payload["rebalance_grid_check"] + assert check.startswith("NOT CHECKED") + assert "UNVERIFIED" in check + # and the annualization fallback is disclosed too, not silently assumed + returns = report.by_name()["return_risk"] + assert "DEFAULT 252" in returns.payload["periods_per_year_basis"] + + +def test_grid_check_is_not_checked_for_a_single_period(rng): + dates = pd.bdate_range("2024-01-02", periods=1, name=DATE) + panel = make_price_panel(dates, ["000001.SZ", "000002.SZ"], rng) + factor = pd.Series(rng.normal(size=len(panel)), index=panel.index) + ir = build_eval_ir( + factor, make_spec(), make_cfg(rebalance="monthly"), EvalContext(price_panel=panel) + ) + assert ir.rebalance_grid_check.startswith("NOT CHECKED") + + +def test_h_on_the_factor_grid_still_holds_under_the_rebalance_check(rng): + """The two fixes must coexist: a monthly grid passes the check AND h=1 period.""" + panel, factor, dates, months, symbols = monthly_grid_setup(rng) + ir = build_eval_ir( + factor, make_spec(forward_return_horizon=1), make_cfg(rebalance="monthly"), + EvalContext(price_panel=panel), + ) + assert ir.rebalance_grid_check.startswith("OK") + close = panel["close"] + first, second = ir.dates[0], ir.dates[1] + expected = close.loc[(second, symbols[0])] / close.loc[(first, symbols[0])] - 1.0 + assert ir.forward_returns.loc[(first, symbols[0])] == pytest.approx(expected) + assert ir.realized_date.loc[first] == second + + +def test_caveats_carry_the_honesty_flags(rng): + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), + make_cfg( + is_exploratory=True, post_hoc_selected=True, tuned=True, + universe_is_pit=False, n_factors_screened=11, + ), + EvalContext(price_panel=panel), + ) + caveats = report.by_name()["caveats"].payload + joined = " ".join(caveats["caveats"]) + assert "EXPLORATORY" in joined + assert "POST-HOC SELECTED" in joined + assert "TUNED" in joined + assert "NON-PIT UNIVERSE" in joined + assert "NO OOS SPLIT" in joined + assert caveats["bonferroni_alpha_for_5pct_family"] == pytest.approx(0.05 / 11) + + +def test_caveats_flag_an_undeclared_multiple_testing_background(rng): + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + note = report.by_name()["caveats"].payload["multiple_testing_note"] + assert "UNDECLARED" in note + + +# ========================================================================== +# 5. OOS & the verdict (end to end) +# ========================================================================== + + +def test_oos_is_skipped_without_a_split_and_says_so(rng): + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + section = report.by_name()["oos_generalization"] + assert isinstance(section, Skipped) + assert "cfg.oos_split not set" in section.reason + assert "no OOS evidence" in section.reason + + +def test_oos_splits_by_the_realized_date_not_the_signal_date(rng): + panel, factor = planted_signal_setup(rng, n_dates=120, n_symbols=25) + dates = panel_dates(factor) + split = dates[60] + report = StandardFactorEvaluator().evaluate( + factor, make_spec(forward_return_horizon=5), make_cfg(oos_split=str(split.date())), + EvalContext(price_panel=panel), + ) + section = report.by_name()["oos_generalization"] + assert section.payload["split_basis"].startswith("realized date") + # h=5 and the split sits at grid position 60. A period is train only if its + # return has ALREADY been realized by the split, i.e. t+5 < split, i.e. t < 55. + assert section.payload["train_periods_settled"] == 55 + # ... and NOT 60, which is what slicing by the SIGNAL date would give. That + # off-by-h is precisely the P3-3 bug: it let 5 periods of post-split return + # leak into the train segment. + assert section.payload["train_periods_settled"] != 60 + assert "realized" in section.note.lower() + + +def test_oos_sign_consistency_on_a_genuine_signal(rng): + panel, factor = planted_signal_setup(rng, n_dates=300, n_symbols=40) + dates = panel_dates(factor) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(oos_split=str(dates[130].date())), + EvalContext(price_panel=panel), + ) + section = report.by_name()["oos_generalization"] + assert section.payload["oos_available"] is True + assert section.payload["sign_consistent"] is True + assert section.payload["sign_flipped"] is False + assert section.payload["monotonicity_reversed"] is False + assert section.payload["independent_cells_evaluated"] == 0 + + +def test_oos_sign_flip_is_detected_and_hard_rejects(rng): + """The project's signature failure: in-sample positive, out-of-sample negative.""" + n_dates, n_symbols = 300, 40 + dates = pd.bdate_range("2024-01-02", periods=n_dates, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(n_symbols)] + raw = rng.normal(size=(n_dates, n_symbols)) + signal = np.zeros_like(raw) + half = n_dates // 2 + signal[1:half] = 0.03 * raw[: half - 1] # train: the hypothesis holds + signal[half:] = -0.03 * raw[half - 1 : -1] # test: it reverses + panel = make_price_panel(dates, symbols, rng, signal=signal) + factor = pd.DataFrame(raw, index=dates, columns=symbols).stack() + factor.index.names = [DATE, SYMBOL] + factor = factor.sort_index() + + # DEFAULT thresholds — no gate-opening fixture. Since v0.4 the hard Rejects + # are decided BEFORE the sample gate, so the flip this run MEASURED is + # reported as such even though a regime-flip IC series carries only ~3 + # effective samples (pinned in its own test below). The sample gate guards + # POSITIVE claims; a visible failure is not one. + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(oos_split=str(dates[half].date())), + EvalContext(price_panel=panel), + ) + section = report.by_name()["oos_generalization"] + assert section.payload["train_ic_mean"] > 0 + assert section.payload["test_ic_mean"] < 0 + assert section.payload["sign_flipped"] is True + assert section.payload["sign_consistent"] is False + assert report.require_verdict().verdict == REJECT + assert any("flipped" in r for r in report.require_verdict().reasons) + + +def test_a_regime_flip_ic_series_carries_almost_no_independent_evidence(rng): + """The N_eff measurement itself, pinned deliberately (it is CORRECT). + + An IC series that is positive for one regime and negative for the next is a + STEP FUNCTION: rho-hat stays positive for ~100 lags, so N_eff collapses to ~3 + out of ~300 raw periods. That is not a defect — two regimes ARE about two + independent observations, and "the factor flipped" is then indistinguishable + from "we saw two draws". The ESTIMATOR is doing its job and this test keeps + pinning it. + + What the v0.4 ruling changed is what the VERDICT does with it. Under v0.3 the + gate ran first, so this run drew no conclusion at all — the tool refused to + say "bad" about the project's signature failure mode (I5e / P3-3 / P3-4). + The gate exists to stop OVERCLAIMING, so it now blocks only the POSITIVE + claims: a measured flip is a NEGATIVE finding and rejects on any sample size. + The thin N_eff below is exactly why this run would never have been allowed an + Adopt or a Watch. + """ + n_dates, n_symbols = 300, 40 + dates = pd.bdate_range("2024-01-02", periods=n_dates, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(n_symbols)] + raw = rng.normal(size=(n_dates, n_symbols)) + signal = np.zeros_like(raw) + half = n_dates // 2 + signal[1:half] = 0.03 * raw[: half - 1] + signal[half:] = -0.03 * raw[half - 1 : -1] + panel = make_price_panel(dates, symbols, rng, signal=signal) + factor = pd.DataFrame(raw, index=dates, columns=symbols).stack() + factor.index.names = [DATE, SYMBOL] + factor = factor.sort_index() + + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(oos_split=str(dates[half].date())), + EvalContext(price_panel=panel), + ) + coverage = report.by_name()["data_coverage"].payload + assert coverage["settled_rebalances"] > 250 # plenty of RAW periods + assert coverage["effective_samples"] < 10 # ... and almost no independent ones + assert coverage["span_days"] > 365 # not a span problem + assert report.by_name()["oos_generalization"].payload["sign_flipped"] is True + + # v0.4: the measured flip is REPORTED, not swallowed by the gate. + verdict = report.require_verdict() + assert verdict.verdict == REJECT + assert any("flipped" in r for r in verdict.reasons) + # ... and it rejects ON THE FLIP, not on a mumbled word about the sample. + assert not any(r.startswith("effective samples (A)") for r in verdict.reasons) + + # The gate is not gone, only NARROWED to the positive claims: this very sample + # would still refuse a Predictive PASS. Same coverage facts, flip flag cleared. + # (The Tradable axis is non-statistical, so it can still PASS -> the deployment + # label is WATCH, but crucially NOT Adopt: the predictive claim stays gated.) + gated = decide_verdict( + VerdictInputs( + expected_ic_sign=make_spec().expected_ic_sign, + settled_rebalances=coverage["settled_rebalances"], + effective_samples=coverage["effective_samples"], + span_days=coverage["span_days"], + ic_ir=0.8, ic_win_rate=0.7, ic_nw_t=3.5, monotonicity_spearman=0.9, + net_long_short_by_cost=((1.0, 0.05),), + oos_available=True, oos_sign_consistent=True, + tradable=True, capacity_sufficient=True, + ) + ) + assert gated.verdict != ADOPT + assert gated.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any(r.startswith("effective samples (A)") for r in gated.predictive.reasons) + + +def test_same_cell_reversal_never_claims_independent_cell_evidence(rng): + """A holdout reversal must NOT be reported as an independent-cell reversal. + + ``verdict.py`` documents ``oos_monotonicity_reversed`` as "# independent-cell + reversal (I5e)" and its Reject reason reads "quantile monotonicity reversed on + an independent cell". This evaluator scores ONE cell and reports + independent_cells_evaluated=0 in the very same payload — so writing a same-cell + number into that key would make the report claim an independent check ran and + failed when none ran at all. The same-cell figure belongs under its own name. + """ + n_dates, n_symbols = 300, 40 + dates = pd.bdate_range("2024-01-02", periods=n_dates, name=DATE) + symbols = [f"{i:06d}.SZ" for i in range(n_symbols)] + raw = rng.normal(size=(n_dates, n_symbols)) + signal = np.zeros_like(raw) + half = n_dates // 2 + signal[1:half] = 0.03 * raw[: half - 1] # train: hypothesis holds + signal[half:] = -0.03 * raw[half - 1 : -1] # test: quantiles reverse + panel = make_price_panel(dates, symbols, rng, signal=signal) + factor = pd.DataFrame(raw, index=dates, columns=symbols).stack() + factor.index.names = [DATE, SYMBOL] + # DEFAULT thresholds: since v0.4 the hard Rejects precede the sample gate, so + # this run reaches the rule under test (the independent-cell CLAIM) on the + # strength of its MEASURED sign flip — no gate-opening fixture needed. + report = StandardFactorEvaluator().evaluate( + factor.sort_index(), make_spec(), make_cfg(oos_split=str(dates[half].date())), + EvalContext(price_panel=panel), + ) + section = report.by_name()["oos_generalization"] + # the same-cell holdout genuinely DID reverse ... + assert section.payload["test_monotonicity_spearman"] == pytest.approx(-1.0) + assert section.payload["test_monotonicity_aligned"] < 0 + # ... but no independent cell was ever evaluated, so the contract's key stays + # False and the report must never utter the independent-cell claim. + assert section.payload["independent_cells_evaluated"] == 0 + assert section.payload["monotonicity_reversed"] is False + assert "NOT ASSESSED" in section.payload["monotonicity_reversed_status"] + + verdict = report.require_verdict() + assert not any("independent cell" in reason for reason in verdict.reasons) + # the factor is still rejected — on the evidence that WAS gathered (the sign flip) + assert verdict.verdict == REJECT + assert any("sign flipped" in reason for reason in verdict.reasons) + + +def test_unparseable_oos_split_raises_instead_of_degrading_silently(rng): + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + with pytest.raises(ValueError, match="not a parseable date"): + StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(oos_split="not-a-date"), + EvalContext(price_panel=panel), + ) + + +def test_no_oos_evidence_can_never_reach_adopt(rng): + """The contract's central promise, verified END TO END on a strong factor. + + The planted signal is deliberately huge — in-sample everything passes. Without + a holdout it must still stop at Watch. + """ + panel, factor = planted_signal_setup(rng, n_dates=300, n_symbols=40, strength=0.05) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), # no oos_split + EvalContext( + price_panel=panel, + # even with tradability fully established: + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ), + ) + verdict = report.require_verdict() + assert verdict.verdict != ADOPT + assert verdict.verdict == WATCH + assert any("no out-of-sample split" in r.lower() for r in verdict.reasons) + + +def test_insufficient_data_short_circuits_before_any_conclusion(rng): + panel, factor = planted_signal_setup(rng, n_dates=12, n_symbols=20, strength=0.05) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + verdict = report.require_verdict() + assert verdict.verdict == INSUFFICIENT_DATA + assert verdict.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any("settled rebalances" in r for r in verdict.reasons) + + +def test_adopt_requires_oos_plus_tradability_plus_a_positive_base_spread(rng): + """The only path to Adopt: every leg of the §6 rule satisfied by real facts. + + ``is_exploratory=False`` is one of those legs since v0.2 — make_cfg() declares + an EXPLORATORY run by default, which the §6 cap holds at Watch. This is the + end-to-end guarantee that Adopt stays REACHABLE for a run that claims it. + """ + panel, factor = planted_signal_setup(rng, n_dates=300, n_symbols=40, strength=0.05) + dates = panel_dates(factor) + # ADOPT now needs all three axes to PASS -> a known-factor book the factor is + # INCREMENTAL to (a default run, with no book, tops out at Watch by design). + ctx = EvalContext( + price_panel=panel, + known_factors=incremental_book(rng, factor), + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ) + cfg = make_cfg(oos_split=str(dates[130].date()), is_exploratory=False) + report = StandardFactorEvaluator().evaluate(factor, make_spec(), cfg, ctx) + verdict = report.require_verdict() + assert verdict.verdict == ADOPT + assert verdict.incremental.verdict == AXIS_PASS + + # ... and remove ONLY the execution evidence: Adopt must fall back to Watch. + without_execution = StandardFactorEvaluator().evaluate( + factor, + make_spec(), + cfg, + EvalContext(price_panel=panel, known_factors=incremental_book(rng, factor)), + ) + assert without_execution.require_verdict().verdict == WATCH + assert without_execution.require_verdict().tradable.verdict == AXIS_NOT_ASSESSED + + +def test_an_exploratory_declaration_caps_an_adopt_grade_run_at_watch(rng): + """Design §6 (v0.2) END TO END: the identical facts that earn Adopt above stop + at Watch when the run declares itself exploratory — and stop at WATCH, not + Reject, because the declaration caps a claim rather than failing the factor. + """ + panel, factor = planted_signal_setup(rng, n_dates=300, n_symbols=40, strength=0.05) + dates = panel_dates(factor) + ctx = EvalContext( + price_panel=panel, + known_factors=incremental_book(rng, factor), + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ) + cfg = make_cfg(oos_split=str(dates[130].date()), is_exploratory=True) + verdict = ( + StandardFactorEvaluator() + .evaluate(factor, make_spec(), cfg, ctx) + .require_verdict() + ) + # all three axes PASS, but the exploratory flag caps the LABEL at Watch. + assert verdict.verdict == WATCH + assert all(a.verdict == AXIS_PASS for a in verdict.axes().values()) + assert any("is_exploratory=True" in r for r in verdict.reasons) + assert any("CAPPED AT WATCH" in r for r in verdict.reasons) + # the qualifying OOS evidence is still reported, not buried by the cap. + assert any("out-of-sample subperiods" in r for r in verdict.reasons) + + +def test_post_hoc_selection_cannot_reach_adopt_end_to_end(rng): + """§4 forces post_hoc_selected=True => is_exploratory=True, and the §6 cap then + denies Adopt: a factor picked after seeing these results cannot be reported as + a confirmation, no matter how good the numbers are.""" + panel, factor = planted_signal_setup(rng, n_dates=300, n_symbols=40, strength=0.05) + dates = panel_dates(factor) + ctx = EvalContext( + price_panel=panel, + known_factors=incremental_book(rng, factor), + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ) + cfg = make_cfg( + oos_split=str(dates[130].date()), is_exploratory=True, post_hoc_selected=True + ) + verdict = ( + StandardFactorEvaluator() + .evaluate(factor, make_spec(), cfg, ctx) + .require_verdict() + ) + assert verdict.verdict == WATCH + assert any("is_exploratory=True" in r for r in verdict.reasons) + + +def test_a_factor_redundant_with_the_book_is_rejected_end_to_end(rng): + """⭐ THE headline new capability (design §6, v0.5): a factor that DUPLICATES a + supplied known factor is REJECTED — even though its raw predictive signal is + strong, its out-of-sample sign is consistent, and it is fully tradable. Judging + a factor in isolation would have Adopted it; the Incremental axis catches that a + cross-sectional multi-factor book already has this signal (a value_ep twin gets + rejected).""" + panel, factor, book = redundant_setup(rng, n_dates=300, n_symbols=40) + dates = panel_dates(factor) + ctx = EvalContext( + price_panel=panel, + known_factors=book, + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ) + cfg = make_cfg(oos_split=str(dates[130].date()), is_exploratory=False) + report = StandardFactorEvaluator().evaluate(factor, make_spec(), cfg, ctx) + verdict = report.require_verdict() + + # the RAW predictive story is genuinely strong AND tradable ... + assert verdict.predictive.verdict == AXIS_PASS + assert verdict.tradable.verdict == AXIS_PASS + purity = report.by_name()["purity"] + assert purity.payload["ic_mean_raw"] > 0.10 # a real raw signal + # ... but residualizing on the book collapses it ~ to zero. + assert abs(purity.payload["ic_mean_orthogonalized_vs_book"]) < 0.3 * purity.payload[ + "ic_mean_raw" + ] + assert verdict.incremental.verdict == AXIS_FAIL + assert verdict.verdict == REJECT + assert any("incremental" in r for r in verdict.reasons) # names the axis + # the FAIL explains itself: it collapsed after residualizing on the book. + assert any( + "known-factor book" in r for r in verdict.incremental.reasons + ) + + +def test_a_factor_incremental_to_a_noise_book_passes_the_incremental_axis(rng): + """The mirror of the headline: a book the factor does NOT duplicate leaves the + orthogonalized IC essentially intact -> the Incremental axis PASSes.""" + # n_dates=300 clears the sample gate (span >= 365 calendar days) so the axis + # can be PASS rather than INSUFFICIENT_DATA. + panel, factor = planted_signal_setup(rng, n_dates=300, n_symbols=40, strength=0.05) + report = StandardFactorEvaluator().evaluate( + factor, + make_spec(), + make_cfg(), + EvalContext(price_panel=panel, known_factors=incremental_book(rng, factor)), + ) + purity = report.by_name()["purity"] + assert purity.payload["known_factors_supplied"] is True + raw = purity.payload["ic_mean_raw"] + # the book explains almost none of the signal, so the residual IC survives. + assert purity.payload["ic_mean_orthogonalized_vs_book"] > 0.7 * raw + assert report.require_verdict().incremental.verdict == AXIS_PASS + + +def test_verdict_thresholds_are_honoured(rng): + panel, factor = planted_signal_setup(rng, n_dates=300, n_symbols=40, strength=0.05) + dates = panel_dates(factor) + cfg = make_cfg(oos_split=str(dates[130].date())) + strict = VerdictThresholds(min_rebalances=10_000) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), cfg, EvalContext(price_panel=panel), thresholds=strict + ) + assert report.require_verdict().verdict == INSUFFICIENT_DATA + + +# ========================================================================== +# 6. report rendering / export +# ========================================================================== + + +def test_report_renders_and_exports_deterministically(rng): + panel, factor = planted_signal_setup(rng, n_dates=60, n_symbols=20) + ctx = EvalContext(price_panel=panel) + evaluator = StandardFactorEvaluator() + first = evaluator.evaluate(factor, make_spec(), make_cfg(), ctx) + second = evaluator.evaluate(factor, make_spec(), make_cfg(), ctx) + assert first.render() == second.render() + assert first.to_json() == second.to_json() + + markdown = first.render() + for heading in ( + "## 2. Predictive Power", "## 3. Return & Risk", "## 4. Stability & Cost", + "## 5. Purity", "## 6. OOS & Generalization", "## 7. Execution & Capacity", + "## 8. Data & Coverage", "## 9. Caveats & Provenance", + ): + assert heading in markdown + assert "_MISSING" not in markdown + + exported = first.to_dict() + assert [s["name"] for s in exported["sections"]] == list(MANDATORY_SECTIONS) + assert all(s["status"] in ("ok", "skipped") for s in exported["sections"]) + assert exported["verdict"]["verdict"] in {ADOPT, WATCH, REJECT, INSUFFICIENT_DATA} + + +def test_report_export_has_no_missing_sections_for_the_standard_evaluator(rng): + panel, factor = planted_signal_setup(rng, n_dates=60, n_symbols=20) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + statuses = {s["name"]: s["status"] for s in report.to_dict()["sections"]} + assert set(statuses) == set(MANDATORY_SECTIONS) + assert "missing" not in statuses.values() + + +def test_a_subclass_cannot_drop_a_mandatory_section(rng): + """Enforcement layer #2 still bites a StandardFactorEvaluator subclass.""" + + class Sneaky(StandardFactorEvaluator): + def evaluate(self, factor_panel, spec, cfg, ctx=None, thresholds=None): + ir = self.build_ir(factor_panel, spec, cfg, ctx) + partial = FactorEvalReport.assemble( + spec, cfg, [self.predictive_power(ir)] + ) + return partial.with_verdict() + + panel, factor = planted_signal_setup(rng, n_dates=40, n_symbols=20) + with pytest.raises(ValueError, match="missing mandatory section"): + Sneaky().evaluate(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + + +def test_evaluator_rejects_a_non_evalcontext_ctx(rng): + panel, factor = planted_signal_setup(rng, n_dates=20, n_symbols=10) + with pytest.raises(TypeError, match="EvalContext"): + StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), {"price_panel": panel} + ) + + +# ========================================================================== +# #3: N_eff confidence intervals + pre-registered criteria (design §6, v0.6) +# ========================================================================== + + +def _normalize(values, mean, std): + """Rescale a series to a target (mean, sample-std), preserving autocorrelation.""" + s = pd.Series(np.asarray(values), dtype=float) + return (s - s.mean()) / s.std(ddof=1) * std + mean + + +def test_information_ratio_ci_uses_n_eff_and_gates_correlated_out_where_iid_passes(): + """⭐ THE whole point of the N_eff CI: two IC series of the SAME raw length and + the SAME ICIR point, one i.i.d. and one heavily autocorrelated. The correlated + one carries far fewer EFFECTIVE observations, so its CI is wider and its LOWER + bound falls below the bar that the i.i.d. one's lower bound clears.""" + rng = np.random.default_rng(7) + iid = _normalize(rng.normal(size=500), mean=0.5, std=1.0) + ar = _normalize(_ar1(0.9, 500, seed=7), mean=0.5, std=1.0) + ci_iid = information_ratio_ci(iid) + ci_ar = information_ratio_ci(ar) + + assert ci_iid["point"] == pytest.approx(0.5) # identical ICIR points ... + assert ci_ar["point"] == pytest.approx(0.5) + assert ci_ar["n_eff"] < ci_iid["n_eff"] # ... but autocorrelation -> fewer + assert ci_ar["se"] > ci_iid["se"] # -> a WIDER interval + assert ci_ar["ci_low"] < ci_iid["ci_low"] # -> a lower lower-bound + + # the SE is the Lo Sharpe formula with N_eff, NOT the raw count: + ir, n_eff = ci_ar["point"], ci_ar["n_eff"] + assert ci_ar["se"] == pytest.approx(math.sqrt((1.0 + 0.5 * ir * ir) / n_eff)) + assert ci_ar["se"] > math.sqrt((1.0 + 0.5 * ir * ir) / 500) # raw N understates + + # the gate consequence: at the default bar the i.i.d. lower CI clears, the + # autocorrelated one does not -- the correlated series fails where iid passes. + bar = VerdictThresholds().min_abs_icir + assert ci_iid["ci_low"] > bar + assert ci_ar["ci_low"] < bar + + +def test_mean_ci_uses_n_eff_not_raw_count(): + ar = _ar1(0.9, 500, seed=3) + ci = mean_ci(ar) + assert ci["n_eff"] < 500 + std = float(pd.Series(ar, dtype=float).std(ddof=1)) + assert ci["se"] == pytest.approx(std / math.sqrt(ci["n_eff"])) + # the interval brackets the point symmetrically at the stated confidence. + assert ci["confidence"] == 0.95 + assert ci["ci_low"] < ci["point"] < ci["ci_high"] + + +def test_degenerate_series_give_nan_cis_never_a_fabricated_number(): + assert math.isnan(information_ratio_ci(pd.Series([1.0]))["ci_low"]) + assert math.isnan(information_ratio_ci(pd.Series([2.0, 2.0, 2.0]))["ci_low"]) # zero var + assert math.isnan(mean_ci(pd.Series([], dtype=float))["ci_low"]) + + +def test_predictive_and_purity_sections_report_n_eff_based_cis(rng): + panel, factor = planted_signal_setup(rng, n_dates=300, n_symbols=40, strength=0.05) + report = StandardFactorEvaluator().evaluate( + factor, + make_spec(), + make_cfg(), + EvalContext(price_panel=panel, known_factors=incremental_book(rng, factor)), + ) + pred = report.by_name()["predictive_power"].payload + assert pred["ic_ir_ci_low"] < pred["ic_ir"] < pred["ic_ir_ci_high"] + assert pred["ic_ir_ci_confidence"] == 0.95 + assert pred["ic_ir_ci_n_eff"] <= pred["ic_periods_finite"] # N_eff never > N + purity = report.by_name()["purity"].payload + assert purity["incremental_ic_ir_ci_low"] < purity["incremental_ic_ir"] + + +def test_reported_cis_are_deterministic(rng): + panel, factor = planted_signal_setup(rng, n_dates=120, n_symbols=25) + ctx = EvalContext(price_panel=panel, known_factors=incremental_book(rng, factor)) + first = StandardFactorEvaluator().evaluate(factor, make_spec(), make_cfg(), ctx) + second = StandardFactorEvaluator().evaluate(factor, make_spec(), make_cfg(), ctx) + assert first.to_json() == second.to_json() + + +def test_declared_success_criteria_flow_through_the_standard_evaluator(rng): + """End to end: a pre-registered bar on EvalConfig changes the deployment label + AND is stamped. The strong signal Adopts under the default bar; a stricter + declared bar (min ICIR 8.0, unreachable by any lower CI) blocks the Predictive + PASS, so the same run no longer Adopts — and the report says which bar it used.""" + panel, factor = planted_signal_setup(rng, n_dates=300, n_symbols=40, strength=0.05) + dates = panel_dates(factor) + ctx = EvalContext( + price_panel=panel, + known_factors=incremental_book(rng, factor), + execution_capacity={"tradable": True, "capacity_sufficient": True}, + ) + base_cfg = make_cfg(oos_split=str(dates[130].date()), is_exploratory=False) + adopted = StandardFactorEvaluator().evaluate(factor, make_spec(), base_cfg, ctx) + assert adopted.require_verdict().verdict == ADOPT + assert adopted.criteria_source == "default" + + # A pre-registered bar set BETWEEN the ICIR lower CI bound and the point: the + # POINT clears it, but the LOWER CI does not -> the CI gate (not the point) + # blocks the Predictive PASS, so the same run reads INSUFFICIENT and no longer + # Adopts. Picked from the run's own reported CI so it is robust to the draw. + pred = adopted.by_name()["predictive_power"].payload + bar = (pred["ic_ir_ci_low"] + pred["ic_ir"]) / 2.0 + assert pred["ic_ir_ci_low"] < bar < pred["ic_ir"] + strict_cfg = make_cfg( + oos_split=str(dates[130].date()), + is_exploratory=False, + success_criteria=VerdictThresholds(min_abs_icir=bar), + ) + strict = StandardFactorEvaluator().evaluate(factor, make_spec(), strict_cfg, ctx) + assert strict.criteria_source == "declared" + assert strict.require_verdict().predictive.verdict == AXIS_INSUFFICIENT_DATA + assert strict.require_verdict().verdict != ADOPT + assert "PRE-REGISTERED" in strict.render() + + +# ========================================================================== +# 7. layering +# ========================================================================== + + +def test_factors_layer_never_imports_analytics(): + """Invariant #1/#3: forward returns live in analytics; factors must not reach up.""" + import pathlib + + root = pathlib.Path(__file__).resolve().parents[1] / "factors" + offenders = [ + str(path.relative_to(root.parent)) + for path in root.rglob("*.py") + if "analytics" in path.read_text(encoding="utf-8") + and any( + line.strip().startswith(("import analytics", "from analytics")) + for line in path.read_text(encoding="utf-8").splitlines() + ) + ] + assert offenders == [] diff --git a/tests/test_jump_amount_corr_factor.py b/tests/test_jump_amount_corr_factor.py new file mode 100644 index 0000000..f231344 --- /dev/null +++ b/tests/test_jump_amount_corr_factor.py @@ -0,0 +1,251 @@ +"""PR-C: price-jump turnover-correlation factor (aggregation + spec + PIT).""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from data.clean.intraday_aggregate import compute_jump_amount_corr +from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars +from factors.compute.intraday_derived import JumpAmountCorrFactor +from factors.spec import FactorSpec + +_SYM = "000001.SZ" + + +def _bars(rows): + """rows = [(time, symbol, open, high, low, amount), ...] -> normalized 1min bars. + + ``close``/``volume`` are filled harmlessly; the factor reads open/high/low/amount. + """ + df = pd.DataFrame( + { + "time": pd.to_datetime([r[0] for r in rows]), + "symbol": [r[1] for r in rows], + "open": [r[2] for r in rows], + "high": [r[3] for r in rows], + "low": [r[4] for r in rows], + "close": [r[2] for r in rows], + "volume": [1.0] * len(rows), + "amount": [r[5] for r in rows], + } + ) + return normalize_intraday_bars(df, freq="1min") + + +def _session(day, amps, amounts, sym=_SYM, start="09:31:00"): + """One session: amplitude ``a`` at open=100 -> high=100+a*50, low=100-a*50.""" + base = pd.Timestamp(day) + pd.Timedelta(start) + return [ + (base + pd.Timedelta(minutes=i), sym, 100.0, 100.0 + a * 50.0, 100.0 - a * 50.0, amt) + for i, (a, amt) in enumerate(zip(amps, amounts)) + ] + + +def _hand_pairs(amps, amounts, jump_z=1.0): + """Reference jump-pairs: consecutive bar i with within-day z(amp)>jump_z, i+1 exists.""" + amp = np.asarray(amps, dtype=float) + z = (amp - amp.mean()) / amp.std(ddof=1) + return [ + (amounts[i], amounts[i + 1]) + for i in range(len(amps)) + if z[i] > jump_z and i + 1 < len(amps) + ] + + +# --------------------------------------------------------------------------- # +# Correctness vs a hand-computed reference +# --------------------------------------------------------------------------- # +def test_corr_matches_hand_reference_two_days(): + amps1 = [0.001, 0.02, 0.001, 0.001, 0.02, 0.001] + amts1 = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0] + amps2 = [0.001, 0.03, 0.001, 0.001, 0.001, 0.001] + amts2 = [11.0, 22.0, 33.0, 44.0, 55.0, 66.0] + bars = _bars(_session("2021-07-01", amps1, amts1) + _session("2021-07-02", amps2, amts2)) + out = compute_jump_amount_corr(bars, lookback_days=20, min_pairs=2) + + p1 = _hand_pairs(amps1, amts1) + all_pairs = p1 + _hand_pairs(amps2, amts2) + x1 = np.array([a for a, _ in p1]) + y1 = np.array([b for _, b in p1]) + xa = np.array([a for a, _ in all_pairs]) + ya = np.array([b for _, b in all_pairs]) + d1, d2 = pd.Timestamp("2021-07-01"), pd.Timestamp("2021-07-02") + assert out.loc[(d1, _SYM)] == pytest.approx(np.corrcoef(x1, y1)[0, 1]) + assert out.loc[(d2, _SYM)] == pytest.approx(np.corrcoef(xa, ya)[0, 1]) + assert out.name == "jump_amount_corr" + + +def test_zscore_threshold_selects_only_high_amplitude_bars(): + # Only bar index 1 has a high amplitude -> exactly one jump; index 5 (last) is + # also large but has no strictly-next minute, so it never pairs. + amps = [0.001, 0.05, 0.001, 0.001, 0.001, 0.05] + amts = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0] + bars = _bars(_session("2021-07-01", amps, amts)) + # exactly one pair -> corr undefined at min_pairs=2 -> NaN; the single pair is + # (20, 30) as the reference confirms. + assert _hand_pairs(amps, amts) == [(20.0, 30.0)] + out = compute_jump_amount_corr(bars, lookback_days=20, min_pairs=2) + assert out.dropna().empty # one pair < min_pairs + + +def test_session_close_jump_is_not_paired_across_days(): + # jump only at the LAST bar of the session; its "next" bar is the next day -> + # gap != 60s -> excluded. + amps = [0.001, 0.001, 0.001, 0.001, 0.001, 0.05] + bars = _bars( + _session("2021-07-05", amps, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) + + _session("2021-07-06", amps, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) + ) + out = compute_jump_amount_corr(bars, lookback_days=20, min_pairs=2) + assert out.dropna().empty + + +def test_lunch_gap_jump_is_not_paired(): + rows = [ + (pd.Timestamp("2021-07-06 09:31"), _SYM, 100.0, 100.05, 99.95, 10.0), + (pd.Timestamp("2021-07-06 09:32"), _SYM, 100.0, 103.0, 97.0, 20.0), # jump + (pd.Timestamp("2021-07-06 13:01"), _SYM, 100.0, 100.05, 99.95, 30.0), # afternoon + ] + out = compute_jump_amount_corr(_bars(rows), lookback_days=20, min_pairs=2) + # the 09:32 jump's next bar is 13:01 (gap != 60s) -> no pair. + assert out.dropna().empty + + +def test_trailing_window_drops_pairs_older_than_lookback(): + # Day A has 2 jump-pairs; then 3 no-jump days; a lookback_days=2 window at the + # later days no longer sees day A's pairs. + amps = [0.001, 0.02, 0.001, 0.001, 0.02, 0.001] + amts = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0] + flat = [0.001] * 6 + rows = ( + _session("2021-07-01", amps, amts) + + _session("2021-07-02", flat, [1.0] * 6) + + _session("2021-07-05", flat, [1.0] * 6) + + _session("2021-07-06", flat, [1.0] * 6) + ) + out = compute_jump_amount_corr(_bars(rows), lookback_days=2, min_pairs=2) + d1 = pd.Timestamp("2021-07-01") + d6 = pd.Timestamp("2021-07-06") + assert np.isfinite(out.loc[(d1, _SYM)]) # window covers day A's 2 pairs + assert np.isnan(out.loc[(d6, _SYM)]) # day A is now outside the 2-day window + + +def test_min_pairs_gate_returns_nan(): + amps = [0.001, 0.02, 0.001, 0.001, 0.02, 0.001] # 2 jump-pairs on the day + bars = _bars(_session("2021-07-01", amps, [10.0, 20.0, 30.0, 40.0, 50.0, 60.0])) + out = compute_jump_amount_corr(bars, lookback_days=20, min_pairs=3) + assert np.isnan(out.loc[(pd.Timestamp("2021-07-01"), _SYM)]) + + +def test_pit_perturbing_future_bars_does_not_change_factor_at_d(): + amps1 = [0.001, 0.02, 0.001, 0.001, 0.02, 0.001] + amts1 = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0] + amps2 = [0.001, 0.03, 0.001, 0.001, 0.001, 0.001] + base = _session("2021-07-01", amps1, amts1) + a = compute_jump_amount_corr( + _bars(base + _session("2021-07-02", amps2, [11.0, 22.0, 33.0, 44.0, 55.0, 66.0])), + min_pairs=2, + ) + # wildly perturb EVERY day-2 bar (amounts + amplitudes) + b = compute_jump_amount_corr( + _bars(base + _session("2021-07-02", [0.09, 0.001, 0.09, 0.001, 0.09, 0.001], + [9e5, 1.0, 9e5, 1.0, 9e5, 1.0])), + min_pairs=2, + ) + d1 = pd.Timestamp("2021-07-01") + assert a.loc[(d1, _SYM)] == pytest.approx(b.loc[(d1, _SYM)]) + + +def test_per_symbol_isolation(): + amps = [0.001, 0.02, 0.001, 0.001, 0.02, 0.001] + amts_a = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0] + amts_b = [60.0, 50.0, 40.0, 30.0, 20.0, 10.0] + rows = _session("2021-07-01", amps, amts_a, sym="AAA.SZ") + _session( + "2021-07-01", amps, amts_b, sym="BBB.SZ" + ) + out = compute_jump_amount_corr(_bars(rows), lookback_days=20, min_pairs=2) + d1 = pd.Timestamp("2021-07-01") + pa = _hand_pairs(amps, amts_a) + pb = _hand_pairs(amps, amts_b) + ea = np.corrcoef([a for a, _ in pa], [b for _, b in pa])[0, 1] + eb = np.corrcoef([a for a, _ in pb], [b for _, b in pb])[0, 1] + assert out.loc[(d1, "AAA.SZ")] == pytest.approx(ea) + assert out.loc[(d1, "BBB.SZ")] == pytest.approx(eb) + + +def test_empty_bars_yield_empty_schema_series(): + out = compute_jump_amount_corr(empty_intraday_bars()) + assert out.empty + assert list(out.index.names) == ["date", "symbol"] + + +def test_open_and_amount_guards(): + rows = [ + (pd.Timestamp("2021-07-01 09:31"), _SYM, 0.0, 100.05, 99.95, 10.0), # open<=0 dropped + (pd.Timestamp("2021-07-01 09:32"), _SYM, 100.0, 103.0, 97.0, np.nan), # amount NaN dropped + (pd.Timestamp("2021-07-01 09:33"), _SYM, 100.0, 100.05, 99.95, 30.0), + ] + # no valid jump-pair survives -> empty (guards drop the bad rows before pairing). + out = compute_jump_amount_corr(_bars(rows), min_pairs=2) + assert out.dropna().empty + + +def test_input_bars_not_mutated(): + bars = _bars(_session("2021-07-01", [0.001, 0.02, 0.001], [10.0, 20.0, 30.0])) + before = bars.copy(deep=True) + compute_jump_amount_corr(bars, min_pairs=2) + pd.testing.assert_frame_equal(bars, before) + + +# --------------------------------------------------------------------------- # +# Spec + Factor subclass +# --------------------------------------------------------------------------- # +def test_factor_spec_is_valid_and_daily(): + spec = JumpAmountCorrFactor().spec + assert isinstance(spec, FactorSpec) + assert spec.factor_id == "jump_amount_corr_20" + assert spec.is_intraday is False + assert spec.expected_ic_sign == -1 + assert spec.return_basis == "close_to_close" + assert spec.forward_return_horizon == 1 + # is_intraday=False => the whole minute block MUST be None (validated by FactorSpec). + for field in ("decision_cutoff", "data_lag", "session_open", "execution_model", "execution_window"): + assert getattr(spec, field) is None + + +def test_factor_subclass_passes_init_subclass_and_window_tracks_name(): + f = JumpAmountCorrFactor(lookback_days=10) + assert f.name == "jump_amount_corr_10" + assert f.spec.factor_id == "jump_amount_corr_10" + + +def test_factor_compute_selects_preaggregated_column(): + f = JumpAmountCorrFactor() + idx = pd.MultiIndex.from_tuples( + [(pd.Timestamp("2021-07-01"), _SYM)], names=["date", "symbol"] + ) + panel = pd.DataFrame({f.name: [0.25]}, index=idx) + out = f.compute(panel) + assert out.loc[(pd.Timestamp("2021-07-01"), _SYM)] == 0.25 + assert out.name == f.name + + +def test_factor_compute_missing_column_raises(): + f = JumpAmountCorrFactor() + idx = pd.MultiIndex.from_tuples( + [(pd.Timestamp("2021-07-01"), _SYM)], names=["date", "symbol"] + ) + with pytest.raises(ValueError, match="pre-aggregated"): + f.compute(pd.DataFrame({"other": [1.0]}, index=idx)) + + +def test_bad_params_raise(): + with pytest.raises(ValueError): + JumpAmountCorrFactor(lookback_days=0) + bars = _bars(_session("2021-07-01", [0.001, 0.02, 0.001], [10.0, 20.0, 30.0])) + with pytest.raises(ValueError): + compute_jump_amount_corr(bars, min_pairs=1) # Pearson needs >= 2 points + with pytest.raises(ValueError): + compute_jump_amount_corr(bars, lookback_days=0)