From 08829bd6b7567ffb5c52b747aae83b127a927ff3 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Mon, 20 Jul 2026 08:06:14 -0700 Subject: [PATCH 1/3] =?UTF-8?q?fix(analytics):=20contract=20v0.8=20?= =?UTF-8?q?=E2=80=94=20cost-correct=20aligned=20spread=20+=20per-date=20mo?= =?UTF-8?q?notonicity=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First authorized change to the frozen `analytics/eval/` contract layer. Two defects surfaced by the ten factor reproductions run under the no-touch rule. 1. Hypothesis-aligned net spread added the cost back at sign=-1. Four sites computed `sign * net` where `net` was already cost-subtracted, so at sign=-1 the expression expanded to `-gross + cost`: the factor was handed its own trading fees back as profit. The hypothesis decides WHICH LEG IS LONG; cost is a drag in either direction, so the legs are flipped first and the cost subtracted after: cost = gross - net (= fee * multiplier * leg_turnover) aligned_net = sign * gross - cost Worked check (locked by test): gross 0.000125, net(1x) -0.001983 gives cost 0.002108 and aligned -0.002233; the old form read +0.001983. Two of the four sites are `_base_spread` / `_all_spreads_negative` in verdict.py, which feed the Tradable axis PASS condition and FAIL reason — a GATE defect, not a display one. It never bit because Tradable has been NOT_ASSESSED in every run to date. `VerdictInputs` gains `gross_long_short_mean`; an unknown gross makes a sign=-1 aligned spread UNKNOWN, which per the existing rule neither convicts nor passes. sign=+1 is bit-identical (`+1*gross - (gross-net) == net`, and the implementation returns `net` directly rather than round-tripping it). `net_long_short_by_cost` and `gross_long_short_mean` keep their plain, sign-agnostic meaning and wording verbatim — existing reports rely on it. 2. `monotonicity_spearman` is magnitude-sensitive but gated a rank axis. The pooled statistic correlates bucket index against cross-date arithmetic MEAN bucket returns: unbounded, so a few extreme-return days concentrated in one bucket can flip it while the daily-capped rank IC barely moves. Adds `monotonicity_spearman_by_date`: each date's own Spearman across its buckets (skipped when that date has fewer than 3 finite buckets, or when its buckets are all tied), each bounded in [-1, 1] before the cross-date mean — structurally parallel to the rank IC. The Predictive axis now gates on it. The pooled field is KEPT as a reported value, unchanged in semantics, so historical reports stay comparable. A missing/NaN per-date figure falls back to the pooled one and DISCLOSES the substitution in the axis reasons, so a pre-v0.8 IR stays judgeable without pretending it was judged on the new statistic. Both are UNVALIDATED-BY-DATA design corrections (design doc §11): reasoned defects in the implementation, not thresholds tuned after seeing results. Impact on the eleven factor reproductions is NOT assessed here — re-running them and disclosing every verdict change is the separate second stage. Gates: pytest 1668 passed (1649 baseline + 19 new), ruff clean, phase0 ic 0.9600 / annual 0.8408 unchanged, secret scan 0 hits. --- analytics/eval/report.py | 11 ++ analytics/eval/standard.py | 37 +++++- analytics/eval/stats.py | 38 ++++++ analytics/eval/verdict.py | 115 ++++++++++++++++-- tests/test_factor_eval_contract.py | 183 +++++++++++++++++++++++++++++ tests/test_factor_eval_standard.py | 162 +++++++++++++++++++++++++ 6 files changed, 531 insertions(+), 15 deletions(-) diff --git a/analytics/eval/report.py b/analytics/eval/report.py index 63ec8a8..f329b99 100644 --- a/analytics/eval/report.py +++ b/analytics/eval/report.py @@ -123,6 +123,11 @@ def payload_of(name: str) -> Mapping[str, object]: monotonicity_spearman=_payload_number( returns, "monotonicity_spearman", float("nan") ), + # design §6 v0.8: the GATED monotonicity. Absent (a pre-v0.8 IR) -> NaN -> + # the verdict falls back to the pooled field and DISCLOSES it. + monotonicity_spearman_by_date=_payload_number( + returns, "monotonicity_spearman_by_date", float("nan") + ), # Incremental axis facts (design §6, v0.5). A Skipped purity section (no # book) leaves the payload empty -> known_factors_supplied defaults False # -> the axis is NOT_ASSESSED. A supplied-but-unmeasurable orthogonalized @@ -141,6 +146,12 @@ def payload_of(name: str) -> Mapping[str, object]: purity, "incremental_ic_ir_ci_high", float("nan") ), net_long_short_by_cost=by_cost, + # design §6 v0.8: needed to align a spread by hypothesis without adding the + # cost back (cost = gross - net). Absent -> NaN -> a sign=-1 aligned spread + # reads UNKNOWN rather than silently mis-signed. + gross_long_short_mean=_payload_number( + returns, "gross_long_short_mean", float("nan") + ), oos_available=bool(_payload_flag(oos, "oos_available", False)), oos_sign_consistent=bool(_payload_flag(oos, "sign_consistent", False)), oos_sign_flipped=bool(_payload_flag(oos, "sign_flipped", False)), diff --git a/analytics/eval/standard.py b/analytics/eval/standard.py index dc6d7c4..c07b1c4 100644 --- a/analytics/eval/standard.py +++ b/analytics/eval/standard.py @@ -65,6 +65,7 @@ newey_west_t, sortino, spearman, + spearman_by_date, ) from analytics.factor import ic_summary from analytics.performance import performance_summary @@ -329,14 +330,17 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: net_by_cost: dict[float, float] = {} cumulative_by_cost: dict[float, float] = {} base_net = None + base_cost = None for multiplier in ir.cfg.cost_scenarios: - net = gross - ir.ctx.fee_rate * multiplier * leg_turnover + cost = ir.ctx.fee_rate * multiplier * leg_turnover + net = gross - cost net_by_cost[float(multiplier)] = as_float(net.mean()) cumulative_by_cost[float(multiplier)] = as_float( (1.0 + net.dropna()).prod() - 1.0 ) if abs(multiplier - 1.0) < 1e-9: base_net = net + base_cost = cost payload: dict[str, object] = { "long_short_legs": f"Q{top} - Q{bottom}", @@ -345,9 +349,21 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: "quantile_periods_with_return": bucket_periods, # RAW Spearman of bucket index vs bucket mean return; the verdict is # the ONE place that multiplies by expected_ic_sign. + # + # POOLED (v0.1..v0.7 semantics, DELIBERATELY UNCHANGED): correlates + # against cross-date arithmetic means, so it is unbounded and carries + # MAGNITUDE structure. Kept as a reported field — historical reports + # stay comparable — but as of v0.8 it is NOT what the Predictive axis + # gates on, because a rank-based axis must not be decided by a + # magnitude-sensitive statistic. "monotonicity_spearman": spearman( [float(q) for q in buckets], [bucket_mean[int(q)] for q in buckets] ), + # PER-DATE version (v0.8, the GATED one): each date's Spearman over the + # buckets is capped in [-1, 1] BEFORE averaging across dates, exactly + # like the rank IC. Robust to a few extreme-return days concentrating + # in one bucket. RAW; the verdict applies expected_ic_sign. + "monotonicity_spearman_by_date": spearman_by_date(quantiles), "gross_long_short_mean": as_float(gross.mean()), "net_long_short_by_cost": net_by_cost, "net_long_short_cumulative_by_cost": cumulative_by_cost, @@ -358,8 +374,19 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: # for the reader (b); the Tradable axis still gates on the POINT base spread # > 0 (the CI on this leg difference is point-only for now — c names only # ICIR / incremental ICIR as the lower-CI magnitude criteria). - if base_net is not None: - base_ci = mean_ci(sign * base_net, confidence=DEFAULT_CONFIDENCE) + # HYPOTHESIS-ALIGNED base-cost spread (v0.8 fix). The hypothesis decides + # WHICH LEG IS LONG; cost is a drag in BOTH directions. So flip the legs + # first, THEN subtract the cost: + # aligned_net = sign * gross - cost + # The pre-v0.8 form `sign * net` expanded, at sign=-1, to `-gross + cost` + # — it ADDED the trading cost back as if shorting earned it. At sign=+1 + # this is bit-identical to the old expression (1 * gross - cost == net). + aligned_base = None + if base_net is not None and base_cost is not None: + aligned_base = sign * gross - base_cost + + if aligned_base is not None: + base_ci = mean_ci(aligned_base, confidence=DEFAULT_CONFIDENCE) payload["net_long_short_base_se"] = base_ci["se"] payload["net_long_short_base_ci_low"] = base_ci["ci_low"] payload["net_long_short_base_ci_high"] = base_ci["ci_high"] @@ -374,8 +401,8 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: f"annualization below is a fallback, not a derived fact." ) - if base_net is not None and base_net.notna().sum() >= 2: - aligned = (sign * base_net).dropna() + if aligned_base is not None and aligned_base.notna().sum() >= 2: + aligned = aligned_base.dropna() nav = (1.0 + aligned).cumprod() simple = performance_summary(nav, periods_per_year=ir.periods_per_year) payload["aligned_spread_annual_return"] = simple["annual_return"] diff --git a/analytics/eval/stats.py b/analytics/eval/stats.py index 29c5c70..7c792f9 100644 --- a/analytics/eval/stats.py +++ b/analytics/eval/stats.py @@ -393,6 +393,43 @@ def spearman(x: pd.Series | list, y: pd.Series | list) -> float: return float(pair["x"].corr(pair["y"], method="spearman")) +def spearman_by_date( + quantile_returns: pd.DataFrame, min_buckets: int = 3 +) -> float: + """Mean over dates of the PER-DATE Spearman(bucket index, bucket return). + + Structurally parallel to the rank IC (design §6, v0.8): each date's statistic + is BOUNDED in [-1, 1] before the cross-date average, so a handful of extreme + return days cannot dominate it. The pooled :func:`spearman` version correlates + against cross-date ARITHMETIC MEANS, which are unbounded and magnitude + sensitive — one outlier bucket-day can flip it while the daily-capped rank IC + barely moves. + + A date is SKIPPED — contributing nothing, never a zero — when it carries fewer + than ``min_buckets`` finite bucket returns (too few points for a meaningful + rank correlation), or when its surviving buckets are all tied (Spearman + undefined there). NaN when NO date qualifies: unknown, not 0.0. + + RAW, like :func:`spearman`: the hypothesis sign is applied by the verdict. + """ + if quantile_returns.empty or quantile_returns.shape[1] < min_buckets: + return float("nan") + index = [float(q) for q in quantile_returns.columns] + daily: list[float] = [] + for _, row in quantile_returns.iterrows(): + values = [float(v) for v in row.to_numpy(dtype=float)] + if sum(1 for v in values if math.isfinite(v)) < min_buckets: + continue + # spearman() drops the non-finite pairs itself, keeping each surviving + # bucket paired with its OWN index (not a renumbered 1..k). + rho = spearman(index, values) + if math.isfinite(rho): + daily.append(rho) + if not daily: + return float("nan") + return float(sum(daily) / len(daily)) + + def as_float(value: object) -> float: """Coerce to a plain float for a payload; anything unusable becomes NaN.""" try: @@ -414,4 +451,5 @@ def as_float(value: object) -> float: "newey_west_t", "sortino", "spearman", + "spearman_by_date", ] diff --git a/analytics/eval/verdict.py b/analytics/eval/verdict.py index 3a7edc0..2ee97ad 100644 --- a/analytics/eval/verdict.py +++ b/analytics/eval/verdict.py @@ -93,6 +93,15 @@ ``net_long_short_by_cost`` is the plain (top - bottom) leg difference. The HYPOTHESIS (``expected_ic_sign``) is applied in exactly ONE place — here — so a low-vol factor (sign -1) reads in its own direction. + + APPLYING THE SIGN TO A NET SPREAD IS NOT A MULTIPLICATION (v0.8). The + hypothesis decides WHICH LEG IS LONG; the trading cost is a drag in EITHER + direction. So the aligned spread flips the legs and THEN subtracts cost — + ``sign * gross - cost`` — never ``sign * net``, which at sign -1 expands to + ``-gross + cost`` and hands the factor its own costs back as profit. This + is why ``gross_long_short_mean`` is an input: the cost is recovered as + ``gross - net``, and an unknown gross makes the ALIGNED spread unknown + (which, per the rule above, then neither convicts nor passes). 2. ``ic_win_rate`` and the orthogonalized ICs are hypothesis-relative by the same rule: direction is applied by multiplying with ``expected_ic_sign``. @@ -308,7 +317,16 @@ class VerdictInputs: ic_ir_ci_high: float = float("nan") ic_win_rate: float = float("nan") # predictive_power (hypothesis-relative) ic_nw_t: float = float("nan") # predictive_power - monotonicity_spearman: float = float("nan") # return_risk (RAW) + #: return_risk: POOLED Spearman(bucket index, cross-date MEAN bucket return), + #: RAW. Unbounded and magnitude-sensitive; as of v0.8 it is a REPORTED figure + #: and only the FALLBACK for the gate below. + monotonicity_spearman: float = float("nan") + #: return_risk: the v0.8 GATED monotonicity — the mean over dates of each + #: date's own Spearman across the buckets, each capped in [-1, 1] before + #: averaging (structurally parallel to the rank IC). RAW. NaN = UNKNOWN, which + #: falls back to ``monotonicity_spearman`` with the substitution DISCLOSED in + #: the axis reasons, so an IR built before v0.8 stays judgeable. + monotonicity_spearman_by_date: float = float("nan") oos_available: bool = False # oos_generalization oos_sign_consistent: bool = False # expected sign holds in BOTH subperiods oos_sign_flipped: bool = False # explicit flip -> Predictive FAIL @@ -333,6 +351,11 @@ class VerdictInputs: # -- return_risk / execution facts ----------------------------------- #: return_risk: {cost multiplier: net (top - bottom) return}, RAW sign. net_long_short_by_cost: tuple[tuple[float, float], ...] = () + #: return_risk: mean GROSS (top - bottom) leg difference, RAW sign, before any + #: cost. Required to align a spread by hypothesis WITHOUT adding the cost back + #: (see :func:`_aligned_net`): the per-scenario cost is recovered as + #: ``gross - net``. NaN = UNKNOWN -> a sign=-1 aligned spread is UNKNOWN too. + gross_long_short_mean: float = float("nan") tradable: bool | None = None # execution_capacity; None = not supplied capacity_sufficient: bool | None = None # None = not supplied @@ -500,25 +523,79 @@ def _sample_gate_failures( return failures +def _aligned_net(net: float, gross: float, sign: int) -> float: + """Hypothesis-aligned NET spread: ``sign * gross - cost`` (design §6, v0.8). + + The hypothesis decides WHICH LEG IS LONG. Cost is a drag REGARDLESS of + direction. So the legs are flipped by the sign FIRST and the cost is + subtracted AFTER:: + + cost = gross - net (= fee * multiplier * leg_turnover) + aligned_net = sign * gross - cost + + Before v0.8 this was computed as ``sign * net``, which at ``sign = -1`` + expands to ``-gross + cost``: the trading cost was ADDED BACK, as though + reversing the legs also reversed who pays the fees. It flattered every + sign=-1 factor by exactly ``2 * cost``. + + ``sign = +1`` returns ``net`` UNCHANGED and needs no ``gross`` — algebraically + ``+1 * gross - (gross - net) == net``, and returning ``net`` directly keeps + the old path bit-identical instead of round-tripping it through a subtraction. + + An unknown (non-finite) ``gross`` at ``sign = -1`` yields NaN: the cost cannot + be recovered from ``net`` alone, so the aligned spread is UNKNOWN — and + unknown is neither evidence for nor against (it never convicts, never passes). + """ + if sign >= 0: + return net + if not (math.isfinite(gross) and math.isfinite(net)): + return float("nan") + return -gross - (gross - net) + + def _base_spread(inputs: VerdictInputs) -> float: """Hypothesis-aligned net long-short return at the BASE (1.0x) cost.""" for multiplier, value in inputs.net_long_short_by_cost: if abs(multiplier - 1.0) < 1e-9: - return inputs.expected_ic_sign * value + return _aligned_net( + value, inputs.gross_long_short_mean, inputs.expected_ic_sign + ) return float("nan") def _all_spreads_negative(inputs: VerdictInputs) -> bool: """True only when EVERY known cost scenario is non-positive (design §6). - An empty/unknown set is NOT "all negative" — unknown is not evidence. + An empty/unknown set is NOT "all negative" — unknown is not evidence. A + sign=-1 run with an unknown gross therefore yields NO scenarios and cannot + convict (:func:`_aligned_net` returns NaN, which is filtered out here). """ aligned = [ - inputs.expected_ic_sign * v + _aligned_net(v, inputs.gross_long_short_mean, inputs.expected_ic_sign) for _, v in inputs.net_long_short_by_cost - if math.isfinite(v) ] - return bool(aligned) and all(v <= 0 for v in aligned) + known = [a for a in aligned if math.isfinite(a)] + return bool(known) and all(a <= 0 for a in known) + + +def _gated_monotonicity(inputs: VerdictInputs) -> tuple[float, bool]: + """The RAW monotonicity the Predictive axis gates on + whether it FELL BACK. + + v0.8 gates on ``monotonicity_spearman_by_date`` (per-date Spearman, each capped + in [-1, 1] before averaging) rather than the pooled ``monotonicity_spearman``: + the Predictive axis is a RANK claim, and the pooled statistic is an unbounded + MAGNITUDE statistic — a few extreme return days landing in one bucket can flip + it while the daily-capped rank IC barely moves. Gating a rank axis on a + magnitude statistic is the category error being corrected. + + Returns ``(value, used_fallback)``. When the per-date figure is UNKNOWN (an IR + produced before v0.8, or too few finite buckets on every date) the pooled field + is used instead and ``used_fallback`` is True, so the caller can DISCLOSE the + substitution rather than silently judge on the weaker statistic. + """ + if math.isfinite(inputs.monotonicity_spearman_by_date): + return inputs.monotonicity_spearman_by_date, False + return inputs.monotonicity_spearman, True def _has_in_sample_point_signal(inputs: VerdictInputs, thr: VerdictThresholds) -> bool: @@ -530,7 +607,8 @@ def _has_in_sample_point_signal(inputs: VerdictInputs, thr: VerdictThresholds) - lower-bound test (design §6, v0.6) is applied SEPARATELY, only to decide PASS vs "promising but unconfirmed" for a factor that DID clear on the point. """ - aligned_monotonicity = inputs.expected_ic_sign * inputs.monotonicity_spearman + monotonicity, _ = _gated_monotonicity(inputs) + aligned_monotonicity = inputs.expected_ic_sign * monotonicity return ( _clears_magnitude(inputs.ic_ir, thr.min_abs_icir) and _clears_magnitude(inputs.ic_nw_t, thr.min_abs_nw_t) @@ -592,6 +670,21 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi if reasons: return AxisVerdict(AXIS_INSUFFICIENT_DATA, tuple(reasons)) + # The monotonicity actually gated on (design §6, v0.8). When the per-date + # figure is absent we judge on the pooled one, but we SAY SO — a silent + # substitution would hide that the decision rests on the magnitude-sensitive + # statistic the v0.8 fix exists to stop gating on. + _, mono_fallback = _gated_monotonicity(inputs) + mono_notes: tuple[str, ...] = () + if mono_fallback: + mono_notes = ( + "monotonicity gate FELL BACK to the pooled 'monotonicity_spearman': " + "the per-date figure ('monotonicity_spearman_by_date', design §6 v0.8) " + "was not supplied or not measurable. The pooled statistic is unbounded " + "and magnitude-sensitive, so this monotonicity judgement is weaker " + "than a v0.8 run's.", + ) + # 3. FAIL: a measured negative finding (OOS sign not consistent, or no point # signal at all). POINT-based — a wide CI is handled below, not here. point_signal = _has_in_sample_point_signal(inputs, thr) @@ -608,7 +701,7 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi "in-sample POINT metrics do NOT clear the predictive thresholds " "(ICIR / NW-t / win rate / monotonicity)." ) - return AxisVerdict(AXIS_FAIL, tuple(reasons)) + return AxisVerdict(AXIS_FAIL, tuple(reasons) + mono_notes) # 4/5. Point signal present + OOS consistent. The #3 lower-CI test decides # PASS vs "promising but unconfirmed" (INSUFFICIENT_DATA). This is ABOVE @@ -624,7 +717,8 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi f"{thr.min_abs_icir}; point |ICIR| {abs(inputs.ic_ir):.3f}, " f"|NW-t| {abs(inputs.ic_nw_t):.2f}, win rate " f"{inputs.ic_win_rate:.3f}.", - ), + ) + + mono_notes, ) lower_txt = "UNKNOWN" if not math.isfinite(icir_lower) else f"{icir_lower:+.3f}" return AxisVerdict( @@ -635,7 +729,8 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi f"pre-registered bar {thr.min_abs_icir}: promising, but not confirmed. " f"The sample size passed the gate; the ESTIMATE is still too imprecise " f"to claim a PASS (a wider bar than the raw count implies).", - ), + ) + + mono_notes, ) diff --git a/tests/test_factor_eval_contract.py b/tests/test_factor_eval_contract.py index 093b663..f23a4b8 100644 --- a/tests/test_factor_eval_contract.py +++ b/tests/test_factor_eval_contract.py @@ -12,6 +12,7 @@ import functools import json +import math import pytest @@ -37,6 +38,10 @@ decide_verdict, ) from analytics.eval.render import MAX_VALUE_CHARS + +# The v0.8 spread fix lived in an exact arithmetic expression, so the lock has to +# be on that expression itself, not on a PASS/FAIL that happens to agree with it. +from analytics.eval.verdict import _aligned_net, _all_spreads_negative, _base_spread from factors.base import Factor from factors.compute.candidates import ( VALUE_FIELDS, @@ -1298,6 +1303,12 @@ def test_negative_sign_factor_is_judged_in_its_own_direction(): ic_nw_t=-3.5, monotonicity_spearman=-0.9, # high-vol bucket underperforms net_long_short_by_cost=((1.0, -0.05),), # QN - Q1 negative: as predicted + # v0.8: aligning a NET spread needs the GROSS leg difference, because the + # cost (gross - net = 0.01) must stay SUBTRACTED after the legs are + # flipped: aligned = -(-0.04) - 0.01 = +0.03. Supplying only the net (as + # this fixture did pre-v0.8) now reads UNKNOWN rather than being scored + # with the costs handed back as profit. + gross_long_short_mean=-0.04, known_factors_supplied=True, incremental_ic_ir=-0.6, # residual IC negative: as predicted incremental_ic_mean=-0.04, @@ -2207,3 +2218,175 @@ def test_template_method_verdict_reflects_a_skipped_oos_section(): assert report.verdict.predictive.verdict == AXIS_INSUFFICIENT_DATA assert any("no out-of-sample split" in r for r in report.verdict.reasons) assert "_Skipped: no oos_split configured_" in report.render() + + +# -------------------------------------------------------------------------- +# v0.8 fix #1: the hypothesis-aligned NET spread must SUBTRACT cost, not add it +# back (design §6, v0.8). The aligned spread is `sign * gross - cost`, never +# `sign * net` — the latter expands at sign=-1 to `-gross + cost`. +# -------------------------------------------------------------------------- + + +def test_aligned_net_at_positive_sign_returns_the_net_untouched(): + """sign=+1 is BIT-IDENTICAL to the pre-v0.8 expression (`1 * net == net`).""" + for gross, net in ((0.05, 0.04), (0.000125, -0.001983), (-0.02, -0.03)): + assert _aligned_net(net, gross, 1) == net + # and it does not even need the gross: an unknown gross must not turn a + # perfectly known sign=+1 spread into UNKNOWN. + assert _aligned_net(0.04, float("nan"), 1) == 0.04 + + +def test_aligned_net_at_negative_sign_flips_the_legs_then_subtracts_cost(): + """The worked example from the v0.8 task card, to the last digit. + + gross = 0.000125, net(1x) = -0.001983 => cost = 0.002108 + aligned = -gross - cost = -0.000125 - 0.002108 = -0.002233 + The pre-v0.8 `sign * net` gave +0.001983 — a POSITIVE spread conjured out of + a factor that loses money gross AND pays 0.002108 to trade. + """ + gross, net = 0.000125, -0.001983 + assert _aligned_net(net, gross, -1) == pytest.approx(-0.002233, abs=1e-12) + # the defective reading, named so a regression cannot quietly restore it + assert _aligned_net(net, gross, -1) != pytest.approx(-1 * net, abs=1e-9) + + +def test_aligned_net_is_always_worse_than_the_gross_flip_by_exactly_the_cost(): + """Cost is a DRAG in both directions: aligned = sign*gross - cost, always.""" + gross, net = 0.000125, -0.001983 + cost = gross - net + assert cost > 0 + assert _aligned_net(net, gross, -1) == pytest.approx(-gross - cost, abs=1e-12) + + +def test_base_spread_reads_the_aligned_cost_subtracted_value_at_sign_minus_one(): + inputs = VerdictInputs( + expected_ic_sign=-1, + net_long_short_by_cost=((1.0, -0.001983), (2.0, -0.004091)), + gross_long_short_mean=0.000125, + ) + assert _base_spread(inputs) == pytest.approx(-0.002233, abs=1e-12) + + +def test_all_spreads_negative_is_true_when_every_aligned_scenario_is_negative(): + """sign=-1, gross known, cost subtracted -> every scenario negative -> the + Tradable axis has a KNOWN failure to convict on.""" + inputs = VerdictInputs( + expected_ic_sign=-1, + net_long_short_by_cost=((1.0, -0.001983), (2.0, -0.004091), (4.0, -0.008307)), + gross_long_short_mean=0.000125, + ) + assert _all_spreads_negative(inputs) is True + r = decide_verdict( + VerdictInputs( + expected_ic_sign=-1, + **_SAMPLE_OK, + **_TRADABLE, + net_long_short_by_cost=( + (1.0, -0.001983), (2.0, -0.004091), (4.0, -0.008307), + ), + gross_long_short_mean=0.000125, + ) + ) + assert r.tradable.verdict == AXIS_FAIL + assert any("EVERY cost scenario" in x for x in r.tradable.reasons) + + +def test_an_unknown_gross_makes_a_negative_sign_aligned_spread_unknown(): + """UNKNOWN NEVER CONVICTS, and never passes either: without the gross the cost + cannot be recovered from the net, so the aligned spread is not a fact.""" + inputs = VerdictInputs( + expected_ic_sign=-1, + net_long_short_by_cost=((1.0, -0.001983), (2.0, -0.004091)), + # gross_long_short_mean deliberately absent -> NaN + ) + assert math.isnan(_base_spread(inputs)) + assert _all_spreads_negative(inputs) is False + r = decide_verdict( + VerdictInputs( + expected_ic_sign=-1, + **_SAMPLE_OK, + **_TRADABLE, + net_long_short_by_cost=((1.0, -0.001983), (2.0, -0.004091)), + ) + ) + assert r.tradable.verdict == AXIS_INSUFFICIENT_DATA + + +def test_positive_sign_tradable_axis_is_unchanged_by_the_v08_fix(): + """The whole sign=+1 world must be bit-identical: same verdicts with the gross + supplied, absent, or nonsense — it is never consulted at sign=+1.""" + for gross in (0.06, float("nan"), -99.0): + r = decide_verdict( + _inputs(**_STRONG, **_OOS_OK, **_TRADABLE, gross_long_short_mean=gross) + ) + assert r.tradable.verdict == AXIS_PASS + kwargs = {**_STRONG, **_OOS_OK, **_TRADABLE} + kwargs["net_long_short_by_cost"] = ((1.0, -0.01), (2.0, -0.03)) + failed = decide_verdict(_inputs(**kwargs, gross_long_short_mean=gross)) + assert failed.tradable.verdict == AXIS_FAIL + + +# -------------------------------------------------------------------------- +# v0.8 fix #2: the Predictive axis gates on the PER-DATE monotonicity, and +# discloses when it had to fall back to the pooled one. +# -------------------------------------------------------------------------- + + +def test_predictive_gate_reads_the_per_date_monotonicity_over_the_pooled_one(): + """When both are present the per-date figure decides — and it decides BOTH + ways, so this is not just 'the new field is accepted'.""" + # pooled says REVERSED, per-date says correctly ordered -> the axis is not + # convicted by the magnitude-sensitive statistic. + rescued = decide_verdict( + _inputs( + **{**_STRONG, "monotonicity_spearman": -0.5, + "monotonicity_spearman_by_date": 0.7}, + **_OOS_OK, + ) + ) + assert rescued.predictive.verdict == AXIS_PASS + # pooled looks fine, per-date says REVERSED -> FAIL. The new gate has teeth. + convicted = decide_verdict( + _inputs( + **{**_STRONG, "monotonicity_spearman": 0.9, + "monotonicity_spearman_by_date": -0.3}, + **_OOS_OK, + ) + ) + assert convicted.predictive.verdict == AXIS_FAIL + assert any("monotonicity" in x for x in convicted.predictive.reasons) + + +def test_a_missing_per_date_monotonicity_falls_back_and_says_so(): + """A pre-v0.8 IR stays judgeable, but the report must not pretend the weaker + statistic is the new one.""" + r = decide_verdict(_inputs(**_STRONG, **_OOS_OK)) # pooled only + assert r.predictive.verdict == AXIS_PASS + assert any("FELL BACK" in x for x in r.predictive.reasons) + assert any("magnitude-sensitive" in x for x in r.predictive.reasons) + + +def test_no_fallback_note_when_the_per_date_monotonicity_is_supplied(): + r = decide_verdict( + _inputs(**{**_STRONG, "monotonicity_spearman_by_date": 0.7}, **_OOS_OK) + ) + assert r.predictive.verdict == AXIS_PASS + assert not any("FELL BACK" in x for x in r.predictive.reasons) + + +def test_the_negative_sign_monotonicity_gate_is_still_direction_aligned(): + """v0.7's direction gate is unchanged: sign=-1 wants a NEGATIVE raw per-date + monotonicity, and a positive one is the reversal that FAILs.""" + common = dict( + expected_ic_sign=-1, **_SAMPLE_OK, **_OOS_OK, + ic_ir=-0.8, ic_ir_ci_low=-1.05, ic_ir_ci_high=-0.55, + ic_win_rate=0.7, ic_nw_t=-3.5, + ) + ok = decide_verdict( + VerdictInputs(**common, monotonicity_spearman_by_date=-0.7) + ) + assert ok.predictive.verdict == AXIS_PASS + reversed_ = decide_verdict( + VerdictInputs(**common, monotonicity_spearman_by_date=0.7) + ) + assert reversed_.predictive.verdict == AXIS_FAIL diff --git a/tests/test_factor_eval_standard.py b/tests/test_factor_eval_standard.py index 499e604..b1c76bd 100644 --- a/tests/test_factor_eval_standard.py +++ b/tests/test_factor_eval_standard.py @@ -52,6 +52,8 @@ newey_west_lag, newey_west_t, sortino, + spearman, + spearman_by_date, ) from analytics.factor import compute_ic from factors.spec import FactorSpec @@ -1896,3 +1898,163 @@ def test_factors_layer_never_imports_analytics(): ) ] assert offenders == [] + + +# -------------------------------------------------------------------------- +# v0.8 fix #2: monotonicity_spearman_by_date — a RANK statistic for a rank axis. +# -------------------------------------------------------------------------- + + +def _quantile_frame(rows: list[list[float]]) -> pd.DataFrame: + """dates x buckets(1..N), the shape ``quantile_return_matrix`` produces.""" + return pd.DataFrame( + rows, + index=pd.bdate_range("2024-01-02", periods=len(rows), name=DATE), + columns=pd.Index(range(1, len(rows[0]) + 1), name="quantile"), + ) + + +def test_per_date_monotonicity_survives_the_outlier_days_that_flip_the_pooled_one(): + """THE proof of the v0.8 monotonicity fix. + + Every single date orders its buckets EXACTLY as the hypothesis says (low + bucket -> low return, monotone up). But on two days the market only realizes + returns in the bottom three buckets, and those returns are ~1000x the normal + scale. Those two days dominate the cross-date ARITHMETIC MEAN of buckets 1-3 + and drag them above buckets 4-5 — so the POOLED Spearman reads the factor as + REVERSED even though no individual day ever was. + + This is the PR-K / PR-G "smile" shape (one outlier bucket driving the pooled + figure) reproduced in miniature, and it is why a rank-based Predictive axis + must not be gated on a magnitude-sensitive statistic. + """ + normal = [0.001, 0.002, 0.003, 0.004, 0.005] + outlier = [1.0, 1.1, 1.2, float("nan"), float("nan")] + frame = _quantile_frame([normal] * 20 + [outlier] * 2) + + bucket_means = [frame[q].mean() for q in frame.columns] + pooled = spearman([float(q) for q in frame.columns], bucket_means) + by_date = spearman_by_date(frame) + + assert pooled == pytest.approx(-0.5) # REVERSED — the defect + assert by_date == pytest.approx(1.0) # every day was perfectly ordered + # and the direction gate (v0.7, threshold 0.0) reads them oppositely + assert pooled < 0.0 < by_date + + +def test_per_date_monotonicity_matches_the_pooled_one_on_clean_monotone_data(): + """No outliers, no gaps: the two statistics must agree, so the v0.8 field is a + ROBUSTNESS change, not a different measurement.""" + frame = _quantile_frame([[0.001, 0.002, 0.003, 0.004, 0.005]] * 12) + bucket_means = [frame[q].mean() for q in frame.columns] + assert spearman([float(q) for q in frame.columns], bucket_means) == pytest.approx(1.0) + assert spearman_by_date(frame) == pytest.approx(1.0) + # and it reads a cleanly REVERSED panel as -1.0, not as "no opinion" + assert spearman_by_date(_quantile_frame([[0.005, 0.004, 0.003, 0.002, 0.001]] * 12)) \ + == pytest.approx(-1.0) + + +def test_a_date_with_fewer_than_three_finite_buckets_is_skipped_not_counted(): + """Two points always give a perfect +/-1 rank correlation, which would be pure + noise injected at full weight. Such a date contributes NOTHING — not a zero.""" + good = [0.001, 0.002, 0.003, 0.004, 0.005] + thin = [0.005, 0.004, float("nan"), float("nan"), float("nan")] # 2 buckets, reversed + with_thin = spearman_by_date(_quantile_frame([good] * 6 + [thin] * 3)) + without_thin = spearman_by_date(_quantile_frame([good] * 6)) + assert with_thin == pytest.approx(without_thin) == pytest.approx(1.0) + # a three-bucket date, by contrast, IS counted + three = [0.005, 0.004, 0.003, float("nan"), float("nan")] # reversed + assert spearman_by_date(_quantile_frame([good] * 3 + [three] * 3)) == pytest.approx(0.0) + + +def test_per_date_monotonicity_is_unknown_rather_than_zero_when_no_date_qualifies(): + nan = float("nan") + assert math.isnan(spearman_by_date(_quantile_frame([[0.1, 0.2, nan, nan, nan]] * 5))) + # all buckets tied on every date: Spearman undefined, so UNKNOWN, not 0.0 + assert math.isnan(spearman_by_date(_quantile_frame([[0.2] * 5] * 5))) + assert math.isnan(spearman_by_date(pd.DataFrame())) + + +def test_return_risk_reports_both_monotonicity_statistics(rng): + """The pooled field is KEPT (historical comparability) alongside the new one.""" + panel, factor = planted_signal_setup(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + payload = report.by_name()["return_risk"].payload + assert payload["monotonicity_spearman"] == pytest.approx(1.0) + by_date = payload["monotonicity_spearman_by_date"] + assert math.isfinite(by_date) and -1.0 <= by_date <= 1.0 + + +# -------------------------------------------------------------------------- +# v0.8 fix #1 at the standard layer: the aligned spread series/CI. +# -------------------------------------------------------------------------- + + +def _aligned_mean_from_ci(payload) -> float: + """The aligned base spread's MEAN, recovered from its symmetric N_eff CI.""" + return (payload["net_long_short_base_ci_low"] + payload["net_long_short_base_ci_high"]) / 2.0 + + +def test_aligned_base_spread_subtracts_cost_after_flipping_the_legs(rng): + """sign=-1: aligned = -gross - cost, NOT -(gross - cost) = -net. + + Recovered from the payload's own numbers, so this pins the real evaluator + output rather than a re-derivation of the formula. + """ + panel, factor = planted_signal_setup(rng, sign=-1) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(expected_ic_sign=-1), make_cfg(), + EvalContext(price_panel=panel, fee_rate=0.001), + ) + payload = report.by_name()["return_risk"].payload + gross = payload["gross_long_short_mean"] + net = payload["net_long_short_by_cost"][1.0] + cost = gross - net + assert cost > 0 # trading is never free + aligned = _aligned_mean_from_ci(payload) + assert aligned == pytest.approx(-gross - cost, rel=1e-9) + # the pre-v0.8 reading handed the cost back as profit; it must be exactly + # 2 * cost better than the truth, and must NOT be what we report. + assert aligned == pytest.approx(-net - 2 * cost, rel=1e-9) + assert aligned < -net + + +def test_aligned_base_spread_is_bit_identical_to_the_net_at_positive_sign(rng): + """The sign=+1 regression lock: aligned mean == net mean, exactly.""" + panel, factor = planted_signal_setup(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), + EvalContext(price_panel=panel, fee_rate=0.001), + ) + payload = report.by_name()["return_risk"].payload + assert _aligned_mean_from_ci(payload) == pytest.approx( + payload["net_long_short_by_cost"][1.0], rel=1e-12 + ) + + +def test_a_higher_fee_makes_the_negative_sign_aligned_spread_worse_not_better(rng): + """The behavioural signature of the defect, at the standard layer. + + The SAME trades at a higher fee must degrade the reported aligned performance. + Under the pre-v0.8 `sign * net` the extra fee was ADDED to a sign=-1 factor's + aligned spread, so raising the fee IMPROVED its reported NAV — a factor could + be made to look better simply by charging it more to trade. + """ + panel, factor = planted_signal_setup(rng, sign=-1) + spec, cfg = make_spec(expected_ic_sign=-1), make_cfg() + cheap = StandardFactorEvaluator().evaluate( + factor, spec, cfg, EvalContext(price_panel=panel, fee_rate=0.0005) + ).by_name()["return_risk"].payload + dear = StandardFactorEvaluator().evaluate( + factor, spec, cfg, EvalContext(price_panel=panel, fee_rate=0.004) + ).by_name()["return_risk"].payload + + # same trades either way: the gross leg difference is untouched by the fee + assert dear["gross_long_short_mean"] == pytest.approx( + cheap["gross_long_short_mean"], rel=1e-12 + ) + assert _aligned_mean_from_ci(dear) < _aligned_mean_from_ci(cheap) + assert dear["aligned_spread_final_nav"] < cheap["aligned_spread_final_nav"] + assert dear["aligned_spread_annual_return"] < cheap["aligned_spread_annual_return"] From 37a70da10b99692145bab2bd0ba35df843958c57 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Tue, 21 Jul 2026 01:54:47 -0700 Subject: [PATCH 2/3] docs(analytics): sync VERDICT_KEYS return_risk with the v0.8 verdict inputs The verdict now reads monotonicity_spearman_by_date (the gated statistic) and gross_long_short_mean (needed to align a net spread), but the documentation dict still listed only the two pre-v0.8 keys. Documentation-only: VERDICT_KEYS is re-exported and referenced in docstrings, never used for runtime validation. --- analytics/eval/sections.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/analytics/eval/sections.py b/analytics/eval/sections.py index 3e63fc3..a4664ea 100644 --- a/analytics/eval/sections.py +++ b/analytics/eval/sections.py @@ -54,9 +54,24 @@ "ic_win_rate", "ic_nw_t", ), - # monotonicity_spearman: RAW Spearman(bucket index, bucket mean return). + # monotonicity_spearman: RAW Spearman(bucket index, bucket mean return) over + # the POOLED (date-equal-weighted) bucket means — unbounded and magnitude + # sensitive, so (v0.8) it is REPORTED but no longer gates. + # monotonicity_spearman_by_date: (v0.8) the gated one — Spearman per date + # (bounded [-1, 1] before averaging, structurally parallel to the rank IC), + # then averaged. The verdict falls back to the pooled field when it is + # absent and SAYS SO in its reasons. # net_long_short_by_cost: {cost multiplier: net (top - bottom) return}, RAW. - "return_risk": ("monotonicity_spearman", "net_long_short_by_cost"), + # gross_long_short_mean: (v0.8) the GROSS (pre-cost) leg difference — the + # verdict needs it to align a net spread, since aligning is + # `sign * gross - cost` and cost = gross - net cannot be recovered from the + # net alone. + "return_risk": ( + "monotonicity_spearman", + "monotonicity_spearman_by_date", + "net_long_short_by_cost", + "gross_long_short_mean", + ), # The Incremental axis (design §6, v0.5). known_factors_supplied = was a # known-factor BOOK supplied at all (False -> the axis is NOT_ASSESSED); # incremental_ic_ir / incremental_ic_mean = mean(orthIC)/std(orthIC) and From 2c128b3b4082364e7a736f60a7166625f0be540e Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Tue, 21 Jul 2026 06:28:26 -0700 Subject: [PATCH 3/3] =?UTF-8?q?fix(analytics):=20contract=20v0.9=20?= =?UTF-8?q?=E2=80=94=20gate=20monotonicity=20direction=20on=20its=20N=5Fef?= =?UTF-8?q?f=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v0.8 moved the Predictive axis' monotonicity gate onto a per-date rank statistic — the right KIND of statistic for a rank axis. Re-running the eleven factors then showed that statistic is heavily attenuated by daily noise: a constructively perfect quantile ladder scores 1.000, while the best real factors score 0.045-0.106 and two of them sit 0.021 apart across a gate that was still a bare 0.0 with no dispersion estimate anywhere in the decision. Comparing a noise-compressed mean against zero is a coin flip wearing a criterion's clothes — the same defect v0.6 fixed for the ICIR by gating on its N_eff lower bound. The direction is now decided by the N_eff-based CI of the per-date series and is THREE-VALUED (user-chosen semantics): aligned CI low > bar -> direction HOLDS (may PASS) aligned CI high < 0 -> direction CONTRADICTED (FAIL) otherwise -> UNKNOWN (neither convicts nor acquits) UNKNOWN NEVER RESCUES. It is evaluated AFTER every other predictive criterion: if ICIR / NW-t / win rate / out-of-sample sign consistency fail, the axis still FAILs and the unknown direction is merely disclosed alongside. Only when everything else has cleared and the direction ALONE cannot be asserted does the axis become INSUFFICIENT_DATA. "We could not tell" withholds a PASS; it never withholds a FAIL that other evidence independently earned — the same asymmetry as "unknown never convicts", read from the other side. The FAIL side stays pinned at 0 rather than tracking the configurable bar: "above the bar" is the positive claim and moves with the bar, but "below zero" is REVERSAL, a fact about the sign. Pinning FAIL to a raised bar would convict a correctly-ordered-but-weak factor of being reversed. Reuses the existing machinery rather than growing a parallel one: the CI is the same mean_ci() v0.6 built for the ICIR, and the aligned bounds come from a single _aligned_bounds() that _aligned_lower_bound() now delegates to, so the ICIR gate and the monotonicity gate cannot drift apart on the min/max swap at sign=-1. Three fallback levels, each DISCLOSED in the axis reasons: CI -> the bare per-date point (v0.8, two-valued: with no dispersion estimate there is nothing to be uncertain with, and inventing an UNKNOWN here would silently upgrade every pre-v0.9 FAIL to INSUFFICIENT_DATA) -> the pooled magnitude statistic (v0.7, wording kept verbatim). Levels 1 and 2 are worded differently on purpose: one degrades WHICH STATISTIC, the other WHICH ESTIMATOR OF ITS UNCERTAINTY, and a reader must be able to tell which happened. spearman_by_date is refactored into the mean of a new spearman_series_by_date and is BIT-IDENTICAL to v0.8 — the mean deliberately keeps the sequential sum()/len() rather than Series.mean(), since the two can differ in the last bit and this is a published, cross-run-comparable field. Locked against the v0.8 body frozen in the test as an oracle. Mutation-tested (standing rule after a prior PR shipped a toothless invariance test): reverting the gate to the bare-sign form fails 8 of the 8 new verdict tests plus the v0.8 fallback-disclosure test; zero-filling skipped dates instead of omitting them fails 5; swapping the sequential sum for Series.mean() fails the bit-identity test alone. Stage A only — no factor evaluation was re-run. Gates: pytest 1682 passed (1668 baseline + 14 new, no pre-existing expectation changed), ruff clean, phase0 ic 0.9600 / annual 0.8408 unchanged, secret scan 0 hits. --- analytics/eval/report.py | 9 + analytics/eval/sections.py | 7 + analytics/eval/standard.py | 37 +++- analytics/eval/stats.py | 94 +++++++-- analytics/eval/verdict.py | 293 ++++++++++++++++++++++++----- tests/test_factor_eval_contract.py | 250 +++++++++++++++++++++++- tests/test_factor_eval_standard.py | 152 +++++++++++++++ 7 files changed, 768 insertions(+), 74 deletions(-) diff --git a/analytics/eval/report.py b/analytics/eval/report.py index f329b99..db4e66f 100644 --- a/analytics/eval/report.py +++ b/analytics/eval/report.py @@ -128,6 +128,15 @@ def payload_of(name: str) -> Mapping[str, object]: monotonicity_spearman_by_date=_payload_number( returns, "monotonicity_spearman_by_date", float("nan") ), + # design §6 v0.9: the per-date monotonicity's N_eff-based CI, which is what + # the DIRECTION gate reads. Absent (a pre-v0.9 IR) -> NaN -> the verdict + # reverts to the bare point (v0.8 behaviour) and DISCLOSES that too. + monotonicity_spearman_by_date_ci_low=_payload_number( + returns, "monotonicity_spearman_by_date_ci_low", float("nan") + ), + monotonicity_spearman_by_date_ci_high=_payload_number( + returns, "monotonicity_spearman_by_date_ci_high", float("nan") + ), # Incremental axis facts (design §6, v0.5). A Skipped purity section (no # book) leaves the payload empty -> known_factors_supplied defaults False # -> the axis is NOT_ASSESSED. A supplied-but-unmeasurable orthogonalized diff --git a/analytics/eval/sections.py b/analytics/eval/sections.py index a4664ea..05d071c 100644 --- a/analytics/eval/sections.py +++ b/analytics/eval/sections.py @@ -61,6 +61,11 @@ # (bounded [-1, 1] before averaging, structurally parallel to the rank IC), # then averaged. The verdict falls back to the pooled field when it is # absent and SAYS SO in its reasons. + # monotonicity_spearman_by_date_ci_low/high: (v0.9) the N_eff-based 95% CI of + # that per-date series — what the DIRECTION gate now reads. Three-valued: + # aligned CI above the bar = direction holds; entirely below 0 = contradicted + # (FAIL); straddling 0 = UNKNOWN, which neither convicts nor acquits. The bare + # point is only the fallback, disclosed as such. # net_long_short_by_cost: {cost multiplier: net (top - bottom) return}, RAW. # gross_long_short_mean: (v0.8) the GROSS (pre-cost) leg difference — the # verdict needs it to align a net spread, since aligning is @@ -69,6 +74,8 @@ "return_risk": ( "monotonicity_spearman", "monotonicity_spearman_by_date", + "monotonicity_spearman_by_date_ci_low", + "monotonicity_spearman_by_date_ci_high", "net_long_short_by_cost", "gross_long_short_mean", ), diff --git a/analytics/eval/standard.py b/analytics/eval/standard.py index c07b1c4..4c60b52 100644 --- a/analytics/eval/standard.py +++ b/analytics/eval/standard.py @@ -61,11 +61,12 @@ half_life, hypothesis_win_rate, information_ratio_ci, + mean_by_date_spearman, mean_ci, newey_west_t, sortino, spearman, - spearman_by_date, + spearman_series_by_date, ) from analytics.factor import ic_summary from analytics.performance import performance_summary @@ -315,6 +316,11 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: top, bottom = ir.cfg.long_short buckets = list(quantiles.columns) + # The PER-DATE monotonicity series, built ONCE (design §6, v0.9): the + # payload's point figure and its CI are then two reductions of the SAME + # observations, and the panel is walked once instead of twice. + mono_by_date = spearman_series_by_date(quantiles) + bucket_mean = {int(q): as_float(quantiles[q].mean()) for q in buckets} bucket_nav = { int(q): as_float((1.0 + quantiles[q].dropna()).prod()) for q in buckets @@ -363,13 +369,40 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: # buckets is capped in [-1, 1] BEFORE averaging across dates, exactly # like the rank IC. Robust to a few extreme-return days concentrating # in one bucket. RAW; the verdict applies expected_ic_sign. - "monotonicity_spearman_by_date": spearman_by_date(quantiles), + # + # (v0.9) The POINT is kept — reported, cross-run comparable — but it is + # no longer what decides the direction on its own; the CI attached below + # is. Identical value to v0.8 by construction (same reduction, same + # series). + "monotonicity_spearman_by_date": mean_by_date_spearman(mono_by_date), "gross_long_short_mean": as_float(gross.mean()), "net_long_short_by_cost": net_by_cost, "net_long_short_cumulative_by_cost": cumulative_by_cost, "fee_rate_base": ir.ctx.fee_rate, "periods_per_year": ir.periods_per_year, } + # N_eff-based CI of the PER-DATE monotonicity (design §6, v0.9) — the same + # mean_ci the ICIR uses, on the per-date rank series instead of the IC + # series. THIS is what the Predictive axis' direction gate reads: the bare + # point carries no dispersion estimate at all, and this project's real runs + # put a perfect quantile ladder at only ~0.05-0.11 on it, so "point > 0.0" + # was a coin flip dressed as a criterion. The CI turns the gate three-valued + # (holds / contradicted / UNKNOWN) — see verdict._monotonicity_direction. + mono_ci = mean_ci(mono_by_date, confidence=DEFAULT_CONFIDENCE) + payload["monotonicity_spearman_by_date_se"] = mono_ci["se"] + payload["monotonicity_spearman_by_date_ci_low"] = mono_ci["ci_low"] + payload["monotonicity_spearman_by_date_ci_high"] = mono_ci["ci_high"] + payload["monotonicity_spearman_by_date_n_eff"] = mono_ci["n_eff"] + payload["monotonicity_spearman_by_date_n_dates"] = int(mono_by_date.size) + payload["monotonicity_spearman_by_date_ci_note"] = ( + "N_eff-based 95% CI of the per-date monotonicity (design §6, v0.9). The " + "Predictive axis gates the monotonicity DIRECTION on this interval, not " + "on the point: aligned CI entirely above the bar = direction holds; " + "entirely below 0 = contradicted (FAIL); straddling 0 = UNKNOWN, which " + "neither convicts nor acquits. n_dates counts the dates that qualified " + "(< 3 finite buckets or all-tied dates are skipped, not zero-filled)." + ) + # N_eff-based CI of the base-cost QN-Q1 spread (design §6, v0.6). REPORTED # for the reader (b); the Tradable axis still gates on the POINT base spread # > 0 (the CI on this leg difference is point-only for now — c names only diff --git a/analytics/eval/stats.py b/analytics/eval/stats.py index 7c792f9..590a1b6 100644 --- a/analytics/eval/stats.py +++ b/analytics/eval/stats.py @@ -393,6 +393,70 @@ def spearman(x: pd.Series | list, y: pd.Series | list) -> float: return float(pair["x"].corr(pair["y"], method="spearman")) +def spearman_series_by_date( + quantile_returns: pd.DataFrame, min_buckets: int = 3 +) -> pd.Series: + """PER-DATE Spearman(bucket index, bucket return), one value per QUALIFYING date. + + The series :func:`spearman_by_date` averages, exposed in its own right (design + §6, v0.9) so the verdict can put a DISPERSION estimate on that average. A bare + cross-date mean of per-date rank correlations is heavily attenuated by daily + noise — an empirically perfect quantile ladder scores ~0.05-0.11 in this + project's real runs — so gating a direction claim on whether that point happens + to land above 0.0 reads noise as evidence. With the series in hand the caller + can hand it to :func:`mean_ci` and gate on the N_eff-based interval instead. + + A date is SKIPPED — ABSENT FROM THE SERIES, never a zero and never a NaN row — + when it carries fewer than ``min_buckets`` finite bucket returns (too few points + for a meaningful rank correlation), or when its surviving buckets are all tied + (Spearman undefined there). NO date qualifying yields an EMPTY series, not a + series of NaN: "no observation" and "an unknown observation" are different + facts, and only the first is true here. + + Because skipped dates are absent rather than held as holes, the index is the + SURVIVING date grid. Downstream N_eff (via :func:`mean_ci`) is therefore read + off that compacted grid — the same convention :func:`mean_ci` already applies to + the IC series, whose degenerate cross-sections it drops before measuring + autocorrelation. Noted rather than hidden (design §11). + + RAW, like :func:`spearman`: the hypothesis sign is applied by the verdict. + """ + dates: list[object] = [] + daily: list[float] = [] + if not (quantile_returns.empty or quantile_returns.shape[1] < min_buckets): + index = [float(q) for q in quantile_returns.columns] + for date, row in quantile_returns.iterrows(): + values = [float(v) for v in row.to_numpy(dtype=float)] + if sum(1 for v in values if math.isfinite(v)) < min_buckets: + continue + # spearman() drops the non-finite pairs itself, keeping each surviving + # bucket paired with its OWN index (not a renumbered 1..k). + rho = spearman(index, values) + if math.isfinite(rho): + dates.append(date) + daily.append(rho) + name = quantile_returns.index.name if isinstance(quantile_returns, pd.DataFrame) else None + return pd.Series(daily, index=pd.Index(dates, name=name), dtype=float) + + +def mean_by_date_spearman(daily: pd.Series) -> float: + """The cross-date average :func:`spearman_by_date` reduces its series with. + + Exposed so a caller that already built the per-date series (the standard layer + does, to attach a CI to it) gets the POINT from the very same expression rather + than paying for a second pass over the panel — the point and the interval can + then never disagree about which observations they describe. + + Deliberately the sequential Python ``sum(...) / len(...)`` of the pre-v0.9 + implementation, not ``Series.mean()``: the two can differ in the last bit, and + this figure is a published, cross-run-comparable report field. + """ + values = daily.tolist() + if not values: + return float("nan") + return float(sum(values) / len(values)) + + def spearman_by_date( quantile_returns: pd.DataFrame, min_buckets: int = 3 ) -> float: @@ -405,29 +469,17 @@ def spearman_by_date( sensitive — one outlier bucket-day can flip it while the daily-capped rank IC barely moves. - A date is SKIPPED — contributing nothing, never a zero — when it carries fewer - than ``min_buckets`` finite bucket returns (too few points for a meaningful - rank correlation), or when its surviving buckets are all tied (Spearman - undefined there). NaN when NO date qualifies: unknown, not 0.0. + As of v0.9 this is the mean of :func:`spearman_series_by_date` (see there for + the skip rules) — a pure refactor: the returned float is BIT-IDENTICAL to the + v0.8 implementation, which is asserted against the v0.8 fixtures. + + NaN when NO date qualifies: unknown, not 0.0. RAW, like :func:`spearman`: the hypothesis sign is applied by the verdict. """ - if quantile_returns.empty or quantile_returns.shape[1] < min_buckets: - return float("nan") - index = [float(q) for q in quantile_returns.columns] - daily: list[float] = [] - for _, row in quantile_returns.iterrows(): - values = [float(v) for v in row.to_numpy(dtype=float)] - if sum(1 for v in values if math.isfinite(v)) < min_buckets: - continue - # spearman() drops the non-finite pairs itself, keeping each surviving - # bucket paired with its OWN index (not a renumbered 1..k). - rho = spearman(index, values) - if math.isfinite(rho): - daily.append(rho) - if not daily: - return float("nan") - return float(sum(daily) / len(daily)) + return mean_by_date_spearman( + spearman_series_by_date(quantile_returns, min_buckets=min_buckets) + ) def as_float(value: object) -> float: @@ -446,10 +498,12 @@ def as_float(value: object) -> float: "half_life", "hypothesis_win_rate", "information_ratio_ci", + "mean_by_date_spearman", "mean_ci", "newey_west_lag", "newey_west_t", "sortino", "spearman", "spearman_by_date", + "spearman_series_by_date", ] diff --git a/analytics/eval/verdict.py b/analytics/eval/verdict.py index 2ee97ad..4ffca79 100644 --- a/analytics/eval/verdict.py +++ b/analytics/eval/verdict.py @@ -118,6 +118,23 @@ (INSUFFICIENT below it); the lower-CI test asks "is it convincingly above the bar?" (PASS above it). FAIL stays POINT-based and known-fact-only; a NaN CI bound never convicts. + +THE MONOTONICITY DIRECTION IS THREE-VALUED (v0.9). v0.8 moved the monotonicity gate +onto a per-date rank statistic, which was the right KIND of statistic — and then the +eleven-factor re-run showed that statistic is heavily attenuated by daily noise: an +empirically perfect quantile ladder scores 0.045-0.106 on it, while the direction +gate still sat at a bare 0.0 with no dispersion estimate anywhere. Two factors +landed 0.021 apart across that boundary. So the direction is now decided by the +N_eff-based CI of the per-date series (:func:`_monotonicity_direction`), and the +answer can be UNKNOWN — CI straddles 0 — as well as holds/contradicted. + + UNKNOWN NEVER RESCUES. An unknown direction is checked AFTER every other + predictive criterion: if ICIR / NW-t / win rate / OOS consistency fail, the axis + still FAILs and the unknown is merely disclosed. Only when everything else has + cleared and the direction ALONE cannot be asserted does the axis become + INSUFFICIENT_DATA. "We could not tell" withholds a PASS; it never withholds a + FAIL that was independently earned. This is the same asymmetry as UNKNOWN NEVER + CONVICTS above, read from the other side. """ from __future__ import annotations @@ -240,6 +257,11 @@ class VerdictThresholds: # monotone". A 0.8 strength bar rejected tail-concentrated real factors whose # power sits in one extreme (e.g. jump-amount-corr: ICIR -0.40, NW-t -14.8, yet # Q3-humped). Raise it to demand strength. UNVALIDATED (§11). + # (v0.9) The bar is now compared against the per-date monotonicity's aligned CI + # LOWER bound, not its point — so it is the bar a direction claim must be + # CONVINCINGLY above. The FAIL side stays pinned at 0 (reversal is a fact about + # the sign, not about how strong this threshold was set); see + # _monotonicity_direction. min_monotonicity_spearman: float = 0.0 def __post_init__(self) -> None: @@ -326,7 +348,19 @@ class VerdictInputs: #: averaging (structurally parallel to the rank IC). RAW. NaN = UNKNOWN, which #: falls back to ``monotonicity_spearman`` with the substitution DISCLOSED in #: the axis reasons, so an IR built before v0.8 stays judgeable. + #: (v0.9) This POINT is now only the FALLBACK for the direction gate; the CI + #: below is what decides when it is available. monotonicity_spearman_by_date: float = float("nan") + #: return_risk: 95% CI bounds of ``monotonicity_spearman_by_date``, computed + #: with the per-date series' OWN N_eff (design §6, v0.9 — the same ``mean_ci`` + #: the ICIR uses). RAW (the verdict applies ``expected_ic_sign``). This is what + #: the monotonicity DIRECTION gate reads: the bare point carries no dispersion + #: estimate, and a cross-date mean of per-date rank correlations is so + #: attenuated by daily noise that "point > 0.0" was a coin flip dressed as a + #: criterion. NaN = UNKNOWN -> the gate falls back to the point (v0.8 behaviour) + #: with the fallback DISCLOSED. + monotonicity_spearman_by_date_ci_low: float = float("nan") + monotonicity_spearman_by_date_ci_high: float = float("nan") oos_available: bool = False # oos_generalization oos_sign_consistent: bool = False # expected sign holds in BOTH subperiods oos_sign_flipped: bool = False # explicit flip -> Predictive FAIL @@ -458,17 +492,31 @@ def _clears_level(point_estimate: float, threshold: float) -> bool: return math.isfinite(point_estimate) and point_estimate > threshold +def _aligned_bounds(sign: int, ci_low: float, ci_high: float) -> tuple[float, float]: + """The CI of ``sign * estimate``: ``(lower, upper)`` in the EXPECTED direction. + + ``ci_low``/``ci_high`` bracket the RAW estimate. Multiplying an INTERVAL by a + negative number REVERSES it, so the endpoints are re-sorted rather than kept in + place — the min/max swap at ``sign = -1`` is the whole content of this helper, + and doing it in one place is why the ICIR gate and the (v0.9) monotonicity gate + cannot drift apart on the convention. + + ``(NaN, NaN)`` when either bound is unknown — the caller then treats the + estimate as not-convincingly-clearing, and never as a FAIL. + """ + if not (math.isfinite(ci_low) and math.isfinite(ci_high)): + nan = float("nan") + return nan, nan + return min(sign * ci_low, sign * ci_high), max(sign * ci_low, sign * ci_high) + + def _aligned_lower_bound(sign: int, ci_low: float, ci_high: float) -> float: """The lower CI bound of ``sign * estimate`` (the bound in the EXPECTED direction). - ``ci_low``/``ci_high`` bracket the RAW estimate; multiplying the interval by the - hypothesis sign and taking the smaller endpoint yields the worst-case value in - the direction the factor claims. NaN when either bound is unknown — the caller - then treats the estimate as not-convincingly-clearing (never a FAIL). + The worst-case value in the direction the factor claims — what a POSITIVE claim + (an axis PASS) must clear. NaN when either bound is unknown. """ - if not (math.isfinite(ci_low) and math.isfinite(ci_high)): - return float("nan") - return min(sign * ci_low, sign * ci_high) + return _aligned_bounds(sign, ci_low, ci_high)[0] # -- shared fact readers ------------------------------------------------- @@ -578,42 +626,147 @@ def _all_spreads_negative(inputs: VerdictInputs) -> bool: return bool(known) and all(a <= 0 for a in known) -def _gated_monotonicity(inputs: VerdictInputs) -> tuple[float, bool]: - """The RAW monotonicity the Predictive axis gates on + whether it FELL BACK. +# -- the (v0.9) three-valued monotonicity DIRECTION gate ------------------ +# +# Monotonicity is a DIRECTION gate, not a strength gate (v0.7) — but v0.8 revealed +# that its statistic cannot support even that claim as a bare point. Averaging +# per-date rank correlations across dates is heavily ATTENUATED by daily noise: in +# this project's real eleven-factor runs an empirically perfect quantile ladder +# scores only 0.045-0.106, and two factors sat 0.021 apart across a threshold of +# exactly 0.0 with NO dispersion estimate anywhere in the decision. "Point > 0.0" +# on a noisy mean is a coin flip wearing a criterion's clothes — the same defect +# v0.6 fixed for the ICIR by gating on its N_eff lower bound. +# +# So the direction is decided by the CI, and it is THREE-VALUED: +# +# aligned CI low > bar -> HOLDS (may PASS) +# aligned CI high < 0 -> CONTRADICTED (FAIL — a measured reversal) +# otherwise -> UNKNOWN (neither convicts nor acquits) +# +# WHY THE CONTRADICTED TEST IS PINNED AT 0 AND NOT AT THE BAR: they are different +# claims. "Above the bar" is the positive claim, so it moves with a configurable +# (possibly strength-demanding) bar. "Below zero" is REVERSAL — the buckets are +# ordered the wrong way round — which is a fact about the sign, not about how +# strong the project decided to be. Pinning FAIL to a raised bar would convict a +# correctly-ordered-but-weak factor of being reversed, which it is not. + +MONO_HOLDS = "HOLDS" +MONO_CONTRADICTED = "CONTRADICTED" +MONO_UNKNOWN = "UNKNOWN" + +#: Level-2 fallback disclosure: judged on the POOLED magnitude statistic (v0.7 +#: behaviour). Wording kept verbatim from v0.8 — reports and tests read it. +_MONO_POOLED_FALLBACK_NOTE = ( + "monotonicity gate FELL BACK to the pooled 'monotonicity_spearman': " + "the per-date figure ('monotonicity_spearman_by_date', design §6 v0.8) " + "was not supplied or not measurable. The pooled statistic is unbounded " + "and magnitude-sensitive, so this monotonicity judgement is weaker " + "than a v0.8 run's." +) + +#: Level-1 fallback disclosure: the per-date statistic IS available but its CI is +#: not, so the gate reverts to the bare sign of the point (v0.8 behaviour). +#: DELIBERATELY NOT the pooled note's wording — the two are different degradations +#: (which STATISTIC vs which ESTIMATOR OF ITS UNCERTAINTY) and a reader must be +#: able to tell which one happened. +_MONO_POINT_FALLBACK_NOTE = ( + "monotonicity gate REVERTED to the BARE per-date POINT (v0.8 behaviour): no " + "'monotonicity_spearman_by_date_ci_low/high' was supplied, so the direction was " + "decided WITHOUT any dispersion estimate. A cross-date mean of per-date rank " + "correlations is strongly attenuated by daily noise, so a point that merely " + "lands on the right side of the bar is NOT the same evidence a v0.9 CI would be." +) - v0.8 gates on ``monotonicity_spearman_by_date`` (per-date Spearman, each capped - in [-1, 1] before averaging) rather than the pooled ``monotonicity_spearman``: - the Predictive axis is a RANK claim, and the pooled statistic is an unbounded - MAGNITUDE statistic — a few extreme return days landing in one bucket can flip - it while the daily-capped rank IC barely moves. Gating a rank axis on a - magnitude statistic is the category error being corrected. - Returns ``(value, used_fallback)``. When the per-date figure is UNKNOWN (an IR - produced before v0.8, or too few finite buckets on every date) the pooled field - is used instead and ``used_fallback`` is True, so the caller can DISCLOSE the - substitution rather than silently judge on the weaker statistic. +def _monotonicity_direction( + inputs: VerdictInputs, thr: VerdictThresholds +) -> tuple[str, tuple[str, ...]]: + """Three-valued monotonicity DIRECTION + its disclosure notes (design §6, v0.9). + + Returns ``(MONO_HOLDS | MONO_CONTRADICTED | MONO_UNKNOWN, notes)``. + + THE FALLBACK CHAIN, each level DISCLOSED in ``notes`` (never silent): + 1. the per-date CI (v0.9) — three-valued, the intended path. + 2. the per-date POINT (v0.8) — CI absent. TWO-valued, bit-for-bit the v0.8 + rule: aligned point above the bar HOLDS, anything else (including an + unknown) CONTRADICTS. It cannot yield UNKNOWN: without a dispersion + estimate there is nothing to be uncertain WITH, and inventing an UNKNOWN + here would silently convert every pre-v0.9 FAIL into INSUFFICIENT_DATA. + 3. the POOLED point (v0.7) — per-date figure absent too. Same two-valued + rule on the weaker, magnitude-sensitive statistic. A NaN here lands on + CONTRADICTED, exactly as v0.7/v0.8 treated an unmeasurable monotonicity. + + RAW inputs in, hypothesis applied here: this is one of the two places (with + :func:`_aligned_net`) that knows what ``expected_ic_sign`` means. """ - if math.isfinite(inputs.monotonicity_spearman_by_date): - return inputs.monotonicity_spearman_by_date, False - return inputs.monotonicity_spearman, True + sign = inputs.expected_ic_sign + bar = thr.min_monotonicity_spearman + + low, high = _aligned_bounds( + sign, + inputs.monotonicity_spearman_by_date_ci_low, + inputs.monotonicity_spearman_by_date_ci_high, + ) + if math.isfinite(low) and math.isfinite(high): + interval = f"[{low:+.4f}, {high:+.4f}]" + if low > bar: + return MONO_HOLDS, ( + f"monotonicity direction HOLDS: the hypothesis-aligned per-date " + f"monotonicity 95% CI {interval} (N_eff-based) lies entirely above " + f"the bar {bar}.", + ) + if high < 0.0: + return MONO_CONTRADICTED, ( + f"monotonicity direction CONTRADICTED: the hypothesis-aligned " + f"per-date monotonicity 95% CI {interval} (N_eff-based) lies " + f"entirely BELOW 0 — the quantile buckets are ordered AGAINST the " + f"stated hypothesis, which is a measured negative finding.", + ) + return MONO_UNKNOWN, ( + f"monotonicity direction is INDISTINGUISHABLE FROM 0: the " + f"hypothesis-aligned per-date monotonicity 95% CI {interval} " + f"(N_eff-based, point {sign * inputs.monotonicity_spearman_by_date:+.4f}) " + f"straddles the bar {bar} / zero, so the evidence is not sufficient to " + f"ASSERT that the direction holds. Not a refutation either — this " + f"neither convicts nor acquits.", + ) + + point = inputs.monotonicity_spearman_by_date + if math.isfinite(point): + state = ( + MONO_HOLDS if _clears_level(sign * point, bar) else MONO_CONTRADICTED + ) + return state, (_MONO_POINT_FALLBACK_NOTE,) + + state = ( + MONO_HOLDS + if _clears_level(sign * inputs.monotonicity_spearman, bar) + else MONO_CONTRADICTED + ) + return state, (_MONO_POOLED_FALLBACK_NOTE, _MONO_POINT_FALLBACK_NOTE) -def _has_in_sample_point_signal(inputs: VerdictInputs, thr: VerdictThresholds) -> bool: - """POINT-estimate in-sample signal: |ICIR|, aligned monotonicity, NW-t, win rate. +def _has_core_point_signal(inputs: VerdictInputs, thr: VerdictThresholds) -> bool: + """POINT-estimate in-sample signal on the MAGNITUDE metrics: |ICIR|, NW-t, win rate. Every component must be a finite, known number: an unknown is not a signal. This is the v0.5 point check, kept for the FAIL determination — a factor whose POINT metrics do not clear has no signal at all (a negative finding). The CI lower-bound test (design §6, v0.6) is applied SEPARATELY, only to decide PASS vs "promising but unconfirmed" for a factor that DID clear on the point. + + MONOTONICITY IS DELIBERATELY NOT IN HERE (v0.9). It used to be, as a fourth + ``and`` term, which made it silently indistinguishable from the others — and + that is exactly what must not happen now that it can come back UNKNOWN. Folding + a three-valued fact into a boolean would collapse UNKNOWN onto one of the two + other answers; keeping it separate is what lets the axis route an unknown + direction to INSUFFICIENT_DATA while an unknown-direction factor that ALSO + fails here still FAILs. """ - monotonicity, _ = _gated_monotonicity(inputs) - aligned_monotonicity = inputs.expected_ic_sign * monotonicity return ( _clears_magnitude(inputs.ic_ir, thr.min_abs_icir) and _clears_magnitude(inputs.ic_nw_t, thr.min_abs_nw_t) and _clears_level(inputs.ic_win_rate, thr.min_ic_win_rate) - and _clears_level(aligned_monotonicity, thr.min_monotonicity_spearman) ) @@ -634,14 +787,29 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi reasons (gate vs no-OOS) are named distinctly. 3. FAIL sufficient data + OOS, but a NEGATIVE finding: the OOS sign is not consistent OR the in-sample POINT metrics do not clear (no signal at - all). POINT-based, so a merely wide CI never manufactures a FAIL. - 4. PASS OOS sign consistent AND point metrics clear AND the ICIR LOWER CI - bound (N_eff-based, expected direction) exceeds the bar — design §6, - v0.6: convincingly above, not the naked point. + all) OR the monotonicity direction is CONTRADICTED (v0.9: its + aligned CI lies entirely below 0). POINT-based on the magnitude + metrics, so a merely wide ICIR CI never manufactures a FAIL. + 3b. INSUFFICIENT_DATA (v0.9) everything else cleared, but the monotonicity + direction is UNKNOWN — its aligned CI straddles 0, so the direction + can be neither asserted nor refuted. + 4. PASS OOS sign consistent AND point metrics clear AND monotonicity + direction HOLDS AND the ICIR LOWER CI bound (N_eff-based, expected + direction) exceeds the bar — design §6, v0.6: convincingly above, + not the naked point. 5. INSUFFICIENT_DATA point metrics clear + OOS consistent, but the ICIR lower CI does NOT clear (or is unknown): promising, yet not confirmed at the pre-registered bar. This is the "gate passed, CI still straddles the bar" case — distinct from the sample gate (step 2). + + WHERE UNKNOWN SITS IN THE ORDER IS THE WHOLE POINT (v0.9). Step 3 runs FIRST and + is checked against the OTHER criteria in full, so an UNKNOWN monotonicity can + never RESCUE a factor that fails on ICIR / NW-t / win rate / OOS consistency: + that factor still FAILs, and the unknown direction is merely disclosed + alongside. Step 3b is reached only when every other criterion has cleared and + the direction ALONE cannot be asserted. "We could not tell" is a reason to + withhold a PASS, never a reason to withhold a FAIL that was independently + earned. """ sign = inputs.expected_ic_sign # 1. FAIL on a KNOWN out-of-sample reversal — before the gate. @@ -670,25 +838,24 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi if reasons: return AxisVerdict(AXIS_INSUFFICIENT_DATA, tuple(reasons)) - # The monotonicity actually gated on (design §6, v0.8). When the per-date - # figure is absent we judge on the pooled one, but we SAY SO — a silent - # substitution would hide that the decision rests on the magnitude-sensitive - # statistic the v0.8 fix exists to stop gating on. - _, mono_fallback = _gated_monotonicity(inputs) - mono_notes: tuple[str, ...] = () - if mono_fallback: - mono_notes = ( - "monotonicity gate FELL BACK to the pooled 'monotonicity_spearman': " - "the per-date figure ('monotonicity_spearman_by_date', design §6 v0.8) " - "was not supplied or not measurable. The pooled statistic is unbounded " - "and magnitude-sensitive, so this monotonicity judgement is weaker " - "than a v0.8 run's.", - ) - - # 3. FAIL: a measured negative finding (OOS sign not consistent, or no point - # signal at all). POINT-based — a wide CI is handled below, not here. - point_signal = _has_in_sample_point_signal(inputs, thr) - if (not inputs.oos_sign_consistent) or (not point_signal): + # The monotonicity direction actually gated on (design §6, v0.9): three-valued, + # decided on the per-date CI, with every fallback level DISCLOSED. A silent + # substitution would hide that the decision rests on a weaker statistic (the + # pooled magnitude one the v0.8 fix exists to stop gating on) or on a point with + # no dispersion estimate at all (the v0.9 fix). + mono_state, mono_notes = _monotonicity_direction(inputs, thr) + + # 3. FAIL: a measured negative finding (OOS sign not consistent, no point signal + # at all, or a CONTRADICTED monotonicity direction). Checked BEFORE the + # UNKNOWN branch, so an unknown direction never rescues a factor that failed + # on its own other evidence. POINT-based on the magnitude metrics — a wide + # ICIR CI is handled below, not here. + point_signal = _has_core_point_signal(inputs, thr) + if ( + (not inputs.oos_sign_consistent) + or (not point_signal) + or mono_state == MONO_CONTRADICTED + ): reasons = [] if not inputs.oos_sign_consistent: reasons.append( @@ -699,10 +866,31 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi if not point_signal: reasons.append( "in-sample POINT metrics do NOT clear the predictive thresholds " - "(ICIR / NW-t / win rate / monotonicity)." + "(ICIR / NW-t / win rate)." + ) + if mono_state == MONO_CONTRADICTED: + reasons.append( + "quantile monotonicity does not hold in the hypothesized direction." ) return AxisVerdict(AXIS_FAIL, tuple(reasons) + mono_notes) + # 3b. UNKNOWN monotonicity with EVERYTHING ELSE cleared -> INSUFFICIENT_DATA + # (v0.9). Reached only after step 3 declined to fail: this branch withholds + # a PASS, it never withholds a FAIL. + if mono_state == MONO_UNKNOWN: + return AxisVerdict( + AXIS_INSUFFICIENT_DATA, + ( + "every other predictive criterion cleared (ICIR / NW-t / win rate " + "point metrics, and the out-of-sample sign is consistent), but the " + "monotonicity DIRECTION cannot be distinguished from 0: its " + "confidence interval straddles zero, so the evidence is NOT " + "sufficient to assert that the direction holds. Withheld, not " + "refuted — an unknown direction is not a PASS and not a FAIL.", + ) + + mono_notes, + ) + # 4/5. Point signal present + OOS consistent. The #3 lower-CI test decides # PASS vs "promising but unconfirmed" (INSUFFICIENT_DATA). This is ABOVE # the sample gate and does NOT duplicate it (see _clears_magnitude). @@ -1027,6 +1215,9 @@ def decide_verdict( "AXIS_NOT_ASSESSED", "AXIS_VERDICTS", "AXIS_NAMES", + "MONO_HOLDS", + "MONO_CONTRADICTED", + "MONO_UNKNOWN", "VerdictThresholds", "VerdictInputs", "AxisVerdict", diff --git a/tests/test_factor_eval_contract.py b/tests/test_factor_eval_contract.py index f23a4b8..bbae9a8 100644 --- a/tests/test_factor_eval_contract.py +++ b/tests/test_factor_eval_contract.py @@ -10,6 +10,7 @@ from __future__ import annotations +import dataclasses import functools import json import math @@ -25,6 +26,7 @@ INSUFFICIENT_DATA, MANDATORY_SECTIONS, REJECT, + VERDICT_KEYS, WATCH, AxisVerdict, EvalConfig, @@ -41,7 +43,15 @@ # The v0.8 spread fix lived in an exact arithmetic expression, so the lock has to # be on that expression itself, not on a PASS/FAIL that happens to agree with it. -from analytics.eval.verdict import _aligned_net, _all_spreads_negative, _base_spread +from analytics.eval.verdict import ( + MONO_CONTRADICTED, + MONO_HOLDS, + MONO_UNKNOWN, + _aligned_net, + _all_spreads_negative, + _base_spread, + _monotonicity_direction, +) from factors.base import Factor from factors.compute.candidates import ( VALUE_FIELDS, @@ -2390,3 +2400,241 @@ def test_the_negative_sign_monotonicity_gate_is_still_direction_aligned(): VerdictInputs(**common, monotonicity_spearman_by_date=0.7) ) assert reversed_.predictive.verdict == AXIS_FAIL + + +# -------------------------------------------------------------------------- +# v0.9: the monotonicity DIRECTION gate is decided by the per-date CI and is +# THREE-VALUED (holds / contradicted / UNKNOWN). +# +# Why: the v0.8 per-date statistic turned out to be heavily attenuated by daily +# noise — an empirically perfect ladder scores ~0.05-0.11 — while the gate sat at +# a bare 0.0 with no dispersion estimate at all. Two real factors landed 0.021 +# apart across that boundary. +# -------------------------------------------------------------------------- + + +#: CI bounds only; the point is supplied too, since a real payload always carries +#: both and the reasons quote it. +def _mono(point: float, low: float, high: float) -> dict: + return { + "monotonicity_spearman_by_date": point, + "monotonicity_spearman_by_date_ci_low": low, + "monotonicity_spearman_by_date_ci_high": high, + } + + +def test_the_monotonicity_direction_gate_is_three_valued_on_the_ci(): + """The three states, each reached by moving ONLY the interval.""" + # (a) entirely above the bar -> the direction may be asserted -> PASS + holds = decide_verdict( + _inputs(**{**_STRONG, **_mono(0.08, 0.03, 0.13)}, **_OOS_OK) + ) + assert holds.predictive.verdict == AXIS_PASS + + # (b) entirely below 0 -> a MEASURED reversal -> FAIL + contradicted = decide_verdict( + _inputs(**{**_STRONG, **_mono(-0.08, -0.13, -0.03)}, **_OOS_OK) + ) + assert contradicted.predictive.verdict == AXIS_FAIL + assert any("CONTRADICTED" in x for x in contradicted.predictive.reasons) + + # (c) straddling 0 -> UNKNOWN -> neither convicts nor acquits. + # THE POINT IS POSITIVE (0.08, which v0.8 would have waved through as a + # direction that "holds"): it is the INTERVAL that withholds the PASS. + unknown = decide_verdict( + _inputs(**{**_STRONG, **_mono(0.08, -0.02, 0.18)}, **_OOS_OK) + ) + assert unknown.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any("straddles" in x for x in unknown.predictive.reasons) + assert any( + "not a PASS and not a FAIL" in x for x in unknown.predictive.reasons + ) + + +def test_an_unknown_monotonicity_does_not_rescue_a_factor_that_fails_elsewhere(): + """THE routing rule. An unknown direction withholds a PASS; it must never + withhold a FAIL that the other evidence already earned — otherwise 'we could not + measure the ladder' would upgrade every genuinely bad factor from Reject to + INSUFFICIENT-DATA, which is the exact opposite of quick-to-reject.""" + straddling = _mono(0.08, -0.02, 0.18) + + # each of the other predictive criteria, broken ONE AT A TIME + weak_icir = {**_STRONG, "ic_ir": 0.05, "ic_ir_ci_low": -0.2, "ic_ir_ci_high": 0.3} + weak_nw_t = {**_STRONG, "ic_nw_t": 0.4} + weak_win = {**_STRONG, "ic_win_rate": 0.41} + for broken in (weak_icir, weak_nw_t, weak_win): + r = decide_verdict(_inputs(**{**broken, **straddling}, **_OOS_OK)) + assert r.predictive.verdict == AXIS_FAIL + assert r.verdict == REJECT + # and the unknown direction is DISCLOSED alongside, not swallowed + assert any("INDISTINGUISHABLE FROM 0" in x for x in r.predictive.reasons) + + # out-of-sample inconsistency is "elsewhere" too + oos_bad = decide_verdict( + _inputs( + **{**_STRONG, **straddling}, + oos_available=True, + oos_sign_consistent=False, + ) + ) + assert oos_bad.predictive.verdict == AXIS_FAIL + + # ... and with everything else intact the SAME unknown yields INSUFFICIENT_DATA + alone = decide_verdict(_inputs(**{**_STRONG, **straddling}, **_OOS_OK)) + assert alone.predictive.verdict == AXIS_INSUFFICIENT_DATA + + +def test_an_unknown_monotonicity_axis_does_not_become_a_reject_label(): + """Predictive INSUFFICIENT_DATA + nothing failing = INSUFFICIENT-DATA, never + Reject: the deployment derivation must not read a withheld PASS as a failure.""" + r = decide_verdict( + _inputs(**{**_STRONG, **_mono(0.08, -0.02, 0.18)}, **_OOS_OK) + ) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert r.verdict == INSUFFICIENT_DATA + + +def test_the_aligned_monotonicity_ci_swaps_min_and_max_at_sign_minus_one(): + """Multiplying an interval by -1 REVERSES it. A low-vol factor (sign=-1) whose + RAW per-date CI is entirely NEGATIVE is a factor whose ALIGNED CI is entirely + positive — i.e. its hypothesis holds. Getting the swap wrong reads it backwards.""" + thr = VerdictThresholds() + # raw [-0.13, -0.03] at sign=-1 -> aligned [+0.03, +0.13] -> HOLDS + state, reasons = _monotonicity_direction( + VerdictInputs(expected_ic_sign=-1, **_mono(-0.08, -0.13, -0.03)), thr + ) + assert state == MONO_HOLDS + assert "[+0.0300, +0.1300]" in reasons[0] # re-sorted, not left reversed + + # raw [+0.03, +0.13] at sign=-1 -> aligned [-0.13, -0.03] -> CONTRADICTED + state, _ = _monotonicity_direction( + VerdictInputs(expected_ic_sign=-1, **_mono(0.08, 0.03, 0.13)), thr + ) + assert state == MONO_CONTRADICTED + + # the same raw intervals at sign=+1 read the other way round + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(0.08, 0.03, 0.13)), thr + )[0] == MONO_HOLDS + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(-0.08, -0.13, -0.03)), thr + )[0] == MONO_CONTRADICTED + + # end to end: the sign=-1 PASS survives the new gate + common = dict( + expected_ic_sign=-1, **_SAMPLE_OK, **_OOS_OK, + ic_ir=-0.8, ic_ir_ci_low=-1.05, ic_ir_ci_high=-0.55, + ic_win_rate=0.7, ic_nw_t=-3.5, + ) + assert decide_verdict( + VerdictInputs(**common, **_mono(-0.08, -0.13, -0.03)) + ).predictive.verdict == AXIS_PASS + assert decide_verdict( + VerdictInputs(**common, **_mono(0.08, 0.03, 0.13)) + ).predictive.verdict == AXIS_FAIL + + +def test_the_direction_gate_is_strict_and_a_ci_touching_the_bar_is_unknown(): + """`> bar`, like every other level comparison in this module: an aligned lower + bound sitting exactly ON 0.0 has not shown the direction holds.""" + thr = VerdictThresholds() + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(0.09, 0.0, 0.18)), thr + )[0] == MONO_UNKNOWN + # and an upper bound exactly at 0 is not a demonstrated reversal either + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(-0.09, -0.18, 0.0)), thr + )[0] == MONO_UNKNOWN + + +def test_a_raised_bar_withholds_a_pass_without_calling_the_factor_reversed(): + """The PASS side moves with the (configurable) bar; the FAIL side stays pinned + at 0. A correctly-ordered-but-weak ladder is UNKNOWN under a strength bar — it + is NOT 'reversed', which is a claim about the sign.""" + strict = VerdictThresholds(min_monotonicity_spearman=0.5) + state, reasons = _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(0.08, 0.03, 0.13)), strict + ) + assert state == MONO_UNKNOWN + assert any("straddles the bar 0.5" in x for x in reasons) + # genuinely reversed is still CONTRADICTED under the same raised bar + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(-0.08, -0.13, -0.03)), strict + )[0] == MONO_CONTRADICTED + + +def test_the_monotonicity_fallback_chain_has_two_disclosed_levels(): + """CI absent -> the bare per-date point (v0.8); per-date point absent too -> + the pooled magnitude statistic (v0.7). Each level is ANNOUNCED in the reasons, + because judging on a weaker statistic while presenting a v0.9 verdict would be + the silent degradation this whole layer exists to prevent.""" + thr = VerdictThresholds() + + # level 1: point present, CI absent -> two-valued, v0.8 behaviour + holds, notes = _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, monotonicity_spearman_by_date=0.7), thr + ) + assert holds == MONO_HOLDS + assert any("REVERTED to the BARE per-date POINT" in n for n in notes) + assert not any("FELL BACK to the pooled" in n for n in notes) + contradicted, _ = _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, monotonicity_spearman_by_date=-0.3), thr + ) + # NOT unknown: without a dispersion estimate there is nothing to be uncertain + # with, and inventing an UNKNOWN would turn every pre-v0.9 FAIL into + # INSUFFICIENT_DATA behind the reader's back. + assert contradicted == MONO_CONTRADICTED + + # level 2: per-date figure absent too -> the pooled one, both notes present + pooled, notes2 = _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, monotonicity_spearman=0.9), thr + ) + assert pooled == MONO_HOLDS + assert any("FELL BACK to the pooled" in n for n in notes2) + assert any("REVERTED to the BARE per-date POINT" in n for n in notes2) + + # nothing at all: unmeasurable monotonicity still lands where v0.7/v0.8 put it + assert _monotonicity_direction(VerdictInputs(expected_ic_sign=+1), thr)[0] \ + == MONO_CONTRADICTED + + # and the levels surface through the real axis, not just the helper + axis = decide_verdict( + _inputs(**{**_STRONG, "monotonicity_spearman_by_date": 0.7}, **_OOS_OK) + ).predictive + assert axis.verdict == AXIS_PASS + assert any("REVERTED to the BARE per-date POINT" in x for x in axis.reasons) + + +def test_a_supplied_ci_is_used_even_when_it_disagrees_with_the_point(): + """The CI is the gate, not a decoration on the point. A point above the bar with + an interval that straddles 0 must NOT pass — that is the v0.9 fix — and a point + below 0 with an interval that clears must not be convicted on the point.""" + thr = VerdictThresholds() + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(0.9, -0.05, 1.85)), thr + )[0] == MONO_UNKNOWN + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(-0.4, 0.05, 0.20)), thr + )[0] == MONO_HOLDS + # a HALF-supplied CI is not a CI: fall back rather than guess the other end + half, notes = _monotonicity_direction( + VerdictInputs( + expected_ic_sign=+1, + monotonicity_spearman_by_date=0.7, + monotonicity_spearman_by_date_ci_low=0.03, + ), + thr, + ) + assert half == MONO_HOLDS + assert any("REVERTED to the BARE per-date POINT" in n for n in notes) + + +def test_the_return_risk_verdict_keys_document_the_monotonicity_ci(): + """VERDICT_KEYS is the documented contract of what the verdict reads; a gate + input missing from it is an undocumented dependency.""" + assert "monotonicity_spearman_by_date_ci_low" in VERDICT_KEYS["return_risk"] + assert "monotonicity_spearman_by_date_ci_high" in VERDICT_KEYS["return_risk"] + # every documented return_risk key is a real VerdictInputs field + fields = {f.name for f in dataclasses.fields(VerdictInputs)} + for key in VERDICT_KEYS["return_risk"]: + assert key in fields diff --git a/tests/test_factor_eval_standard.py b/tests/test_factor_eval_standard.py index b1c76bd..6c9aab5 100644 --- a/tests/test_factor_eval_standard.py +++ b/tests/test_factor_eval_standard.py @@ -48,12 +48,14 @@ half_life, hypothesis_win_rate, information_ratio_ci, + mean_by_date_spearman, mean_ci, newey_west_lag, newey_west_t, sortino, spearman, spearman_by_date, + spearman_series_by_date, ) from analytics.factor import compute_ic from factors.spec import FactorSpec @@ -1987,6 +1989,156 @@ def test_return_risk_reports_both_monotonicity_statistics(rng): assert math.isfinite(by_date) and -1.0 <= by_date <= 1.0 +# -------------------------------------------------------------------------- +# v0.9 fix: the per-date monotonicity SERIES + its N_eff CI. The point figure +# stays BIT-IDENTICAL to v0.8 — this is a refactor plus an addition, not a +# re-measurement. +# -------------------------------------------------------------------------- + + +def _v08_spearman_by_date(quantile_returns: pd.DataFrame, min_buckets: int = 3) -> float: + """The v0.8 ``spearman_by_date`` body, VERBATIM — the bit-identity oracle. + + Frozen here on purpose: v0.9 re-expresses the same reduction as the mean of a + new per-date series, and "the returned float did not move" is a claim that needs + the OLD code to compare against, not a re-reading of the new code. + """ + if quantile_returns.empty or quantile_returns.shape[1] < min_buckets: + return float("nan") + index = [float(q) for q in quantile_returns.columns] + daily: list[float] = [] + for _, row in quantile_returns.iterrows(): + values = [float(v) for v in row.to_numpy(dtype=float)] + if sum(1 for v in values if math.isfinite(v)) < min_buckets: + continue + rho = spearman(index, values) + if math.isfinite(rho): + daily.append(rho) + if not daily: + return float("nan") + return float(sum(daily) / len(daily)) + + +def _same_float(a: float, b: float) -> bool: + """Exact equality, with NaN == NaN. No tolerance: this is a bit-identity check.""" + return (math.isnan(a) and math.isnan(b)) or a == b + + +def test_spearman_by_date_is_bit_identical_to_the_v08_implementation(): + """Every v0.8 fixture in this file, plus a messy random panel, must return the + EXACT same float — not approx. A v0.9 report field that drifted in the last bit + would silently break cross-run comparability with the ten factors already + evaluated under v0.8.""" + nan = float("nan") + normal = [0.001, 0.002, 0.003, 0.004, 0.005] + fixtures = [ + # the four v0.8 test fixtures, verbatim + _quantile_frame([normal] * 20 + [[1.0, 1.1, 1.2, nan, nan]] * 2), + _quantile_frame([normal] * 12), + _quantile_frame([[0.005, 0.004, 0.003, 0.002, 0.001]] * 12), + _quantile_frame([normal] * 6 + [[0.005, 0.004, nan, nan, nan]] * 3), + _quantile_frame([normal] * 3 + [[0.005, 0.004, 0.003, nan, nan]] * 3), + _quantile_frame([[0.1, 0.2, nan, nan, nan]] * 5), + _quantile_frame([[0.2] * 5] * 5), + pd.DataFrame(), + ] + # ... and a panel whose per-date values are irrational-ish, so a changed + # summation order (Series.mean vs the sequential sum) WOULD show up. + rng = np.random.default_rng(20260721) + messy = rng.normal(size=(97, 5)) / 3.0 + messy[messy > 1.6] = np.nan + fixtures.append(_quantile_frame(messy.tolist())) + + for frame in fixtures: + assert _same_float(spearman_by_date(frame), _v08_spearman_by_date(frame)) + # and the messy one really does exercise a non-trivial average + tail = spearman_by_date(fixtures[-1]) + assert math.isfinite(tail) and tail not in (-1.0, 0.0, 1.0) + + +def test_the_per_date_series_omits_skipped_dates_rather_than_holding_them(): + """A skipped date must be ABSENT from the index, not a NaN row: mean_ci reads + N_eff off this series, and a NaN row would be dropped anyway while an invented + 0.0 would be a fabricated observation.""" + nan = float("nan") + good = [0.001, 0.002, 0.003, 0.004, 0.005] + thin = [0.005, 0.004, nan, nan, nan] # 2 finite buckets -> skipped + tied = [0.2] * 5 # all tied -> Spearman undefined -> skipped + frame = _quantile_frame([good, thin, good, tied, good]) + + series = spearman_series_by_date(frame) + assert list(series.index) == [frame.index[0], frame.index[2], frame.index[4]] + assert series.notna().all() + assert list(series) == [pytest.approx(1.0)] * 3 + assert series.index.name == frame.index.name + # and the point figure is exactly this series' mean + assert _same_float(spearman_by_date(frame), mean_by_date_spearman(series)) + + +def test_the_per_date_series_is_empty_not_all_nan_when_no_date_qualifies(): + """'No observation' and 'an unknown observation' are different facts, and only + the first is true here — an all-NaN series would let a downstream reader believe + dates were measured and came back unknown.""" + nan = float("nan") + for frame in ( + _quantile_frame([[0.1, 0.2, nan, nan, nan]] * 5), # never 3 finite buckets + _quantile_frame([[0.2] * 5] * 5), # always tied + pd.DataFrame(), # nothing at all + _quantile_frame([[0.1, 0.2]] * 5), # fewer buckets than min + ): + series = spearman_series_by_date(frame) + assert isinstance(series, pd.Series) + assert series.empty and len(series) == 0 + assert series.isna().sum() == 0 # empty, NOT a column of NaN + assert math.isnan(mean_by_date_spearman(series)) + + +def test_return_risk_attaches_an_n_eff_ci_to_the_per_date_monotonicity(rng): + """v0.9: the direction gate reads an INTERVAL, so the section must emit one — + and it must describe the very same observations as the point.""" + panel, factor = planted_signal_setup(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + payload = report.by_name()["return_risk"].payload + for key in ( + "monotonicity_spearman_by_date_se", + "monotonicity_spearman_by_date_ci_low", + "monotonicity_spearman_by_date_ci_high", + "monotonicity_spearman_by_date_n_eff", + ): + assert key in payload + + low = payload["monotonicity_spearman_by_date_ci_low"] + high = payload["monotonicity_spearman_by_date_ci_high"] + point = payload["monotonicity_spearman_by_date"] + assert math.isfinite(low) and math.isfinite(high) + assert low < point < high + # symmetric around the point, i.e. the CI is OF this point and not of some + # other reduction of the panel + assert (low + high) / 2.0 == pytest.approx(point) + n_eff = payload["monotonicity_spearman_by_date_n_eff"] + n_dates = payload["monotonicity_spearman_by_date_n_dates"] + assert 1.0 <= n_eff <= n_dates # N_eff can never exceed the observations + assert n_dates > 0 + + +def test_the_monotonicity_ci_is_the_shared_mean_ci_on_the_shared_series(rng): + """Not a second CI implementation: the payload must equal mean_ci() applied to + spearman_series_by_date() on the same quantile matrix.""" + panel, factor = planted_signal_setup(rng) + ir = build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + expected = mean_ci(spearman_series_by_date(ir.quantile_returns)) + + payload = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ).by_name()["return_risk"].payload + assert payload["monotonicity_spearman_by_date_ci_low"] == expected["ci_low"] + assert payload["monotonicity_spearman_by_date_ci_high"] == expected["ci_high"] + assert payload["monotonicity_spearman_by_date_se"] == expected["se"] + assert payload["monotonicity_spearman_by_date_n_eff"] == expected["n_eff"] + + # -------------------------------------------------------------------------- # v0.8 fix #1 at the standard layer: the aligned spread series/CI. # --------------------------------------------------------------------------