diff --git a/analytics/eval/report.py b/analytics/eval/report.py index 63ec8a8..db4e66f 100644 --- a/analytics/eval/report.py +++ b/analytics/eval/report.py @@ -123,6 +123,20 @@ def payload_of(name: str) -> Mapping[str, object]: monotonicity_spearman=_payload_number( returns, "monotonicity_spearman", float("nan") ), + # design §6 v0.8: the GATED monotonicity. Absent (a pre-v0.8 IR) -> NaN -> + # the verdict falls back to the pooled field and DISCLOSES it. + monotonicity_spearman_by_date=_payload_number( + returns, "monotonicity_spearman_by_date", float("nan") + ), + # design §6 v0.9: the per-date monotonicity's N_eff-based CI, which is what + # the DIRECTION gate reads. Absent (a pre-v0.9 IR) -> NaN -> the verdict + # reverts to the bare point (v0.8 behaviour) and DISCLOSES that too. + monotonicity_spearman_by_date_ci_low=_payload_number( + returns, "monotonicity_spearman_by_date_ci_low", float("nan") + ), + monotonicity_spearman_by_date_ci_high=_payload_number( + returns, "monotonicity_spearman_by_date_ci_high", float("nan") + ), # Incremental axis facts (design §6, v0.5). A Skipped purity section (no # book) leaves the payload empty -> known_factors_supplied defaults False # -> the axis is NOT_ASSESSED. A supplied-but-unmeasurable orthogonalized @@ -141,6 +155,12 @@ def payload_of(name: str) -> Mapping[str, object]: purity, "incremental_ic_ir_ci_high", float("nan") ), net_long_short_by_cost=by_cost, + # design §6 v0.8: needed to align a spread by hypothesis without adding the + # cost back (cost = gross - net). Absent -> NaN -> a sign=-1 aligned spread + # reads UNKNOWN rather than silently mis-signed. + gross_long_short_mean=_payload_number( + returns, "gross_long_short_mean", float("nan") + ), oos_available=bool(_payload_flag(oos, "oos_available", False)), oos_sign_consistent=bool(_payload_flag(oos, "sign_consistent", False)), oos_sign_flipped=bool(_payload_flag(oos, "sign_flipped", False)), diff --git a/analytics/eval/sections.py b/analytics/eval/sections.py index 3e63fc3..05d071c 100644 --- a/analytics/eval/sections.py +++ b/analytics/eval/sections.py @@ -54,9 +54,31 @@ "ic_win_rate", "ic_nw_t", ), - # monotonicity_spearman: RAW Spearman(bucket index, bucket mean return). + # monotonicity_spearman: RAW Spearman(bucket index, bucket mean return) over + # the POOLED (date-equal-weighted) bucket means — unbounded and magnitude + # sensitive, so (v0.8) it is REPORTED but no longer gates. + # monotonicity_spearman_by_date: (v0.8) the gated one — Spearman per date + # (bounded [-1, 1] before averaging, structurally parallel to the rank IC), + # then averaged. The verdict falls back to the pooled field when it is + # absent and SAYS SO in its reasons. + # monotonicity_spearman_by_date_ci_low/high: (v0.9) the N_eff-based 95% CI of + # that per-date series — what the DIRECTION gate now reads. Three-valued: + # aligned CI above the bar = direction holds; entirely below 0 = contradicted + # (FAIL); straddling 0 = UNKNOWN, which neither convicts nor acquits. The bare + # point is only the fallback, disclosed as such. # net_long_short_by_cost: {cost multiplier: net (top - bottom) return}, RAW. - "return_risk": ("monotonicity_spearman", "net_long_short_by_cost"), + # gross_long_short_mean: (v0.8) the GROSS (pre-cost) leg difference — the + # verdict needs it to align a net spread, since aligning is + # `sign * gross - cost` and cost = gross - net cannot be recovered from the + # net alone. + "return_risk": ( + "monotonicity_spearman", + "monotonicity_spearman_by_date", + "monotonicity_spearman_by_date_ci_low", + "monotonicity_spearman_by_date_ci_high", + "net_long_short_by_cost", + "gross_long_short_mean", + ), # The Incremental axis (design §6, v0.5). known_factors_supplied = was a # known-factor BOOK supplied at all (False -> the axis is NOT_ASSESSED); # incremental_ic_ir / incremental_ic_mean = mean(orthIC)/std(orthIC) and diff --git a/analytics/eval/standard.py b/analytics/eval/standard.py index dc6d7c4..4c60b52 100644 --- a/analytics/eval/standard.py +++ b/analytics/eval/standard.py @@ -61,10 +61,12 @@ half_life, hypothesis_win_rate, information_ratio_ci, + mean_by_date_spearman, mean_ci, newey_west_t, sortino, spearman, + spearman_series_by_date, ) from analytics.factor import ic_summary from analytics.performance import performance_summary @@ -314,6 +316,11 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: top, bottom = ir.cfg.long_short buckets = list(quantiles.columns) + # The PER-DATE monotonicity series, built ONCE (design §6, v0.9): the + # payload's point figure and its CI are then two reductions of the SAME + # observations, and the panel is walked once instead of twice. + mono_by_date = spearman_series_by_date(quantiles) + bucket_mean = {int(q): as_float(quantiles[q].mean()) for q in buckets} bucket_nav = { int(q): as_float((1.0 + quantiles[q].dropna()).prod()) for q in buckets @@ -329,14 +336,17 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: net_by_cost: dict[float, float] = {} cumulative_by_cost: dict[float, float] = {} base_net = None + base_cost = None for multiplier in ir.cfg.cost_scenarios: - net = gross - ir.ctx.fee_rate * multiplier * leg_turnover + cost = ir.ctx.fee_rate * multiplier * leg_turnover + net = gross - cost net_by_cost[float(multiplier)] = as_float(net.mean()) cumulative_by_cost[float(multiplier)] = as_float( (1.0 + net.dropna()).prod() - 1.0 ) if abs(multiplier - 1.0) < 1e-9: base_net = net + base_cost = cost payload: dict[str, object] = { "long_short_legs": f"Q{top} - Q{bottom}", @@ -345,21 +355,71 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: "quantile_periods_with_return": bucket_periods, # RAW Spearman of bucket index vs bucket mean return; the verdict is # the ONE place that multiplies by expected_ic_sign. + # + # POOLED (v0.1..v0.7 semantics, DELIBERATELY UNCHANGED): correlates + # against cross-date arithmetic means, so it is unbounded and carries + # MAGNITUDE structure. Kept as a reported field — historical reports + # stay comparable — but as of v0.8 it is NOT what the Predictive axis + # gates on, because a rank-based axis must not be decided by a + # magnitude-sensitive statistic. "monotonicity_spearman": spearman( [float(q) for q in buckets], [bucket_mean[int(q)] for q in buckets] ), + # PER-DATE version (v0.8, the GATED one): each date's Spearman over the + # buckets is capped in [-1, 1] BEFORE averaging across dates, exactly + # like the rank IC. Robust to a few extreme-return days concentrating + # in one bucket. RAW; the verdict applies expected_ic_sign. + # + # (v0.9) The POINT is kept — reported, cross-run comparable — but it is + # no longer what decides the direction on its own; the CI attached below + # is. Identical value to v0.8 by construction (same reduction, same + # series). + "monotonicity_spearman_by_date": mean_by_date_spearman(mono_by_date), "gross_long_short_mean": as_float(gross.mean()), "net_long_short_by_cost": net_by_cost, "net_long_short_cumulative_by_cost": cumulative_by_cost, "fee_rate_base": ir.ctx.fee_rate, "periods_per_year": ir.periods_per_year, } + # N_eff-based CI of the PER-DATE monotonicity (design §6, v0.9) — the same + # mean_ci the ICIR uses, on the per-date rank series instead of the IC + # series. THIS is what the Predictive axis' direction gate reads: the bare + # point carries no dispersion estimate at all, and this project's real runs + # put a perfect quantile ladder at only ~0.05-0.11 on it, so "point > 0.0" + # was a coin flip dressed as a criterion. The CI turns the gate three-valued + # (holds / contradicted / UNKNOWN) — see verdict._monotonicity_direction. + mono_ci = mean_ci(mono_by_date, confidence=DEFAULT_CONFIDENCE) + payload["monotonicity_spearman_by_date_se"] = mono_ci["se"] + payload["monotonicity_spearman_by_date_ci_low"] = mono_ci["ci_low"] + payload["monotonicity_spearman_by_date_ci_high"] = mono_ci["ci_high"] + payload["monotonicity_spearman_by_date_n_eff"] = mono_ci["n_eff"] + payload["monotonicity_spearman_by_date_n_dates"] = int(mono_by_date.size) + payload["monotonicity_spearman_by_date_ci_note"] = ( + "N_eff-based 95% CI of the per-date monotonicity (design §6, v0.9). The " + "Predictive axis gates the monotonicity DIRECTION on this interval, not " + "on the point: aligned CI entirely above the bar = direction holds; " + "entirely below 0 = contradicted (FAIL); straddling 0 = UNKNOWN, which " + "neither convicts nor acquits. n_dates counts the dates that qualified " + "(< 3 finite buckets or all-tied dates are skipped, not zero-filled)." + ) + # N_eff-based CI of the base-cost QN-Q1 spread (design §6, v0.6). REPORTED # for the reader (b); the Tradable axis still gates on the POINT base spread # > 0 (the CI on this leg difference is point-only for now — c names only # ICIR / incremental ICIR as the lower-CI magnitude criteria). - if base_net is not None: - base_ci = mean_ci(sign * base_net, confidence=DEFAULT_CONFIDENCE) + # HYPOTHESIS-ALIGNED base-cost spread (v0.8 fix). The hypothesis decides + # WHICH LEG IS LONG; cost is a drag in BOTH directions. So flip the legs + # first, THEN subtract the cost: + # aligned_net = sign * gross - cost + # The pre-v0.8 form `sign * net` expanded, at sign=-1, to `-gross + cost` + # — it ADDED the trading cost back as if shorting earned it. At sign=+1 + # this is bit-identical to the old expression (1 * gross - cost == net). + aligned_base = None + if base_net is not None and base_cost is not None: + aligned_base = sign * gross - base_cost + + if aligned_base is not None: + base_ci = mean_ci(aligned_base, confidence=DEFAULT_CONFIDENCE) payload["net_long_short_base_se"] = base_ci["se"] payload["net_long_short_base_ci_low"] = base_ci["ci_low"] payload["net_long_short_base_ci_high"] = base_ci["ci_high"] @@ -374,8 +434,8 @@ def return_risk(self, ir: StandardEvalIR) -> Section | Skipped: f"annualization below is a fallback, not a derived fact." ) - if base_net is not None and base_net.notna().sum() >= 2: - aligned = (sign * base_net).dropna() + if aligned_base is not None and aligned_base.notna().sum() >= 2: + aligned = aligned_base.dropna() nav = (1.0 + aligned).cumprod() simple = performance_summary(nav, periods_per_year=ir.periods_per_year) payload["aligned_spread_annual_return"] = simple["annual_return"] diff --git a/analytics/eval/stats.py b/analytics/eval/stats.py index 29c5c70..590a1b6 100644 --- a/analytics/eval/stats.py +++ b/analytics/eval/stats.py @@ -393,6 +393,95 @@ def spearman(x: pd.Series | list, y: pd.Series | list) -> float: return float(pair["x"].corr(pair["y"], method="spearman")) +def spearman_series_by_date( + quantile_returns: pd.DataFrame, min_buckets: int = 3 +) -> pd.Series: + """PER-DATE Spearman(bucket index, bucket return), one value per QUALIFYING date. + + The series :func:`spearman_by_date` averages, exposed in its own right (design + §6, v0.9) so the verdict can put a DISPERSION estimate on that average. A bare + cross-date mean of per-date rank correlations is heavily attenuated by daily + noise — an empirically perfect quantile ladder scores ~0.05-0.11 in this + project's real runs — so gating a direction claim on whether that point happens + to land above 0.0 reads noise as evidence. With the series in hand the caller + can hand it to :func:`mean_ci` and gate on the N_eff-based interval instead. + + A date is SKIPPED — ABSENT FROM THE SERIES, never a zero and never a NaN row — + when it carries fewer than ``min_buckets`` finite bucket returns (too few points + for a meaningful rank correlation), or when its surviving buckets are all tied + (Spearman undefined there). NO date qualifying yields an EMPTY series, not a + series of NaN: "no observation" and "an unknown observation" are different + facts, and only the first is true here. + + Because skipped dates are absent rather than held as holes, the index is the + SURVIVING date grid. Downstream N_eff (via :func:`mean_ci`) is therefore read + off that compacted grid — the same convention :func:`mean_ci` already applies to + the IC series, whose degenerate cross-sections it drops before measuring + autocorrelation. Noted rather than hidden (design §11). + + RAW, like :func:`spearman`: the hypothesis sign is applied by the verdict. + """ + dates: list[object] = [] + daily: list[float] = [] + if not (quantile_returns.empty or quantile_returns.shape[1] < min_buckets): + index = [float(q) for q in quantile_returns.columns] + for date, row in quantile_returns.iterrows(): + values = [float(v) for v in row.to_numpy(dtype=float)] + if sum(1 for v in values if math.isfinite(v)) < min_buckets: + continue + # spearman() drops the non-finite pairs itself, keeping each surviving + # bucket paired with its OWN index (not a renumbered 1..k). + rho = spearman(index, values) + if math.isfinite(rho): + dates.append(date) + daily.append(rho) + name = quantile_returns.index.name if isinstance(quantile_returns, pd.DataFrame) else None + return pd.Series(daily, index=pd.Index(dates, name=name), dtype=float) + + +def mean_by_date_spearman(daily: pd.Series) -> float: + """The cross-date average :func:`spearman_by_date` reduces its series with. + + Exposed so a caller that already built the per-date series (the standard layer + does, to attach a CI to it) gets the POINT from the very same expression rather + than paying for a second pass over the panel — the point and the interval can + then never disagree about which observations they describe. + + Deliberately the sequential Python ``sum(...) / len(...)`` of the pre-v0.9 + implementation, not ``Series.mean()``: the two can differ in the last bit, and + this figure is a published, cross-run-comparable report field. + """ + values = daily.tolist() + if not values: + return float("nan") + return float(sum(values) / len(values)) + + +def spearman_by_date( + quantile_returns: pd.DataFrame, min_buckets: int = 3 +) -> float: + """Mean over dates of the PER-DATE Spearman(bucket index, bucket return). + + Structurally parallel to the rank IC (design §6, v0.8): each date's statistic + is BOUNDED in [-1, 1] before the cross-date average, so a handful of extreme + return days cannot dominate it. The pooled :func:`spearman` version correlates + against cross-date ARITHMETIC MEANS, which are unbounded and magnitude + sensitive — one outlier bucket-day can flip it while the daily-capped rank IC + barely moves. + + As of v0.9 this is the mean of :func:`spearman_series_by_date` (see there for + the skip rules) — a pure refactor: the returned float is BIT-IDENTICAL to the + v0.8 implementation, which is asserted against the v0.8 fixtures. + + NaN when NO date qualifies: unknown, not 0.0. + + RAW, like :func:`spearman`: the hypothesis sign is applied by the verdict. + """ + return mean_by_date_spearman( + spearman_series_by_date(quantile_returns, min_buckets=min_buckets) + ) + + def as_float(value: object) -> float: """Coerce to a plain float for a payload; anything unusable becomes NaN.""" try: @@ -409,9 +498,12 @@ def as_float(value: object) -> float: "half_life", "hypothesis_win_rate", "information_ratio_ci", + "mean_by_date_spearman", "mean_ci", "newey_west_lag", "newey_west_t", "sortino", "spearman", + "spearman_by_date", + "spearman_series_by_date", ] diff --git a/analytics/eval/verdict.py b/analytics/eval/verdict.py index 3a7edc0..4ffca79 100644 --- a/analytics/eval/verdict.py +++ b/analytics/eval/verdict.py @@ -93,6 +93,15 @@ ``net_long_short_by_cost`` is the plain (top - bottom) leg difference. The HYPOTHESIS (``expected_ic_sign``) is applied in exactly ONE place — here — so a low-vol factor (sign -1) reads in its own direction. + + APPLYING THE SIGN TO A NET SPREAD IS NOT A MULTIPLICATION (v0.8). The + hypothesis decides WHICH LEG IS LONG; the trading cost is a drag in EITHER + direction. So the aligned spread flips the legs and THEN subtracts cost — + ``sign * gross - cost`` — never ``sign * net``, which at sign -1 expands to + ``-gross + cost`` and hands the factor its own costs back as profit. This + is why ``gross_long_short_mean`` is an input: the cost is recovered as + ``gross - net``, and an unknown gross makes the ALIGNED spread unknown + (which, per the rule above, then neither convicts nor passes). 2. ``ic_win_rate`` and the orthogonalized ICs are hypothesis-relative by the same rule: direction is applied by multiplying with ``expected_ic_sign``. @@ -109,6 +118,23 @@ (INSUFFICIENT below it); the lower-CI test asks "is it convincingly above the bar?" (PASS above it). FAIL stays POINT-based and known-fact-only; a NaN CI bound never convicts. + +THE MONOTONICITY DIRECTION IS THREE-VALUED (v0.9). v0.8 moved the monotonicity gate +onto a per-date rank statistic, which was the right KIND of statistic — and then the +eleven-factor re-run showed that statistic is heavily attenuated by daily noise: an +empirically perfect quantile ladder scores 0.045-0.106 on it, while the direction +gate still sat at a bare 0.0 with no dispersion estimate anywhere. Two factors +landed 0.021 apart across that boundary. So the direction is now decided by the +N_eff-based CI of the per-date series (:func:`_monotonicity_direction`), and the +answer can be UNKNOWN — CI straddles 0 — as well as holds/contradicted. + + UNKNOWN NEVER RESCUES. An unknown direction is checked AFTER every other + predictive criterion: if ICIR / NW-t / win rate / OOS consistency fail, the axis + still FAILs and the unknown is merely disclosed. Only when everything else has + cleared and the direction ALONE cannot be asserted does the axis become + INSUFFICIENT_DATA. "We could not tell" withholds a PASS; it never withholds a + FAIL that was independently earned. This is the same asymmetry as UNKNOWN NEVER + CONVICTS above, read from the other side. """ from __future__ import annotations @@ -231,6 +257,11 @@ class VerdictThresholds: # monotone". A 0.8 strength bar rejected tail-concentrated real factors whose # power sits in one extreme (e.g. jump-amount-corr: ICIR -0.40, NW-t -14.8, yet # Q3-humped). Raise it to demand strength. UNVALIDATED (§11). + # (v0.9) The bar is now compared against the per-date monotonicity's aligned CI + # LOWER bound, not its point — so it is the bar a direction claim must be + # CONVINCINGLY above. The FAIL side stays pinned at 0 (reversal is a fact about + # the sign, not about how strong this threshold was set); see + # _monotonicity_direction. min_monotonicity_spearman: float = 0.0 def __post_init__(self) -> None: @@ -308,7 +339,28 @@ class VerdictInputs: ic_ir_ci_high: float = float("nan") ic_win_rate: float = float("nan") # predictive_power (hypothesis-relative) ic_nw_t: float = float("nan") # predictive_power - monotonicity_spearman: float = float("nan") # return_risk (RAW) + #: return_risk: POOLED Spearman(bucket index, cross-date MEAN bucket return), + #: RAW. Unbounded and magnitude-sensitive; as of v0.8 it is a REPORTED figure + #: and only the FALLBACK for the gate below. + monotonicity_spearman: float = float("nan") + #: return_risk: the v0.8 GATED monotonicity — the mean over dates of each + #: date's own Spearman across the buckets, each capped in [-1, 1] before + #: averaging (structurally parallel to the rank IC). RAW. NaN = UNKNOWN, which + #: falls back to ``monotonicity_spearman`` with the substitution DISCLOSED in + #: the axis reasons, so an IR built before v0.8 stays judgeable. + #: (v0.9) This POINT is now only the FALLBACK for the direction gate; the CI + #: below is what decides when it is available. + monotonicity_spearman_by_date: float = float("nan") + #: return_risk: 95% CI bounds of ``monotonicity_spearman_by_date``, computed + #: with the per-date series' OWN N_eff (design §6, v0.9 — the same ``mean_ci`` + #: the ICIR uses). RAW (the verdict applies ``expected_ic_sign``). This is what + #: the monotonicity DIRECTION gate reads: the bare point carries no dispersion + #: estimate, and a cross-date mean of per-date rank correlations is so + #: attenuated by daily noise that "point > 0.0" was a coin flip dressed as a + #: criterion. NaN = UNKNOWN -> the gate falls back to the point (v0.8 behaviour) + #: with the fallback DISCLOSED. + monotonicity_spearman_by_date_ci_low: float = float("nan") + monotonicity_spearman_by_date_ci_high: float = float("nan") oos_available: bool = False # oos_generalization oos_sign_consistent: bool = False # expected sign holds in BOTH subperiods oos_sign_flipped: bool = False # explicit flip -> Predictive FAIL @@ -333,6 +385,11 @@ class VerdictInputs: # -- return_risk / execution facts ----------------------------------- #: return_risk: {cost multiplier: net (top - bottom) return}, RAW sign. net_long_short_by_cost: tuple[tuple[float, float], ...] = () + #: return_risk: mean GROSS (top - bottom) leg difference, RAW sign, before any + #: cost. Required to align a spread by hypothesis WITHOUT adding the cost back + #: (see :func:`_aligned_net`): the per-scenario cost is recovered as + #: ``gross - net``. NaN = UNKNOWN -> a sign=-1 aligned spread is UNKNOWN too. + gross_long_short_mean: float = float("nan") tradable: bool | None = None # execution_capacity; None = not supplied capacity_sufficient: bool | None = None # None = not supplied @@ -435,17 +492,31 @@ def _clears_level(point_estimate: float, threshold: float) -> bool: return math.isfinite(point_estimate) and point_estimate > threshold +def _aligned_bounds(sign: int, ci_low: float, ci_high: float) -> tuple[float, float]: + """The CI of ``sign * estimate``: ``(lower, upper)`` in the EXPECTED direction. + + ``ci_low``/``ci_high`` bracket the RAW estimate. Multiplying an INTERVAL by a + negative number REVERSES it, so the endpoints are re-sorted rather than kept in + place — the min/max swap at ``sign = -1`` is the whole content of this helper, + and doing it in one place is why the ICIR gate and the (v0.9) monotonicity gate + cannot drift apart on the convention. + + ``(NaN, NaN)`` when either bound is unknown — the caller then treats the + estimate as not-convincingly-clearing, and never as a FAIL. + """ + if not (math.isfinite(ci_low) and math.isfinite(ci_high)): + nan = float("nan") + return nan, nan + return min(sign * ci_low, sign * ci_high), max(sign * ci_low, sign * ci_high) + + def _aligned_lower_bound(sign: int, ci_low: float, ci_high: float) -> float: """The lower CI bound of ``sign * estimate`` (the bound in the EXPECTED direction). - ``ci_low``/``ci_high`` bracket the RAW estimate; multiplying the interval by the - hypothesis sign and taking the smaller endpoint yields the worst-case value in - the direction the factor claims. NaN when either bound is unknown — the caller - then treats the estimate as not-convincingly-clearing (never a FAIL). + The worst-case value in the direction the factor claims — what a POSITIVE claim + (an axis PASS) must clear. NaN when either bound is unknown. """ - if not (math.isfinite(ci_low) and math.isfinite(ci_high)): - return float("nan") - return min(sign * ci_low, sign * ci_high) + return _aligned_bounds(sign, ci_low, ci_high)[0] # -- shared fact readers ------------------------------------------------- @@ -500,42 +571,202 @@ def _sample_gate_failures( return failures +def _aligned_net(net: float, gross: float, sign: int) -> float: + """Hypothesis-aligned NET spread: ``sign * gross - cost`` (design §6, v0.8). + + The hypothesis decides WHICH LEG IS LONG. Cost is a drag REGARDLESS of + direction. So the legs are flipped by the sign FIRST and the cost is + subtracted AFTER:: + + cost = gross - net (= fee * multiplier * leg_turnover) + aligned_net = sign * gross - cost + + Before v0.8 this was computed as ``sign * net``, which at ``sign = -1`` + expands to ``-gross + cost``: the trading cost was ADDED BACK, as though + reversing the legs also reversed who pays the fees. It flattered every + sign=-1 factor by exactly ``2 * cost``. + + ``sign = +1`` returns ``net`` UNCHANGED and needs no ``gross`` — algebraically + ``+1 * gross - (gross - net) == net``, and returning ``net`` directly keeps + the old path bit-identical instead of round-tripping it through a subtraction. + + An unknown (non-finite) ``gross`` at ``sign = -1`` yields NaN: the cost cannot + be recovered from ``net`` alone, so the aligned spread is UNKNOWN — and + unknown is neither evidence for nor against (it never convicts, never passes). + """ + if sign >= 0: + return net + if not (math.isfinite(gross) and math.isfinite(net)): + return float("nan") + return -gross - (gross - net) + + def _base_spread(inputs: VerdictInputs) -> float: """Hypothesis-aligned net long-short return at the BASE (1.0x) cost.""" for multiplier, value in inputs.net_long_short_by_cost: if abs(multiplier - 1.0) < 1e-9: - return inputs.expected_ic_sign * value + return _aligned_net( + value, inputs.gross_long_short_mean, inputs.expected_ic_sign + ) return float("nan") def _all_spreads_negative(inputs: VerdictInputs) -> bool: """True only when EVERY known cost scenario is non-positive (design §6). - An empty/unknown set is NOT "all negative" — unknown is not evidence. + An empty/unknown set is NOT "all negative" — unknown is not evidence. A + sign=-1 run with an unknown gross therefore yields NO scenarios and cannot + convict (:func:`_aligned_net` returns NaN, which is filtered out here). """ aligned = [ - inputs.expected_ic_sign * v + _aligned_net(v, inputs.gross_long_short_mean, inputs.expected_ic_sign) for _, v in inputs.net_long_short_by_cost - if math.isfinite(v) ] - return bool(aligned) and all(v <= 0 for v in aligned) + known = [a for a in aligned if math.isfinite(a)] + return bool(known) and all(a <= 0 for a in known) -def _has_in_sample_point_signal(inputs: VerdictInputs, thr: VerdictThresholds) -> bool: - """POINT-estimate in-sample signal: |ICIR|, aligned monotonicity, NW-t, win rate. +# -- the (v0.9) three-valued monotonicity DIRECTION gate ------------------ +# +# Monotonicity is a DIRECTION gate, not a strength gate (v0.7) — but v0.8 revealed +# that its statistic cannot support even that claim as a bare point. Averaging +# per-date rank correlations across dates is heavily ATTENUATED by daily noise: in +# this project's real eleven-factor runs an empirically perfect quantile ladder +# scores only 0.045-0.106, and two factors sat 0.021 apart across a threshold of +# exactly 0.0 with NO dispersion estimate anywhere in the decision. "Point > 0.0" +# on a noisy mean is a coin flip wearing a criterion's clothes — the same defect +# v0.6 fixed for the ICIR by gating on its N_eff lower bound. +# +# So the direction is decided by the CI, and it is THREE-VALUED: +# +# aligned CI low > bar -> HOLDS (may PASS) +# aligned CI high < 0 -> CONTRADICTED (FAIL — a measured reversal) +# otherwise -> UNKNOWN (neither convicts nor acquits) +# +# WHY THE CONTRADICTED TEST IS PINNED AT 0 AND NOT AT THE BAR: they are different +# claims. "Above the bar" is the positive claim, so it moves with a configurable +# (possibly strength-demanding) bar. "Below zero" is REVERSAL — the buckets are +# ordered the wrong way round — which is a fact about the sign, not about how +# strong the project decided to be. Pinning FAIL to a raised bar would convict a +# correctly-ordered-but-weak factor of being reversed, which it is not. + +MONO_HOLDS = "HOLDS" +MONO_CONTRADICTED = "CONTRADICTED" +MONO_UNKNOWN = "UNKNOWN" + +#: Level-2 fallback disclosure: judged on the POOLED magnitude statistic (v0.7 +#: behaviour). Wording kept verbatim from v0.8 — reports and tests read it. +_MONO_POOLED_FALLBACK_NOTE = ( + "monotonicity gate FELL BACK to the pooled 'monotonicity_spearman': " + "the per-date figure ('monotonicity_spearman_by_date', design §6 v0.8) " + "was not supplied or not measurable. The pooled statistic is unbounded " + "and magnitude-sensitive, so this monotonicity judgement is weaker " + "than a v0.8 run's." +) + +#: Level-1 fallback disclosure: the per-date statistic IS available but its CI is +#: not, so the gate reverts to the bare sign of the point (v0.8 behaviour). +#: DELIBERATELY NOT the pooled note's wording — the two are different degradations +#: (which STATISTIC vs which ESTIMATOR OF ITS UNCERTAINTY) and a reader must be +#: able to tell which one happened. +_MONO_POINT_FALLBACK_NOTE = ( + "monotonicity gate REVERTED to the BARE per-date POINT (v0.8 behaviour): no " + "'monotonicity_spearman_by_date_ci_low/high' was supplied, so the direction was " + "decided WITHOUT any dispersion estimate. A cross-date mean of per-date rank " + "correlations is strongly attenuated by daily noise, so a point that merely " + "lands on the right side of the bar is NOT the same evidence a v0.9 CI would be." +) + + +def _monotonicity_direction( + inputs: VerdictInputs, thr: VerdictThresholds +) -> tuple[str, tuple[str, ...]]: + """Three-valued monotonicity DIRECTION + its disclosure notes (design §6, v0.9). + + Returns ``(MONO_HOLDS | MONO_CONTRADICTED | MONO_UNKNOWN, notes)``. + + THE FALLBACK CHAIN, each level DISCLOSED in ``notes`` (never silent): + 1. the per-date CI (v0.9) — three-valued, the intended path. + 2. the per-date POINT (v0.8) — CI absent. TWO-valued, bit-for-bit the v0.8 + rule: aligned point above the bar HOLDS, anything else (including an + unknown) CONTRADICTS. It cannot yield UNKNOWN: without a dispersion + estimate there is nothing to be uncertain WITH, and inventing an UNKNOWN + here would silently convert every pre-v0.9 FAIL into INSUFFICIENT_DATA. + 3. the POOLED point (v0.7) — per-date figure absent too. Same two-valued + rule on the weaker, magnitude-sensitive statistic. A NaN here lands on + CONTRADICTED, exactly as v0.7/v0.8 treated an unmeasurable monotonicity. + + RAW inputs in, hypothesis applied here: this is one of the two places (with + :func:`_aligned_net`) that knows what ``expected_ic_sign`` means. + """ + sign = inputs.expected_ic_sign + bar = thr.min_monotonicity_spearman + + low, high = _aligned_bounds( + sign, + inputs.monotonicity_spearman_by_date_ci_low, + inputs.monotonicity_spearman_by_date_ci_high, + ) + if math.isfinite(low) and math.isfinite(high): + interval = f"[{low:+.4f}, {high:+.4f}]" + if low > bar: + return MONO_HOLDS, ( + f"monotonicity direction HOLDS: the hypothesis-aligned per-date " + f"monotonicity 95% CI {interval} (N_eff-based) lies entirely above " + f"the bar {bar}.", + ) + if high < 0.0: + return MONO_CONTRADICTED, ( + f"monotonicity direction CONTRADICTED: the hypothesis-aligned " + f"per-date monotonicity 95% CI {interval} (N_eff-based) lies " + f"entirely BELOW 0 — the quantile buckets are ordered AGAINST the " + f"stated hypothesis, which is a measured negative finding.", + ) + return MONO_UNKNOWN, ( + f"monotonicity direction is INDISTINGUISHABLE FROM 0: the " + f"hypothesis-aligned per-date monotonicity 95% CI {interval} " + f"(N_eff-based, point {sign * inputs.monotonicity_spearman_by_date:+.4f}) " + f"straddles the bar {bar} / zero, so the evidence is not sufficient to " + f"ASSERT that the direction holds. Not a refutation either — this " + f"neither convicts nor acquits.", + ) + + point = inputs.monotonicity_spearman_by_date + if math.isfinite(point): + state = ( + MONO_HOLDS if _clears_level(sign * point, bar) else MONO_CONTRADICTED + ) + return state, (_MONO_POINT_FALLBACK_NOTE,) + + state = ( + MONO_HOLDS + if _clears_level(sign * inputs.monotonicity_spearman, bar) + else MONO_CONTRADICTED + ) + return state, (_MONO_POOLED_FALLBACK_NOTE, _MONO_POINT_FALLBACK_NOTE) + + +def _has_core_point_signal(inputs: VerdictInputs, thr: VerdictThresholds) -> bool: + """POINT-estimate in-sample signal on the MAGNITUDE metrics: |ICIR|, NW-t, win rate. Every component must be a finite, known number: an unknown is not a signal. This is the v0.5 point check, kept for the FAIL determination — a factor whose POINT metrics do not clear has no signal at all (a negative finding). The CI lower-bound test (design §6, v0.6) is applied SEPARATELY, only to decide PASS vs "promising but unconfirmed" for a factor that DID clear on the point. + + MONOTONICITY IS DELIBERATELY NOT IN HERE (v0.9). It used to be, as a fourth + ``and`` term, which made it silently indistinguishable from the others — and + that is exactly what must not happen now that it can come back UNKNOWN. Folding + a three-valued fact into a boolean would collapse UNKNOWN onto one of the two + other answers; keeping it separate is what lets the axis route an unknown + direction to INSUFFICIENT_DATA while an unknown-direction factor that ALSO + fails here still FAILs. """ - aligned_monotonicity = inputs.expected_ic_sign * inputs.monotonicity_spearman return ( _clears_magnitude(inputs.ic_ir, thr.min_abs_icir) and _clears_magnitude(inputs.ic_nw_t, thr.min_abs_nw_t) and _clears_level(inputs.ic_win_rate, thr.min_ic_win_rate) - and _clears_level(aligned_monotonicity, thr.min_monotonicity_spearman) ) @@ -556,14 +787,29 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi reasons (gate vs no-OOS) are named distinctly. 3. FAIL sufficient data + OOS, but a NEGATIVE finding: the OOS sign is not consistent OR the in-sample POINT metrics do not clear (no signal at - all). POINT-based, so a merely wide CI never manufactures a FAIL. - 4. PASS OOS sign consistent AND point metrics clear AND the ICIR LOWER CI - bound (N_eff-based, expected direction) exceeds the bar — design §6, - v0.6: convincingly above, not the naked point. + all) OR the monotonicity direction is CONTRADICTED (v0.9: its + aligned CI lies entirely below 0). POINT-based on the magnitude + metrics, so a merely wide ICIR CI never manufactures a FAIL. + 3b. INSUFFICIENT_DATA (v0.9) everything else cleared, but the monotonicity + direction is UNKNOWN — its aligned CI straddles 0, so the direction + can be neither asserted nor refuted. + 4. PASS OOS sign consistent AND point metrics clear AND monotonicity + direction HOLDS AND the ICIR LOWER CI bound (N_eff-based, expected + direction) exceeds the bar — design §6, v0.6: convincingly above, + not the naked point. 5. INSUFFICIENT_DATA point metrics clear + OOS consistent, but the ICIR lower CI does NOT clear (or is unknown): promising, yet not confirmed at the pre-registered bar. This is the "gate passed, CI still straddles the bar" case — distinct from the sample gate (step 2). + + WHERE UNKNOWN SITS IN THE ORDER IS THE WHOLE POINT (v0.9). Step 3 runs FIRST and + is checked against the OTHER criteria in full, so an UNKNOWN monotonicity can + never RESCUE a factor that fails on ICIR / NW-t / win rate / OOS consistency: + that factor still FAILs, and the unknown direction is merely disclosed + alongside. Step 3b is reached only when every other criterion has cleared and + the direction ALONE cannot be asserted. "We could not tell" is a reason to + withhold a PASS, never a reason to withhold a FAIL that was independently + earned. """ sign = inputs.expected_ic_sign # 1. FAIL on a KNOWN out-of-sample reversal — before the gate. @@ -592,10 +838,24 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi if reasons: return AxisVerdict(AXIS_INSUFFICIENT_DATA, tuple(reasons)) - # 3. FAIL: a measured negative finding (OOS sign not consistent, or no point - # signal at all). POINT-based — a wide CI is handled below, not here. - point_signal = _has_in_sample_point_signal(inputs, thr) - if (not inputs.oos_sign_consistent) or (not point_signal): + # The monotonicity direction actually gated on (design §6, v0.9): three-valued, + # decided on the per-date CI, with every fallback level DISCLOSED. A silent + # substitution would hide that the decision rests on a weaker statistic (the + # pooled magnitude one the v0.8 fix exists to stop gating on) or on a point with + # no dispersion estimate at all (the v0.9 fix). + mono_state, mono_notes = _monotonicity_direction(inputs, thr) + + # 3. FAIL: a measured negative finding (OOS sign not consistent, no point signal + # at all, or a CONTRADICTED monotonicity direction). Checked BEFORE the + # UNKNOWN branch, so an unknown direction never rescues a factor that failed + # on its own other evidence. POINT-based on the magnitude metrics — a wide + # ICIR CI is handled below, not here. + point_signal = _has_core_point_signal(inputs, thr) + if ( + (not inputs.oos_sign_consistent) + or (not point_signal) + or mono_state == MONO_CONTRADICTED + ): reasons = [] if not inputs.oos_sign_consistent: reasons.append( @@ -606,9 +866,30 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi if not point_signal: reasons.append( "in-sample POINT metrics do NOT clear the predictive thresholds " - "(ICIR / NW-t / win rate / monotonicity)." + "(ICIR / NW-t / win rate)." ) - return AxisVerdict(AXIS_FAIL, tuple(reasons)) + if mono_state == MONO_CONTRADICTED: + reasons.append( + "quantile monotonicity does not hold in the hypothesized direction." + ) + return AxisVerdict(AXIS_FAIL, tuple(reasons) + mono_notes) + + # 3b. UNKNOWN monotonicity with EVERYTHING ELSE cleared -> INSUFFICIENT_DATA + # (v0.9). Reached only after step 3 declined to fail: this branch withholds + # a PASS, it never withholds a FAIL. + if mono_state == MONO_UNKNOWN: + return AxisVerdict( + AXIS_INSUFFICIENT_DATA, + ( + "every other predictive criterion cleared (ICIR / NW-t / win rate " + "point metrics, and the out-of-sample sign is consistent), but the " + "monotonicity DIRECTION cannot be distinguished from 0: its " + "confidence interval straddles zero, so the evidence is NOT " + "sufficient to assert that the direction holds. Withheld, not " + "refuted — an unknown direction is not a PASS and not a FAIL.", + ) + + mono_notes, + ) # 4/5. Point signal present + OOS consistent. The #3 lower-CI test decides # PASS vs "promising but unconfirmed" (INSUFFICIENT_DATA). This is ABOVE @@ -624,7 +905,8 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi f"{thr.min_abs_icir}; point |ICIR| {abs(inputs.ic_ir):.3f}, " f"|NW-t| {abs(inputs.ic_nw_t):.2f}, win rate " f"{inputs.ic_win_rate:.3f}.", - ), + ) + + mono_notes, ) lower_txt = "UNKNOWN" if not math.isfinite(icir_lower) else f"{icir_lower:+.3f}" return AxisVerdict( @@ -635,7 +917,8 @@ def _predictive_axis(inputs: VerdictInputs, thr: VerdictThresholds) -> AxisVerdi f"pre-registered bar {thr.min_abs_icir}: promising, but not confirmed. " f"The sample size passed the gate; the ESTIMATE is still too imprecise " f"to claim a PASS (a wider bar than the raw count implies).", - ), + ) + + mono_notes, ) @@ -932,6 +1215,9 @@ def decide_verdict( "AXIS_NOT_ASSESSED", "AXIS_VERDICTS", "AXIS_NAMES", + "MONO_HOLDS", + "MONO_CONTRADICTED", + "MONO_UNKNOWN", "VerdictThresholds", "VerdictInputs", "AxisVerdict", diff --git a/tests/test_factor_eval_contract.py b/tests/test_factor_eval_contract.py index 093b663..bbae9a8 100644 --- a/tests/test_factor_eval_contract.py +++ b/tests/test_factor_eval_contract.py @@ -10,8 +10,10 @@ from __future__ import annotations +import dataclasses import functools import json +import math import pytest @@ -24,6 +26,7 @@ INSUFFICIENT_DATA, MANDATORY_SECTIONS, REJECT, + VERDICT_KEYS, WATCH, AxisVerdict, EvalConfig, @@ -37,6 +40,18 @@ decide_verdict, ) from analytics.eval.render import MAX_VALUE_CHARS + +# The v0.8 spread fix lived in an exact arithmetic expression, so the lock has to +# be on that expression itself, not on a PASS/FAIL that happens to agree with it. +from analytics.eval.verdict import ( + MONO_CONTRADICTED, + MONO_HOLDS, + MONO_UNKNOWN, + _aligned_net, + _all_spreads_negative, + _base_spread, + _monotonicity_direction, +) from factors.base import Factor from factors.compute.candidates import ( VALUE_FIELDS, @@ -1298,6 +1313,12 @@ def test_negative_sign_factor_is_judged_in_its_own_direction(): ic_nw_t=-3.5, monotonicity_spearman=-0.9, # high-vol bucket underperforms net_long_short_by_cost=((1.0, -0.05),), # QN - Q1 negative: as predicted + # v0.8: aligning a NET spread needs the GROSS leg difference, because the + # cost (gross - net = 0.01) must stay SUBTRACTED after the legs are + # flipped: aligned = -(-0.04) - 0.01 = +0.03. Supplying only the net (as + # this fixture did pre-v0.8) now reads UNKNOWN rather than being scored + # with the costs handed back as profit. + gross_long_short_mean=-0.04, known_factors_supplied=True, incremental_ic_ir=-0.6, # residual IC negative: as predicted incremental_ic_mean=-0.04, @@ -2207,3 +2228,413 @@ def test_template_method_verdict_reflects_a_skipped_oos_section(): assert report.verdict.predictive.verdict == AXIS_INSUFFICIENT_DATA assert any("no out-of-sample split" in r for r in report.verdict.reasons) assert "_Skipped: no oos_split configured_" in report.render() + + +# -------------------------------------------------------------------------- +# v0.8 fix #1: the hypothesis-aligned NET spread must SUBTRACT cost, not add it +# back (design §6, v0.8). The aligned spread is `sign * gross - cost`, never +# `sign * net` — the latter expands at sign=-1 to `-gross + cost`. +# -------------------------------------------------------------------------- + + +def test_aligned_net_at_positive_sign_returns_the_net_untouched(): + """sign=+1 is BIT-IDENTICAL to the pre-v0.8 expression (`1 * net == net`).""" + for gross, net in ((0.05, 0.04), (0.000125, -0.001983), (-0.02, -0.03)): + assert _aligned_net(net, gross, 1) == net + # and it does not even need the gross: an unknown gross must not turn a + # perfectly known sign=+1 spread into UNKNOWN. + assert _aligned_net(0.04, float("nan"), 1) == 0.04 + + +def test_aligned_net_at_negative_sign_flips_the_legs_then_subtracts_cost(): + """The worked example from the v0.8 task card, to the last digit. + + gross = 0.000125, net(1x) = -0.001983 => cost = 0.002108 + aligned = -gross - cost = -0.000125 - 0.002108 = -0.002233 + The pre-v0.8 `sign * net` gave +0.001983 — a POSITIVE spread conjured out of + a factor that loses money gross AND pays 0.002108 to trade. + """ + gross, net = 0.000125, -0.001983 + assert _aligned_net(net, gross, -1) == pytest.approx(-0.002233, abs=1e-12) + # the defective reading, named so a regression cannot quietly restore it + assert _aligned_net(net, gross, -1) != pytest.approx(-1 * net, abs=1e-9) + + +def test_aligned_net_is_always_worse_than_the_gross_flip_by_exactly_the_cost(): + """Cost is a DRAG in both directions: aligned = sign*gross - cost, always.""" + gross, net = 0.000125, -0.001983 + cost = gross - net + assert cost > 0 + assert _aligned_net(net, gross, -1) == pytest.approx(-gross - cost, abs=1e-12) + + +def test_base_spread_reads_the_aligned_cost_subtracted_value_at_sign_minus_one(): + inputs = VerdictInputs( + expected_ic_sign=-1, + net_long_short_by_cost=((1.0, -0.001983), (2.0, -0.004091)), + gross_long_short_mean=0.000125, + ) + assert _base_spread(inputs) == pytest.approx(-0.002233, abs=1e-12) + + +def test_all_spreads_negative_is_true_when_every_aligned_scenario_is_negative(): + """sign=-1, gross known, cost subtracted -> every scenario negative -> the + Tradable axis has a KNOWN failure to convict on.""" + inputs = VerdictInputs( + expected_ic_sign=-1, + net_long_short_by_cost=((1.0, -0.001983), (2.0, -0.004091), (4.0, -0.008307)), + gross_long_short_mean=0.000125, + ) + assert _all_spreads_negative(inputs) is True + r = decide_verdict( + VerdictInputs( + expected_ic_sign=-1, + **_SAMPLE_OK, + **_TRADABLE, + net_long_short_by_cost=( + (1.0, -0.001983), (2.0, -0.004091), (4.0, -0.008307), + ), + gross_long_short_mean=0.000125, + ) + ) + assert r.tradable.verdict == AXIS_FAIL + assert any("EVERY cost scenario" in x for x in r.tradable.reasons) + + +def test_an_unknown_gross_makes_a_negative_sign_aligned_spread_unknown(): + """UNKNOWN NEVER CONVICTS, and never passes either: without the gross the cost + cannot be recovered from the net, so the aligned spread is not a fact.""" + inputs = VerdictInputs( + expected_ic_sign=-1, + net_long_short_by_cost=((1.0, -0.001983), (2.0, -0.004091)), + # gross_long_short_mean deliberately absent -> NaN + ) + assert math.isnan(_base_spread(inputs)) + assert _all_spreads_negative(inputs) is False + r = decide_verdict( + VerdictInputs( + expected_ic_sign=-1, + **_SAMPLE_OK, + **_TRADABLE, + net_long_short_by_cost=((1.0, -0.001983), (2.0, -0.004091)), + ) + ) + assert r.tradable.verdict == AXIS_INSUFFICIENT_DATA + + +def test_positive_sign_tradable_axis_is_unchanged_by_the_v08_fix(): + """The whole sign=+1 world must be bit-identical: same verdicts with the gross + supplied, absent, or nonsense — it is never consulted at sign=+1.""" + for gross in (0.06, float("nan"), -99.0): + r = decide_verdict( + _inputs(**_STRONG, **_OOS_OK, **_TRADABLE, gross_long_short_mean=gross) + ) + assert r.tradable.verdict == AXIS_PASS + kwargs = {**_STRONG, **_OOS_OK, **_TRADABLE} + kwargs["net_long_short_by_cost"] = ((1.0, -0.01), (2.0, -0.03)) + failed = decide_verdict(_inputs(**kwargs, gross_long_short_mean=gross)) + assert failed.tradable.verdict == AXIS_FAIL + + +# -------------------------------------------------------------------------- +# v0.8 fix #2: the Predictive axis gates on the PER-DATE monotonicity, and +# discloses when it had to fall back to the pooled one. +# -------------------------------------------------------------------------- + + +def test_predictive_gate_reads_the_per_date_monotonicity_over_the_pooled_one(): + """When both are present the per-date figure decides — and it decides BOTH + ways, so this is not just 'the new field is accepted'.""" + # pooled says REVERSED, per-date says correctly ordered -> the axis is not + # convicted by the magnitude-sensitive statistic. + rescued = decide_verdict( + _inputs( + **{**_STRONG, "monotonicity_spearman": -0.5, + "monotonicity_spearman_by_date": 0.7}, + **_OOS_OK, + ) + ) + assert rescued.predictive.verdict == AXIS_PASS + # pooled looks fine, per-date says REVERSED -> FAIL. The new gate has teeth. + convicted = decide_verdict( + _inputs( + **{**_STRONG, "monotonicity_spearman": 0.9, + "monotonicity_spearman_by_date": -0.3}, + **_OOS_OK, + ) + ) + assert convicted.predictive.verdict == AXIS_FAIL + assert any("monotonicity" in x for x in convicted.predictive.reasons) + + +def test_a_missing_per_date_monotonicity_falls_back_and_says_so(): + """A pre-v0.8 IR stays judgeable, but the report must not pretend the weaker + statistic is the new one.""" + r = decide_verdict(_inputs(**_STRONG, **_OOS_OK)) # pooled only + assert r.predictive.verdict == AXIS_PASS + assert any("FELL BACK" in x for x in r.predictive.reasons) + assert any("magnitude-sensitive" in x for x in r.predictive.reasons) + + +def test_no_fallback_note_when_the_per_date_monotonicity_is_supplied(): + r = decide_verdict( + _inputs(**{**_STRONG, "monotonicity_spearman_by_date": 0.7}, **_OOS_OK) + ) + assert r.predictive.verdict == AXIS_PASS + assert not any("FELL BACK" in x for x in r.predictive.reasons) + + +def test_the_negative_sign_monotonicity_gate_is_still_direction_aligned(): + """v0.7's direction gate is unchanged: sign=-1 wants a NEGATIVE raw per-date + monotonicity, and a positive one is the reversal that FAILs.""" + common = dict( + expected_ic_sign=-1, **_SAMPLE_OK, **_OOS_OK, + ic_ir=-0.8, ic_ir_ci_low=-1.05, ic_ir_ci_high=-0.55, + ic_win_rate=0.7, ic_nw_t=-3.5, + ) + ok = decide_verdict( + VerdictInputs(**common, monotonicity_spearman_by_date=-0.7) + ) + assert ok.predictive.verdict == AXIS_PASS + reversed_ = decide_verdict( + VerdictInputs(**common, monotonicity_spearman_by_date=0.7) + ) + assert reversed_.predictive.verdict == AXIS_FAIL + + +# -------------------------------------------------------------------------- +# v0.9: the monotonicity DIRECTION gate is decided by the per-date CI and is +# THREE-VALUED (holds / contradicted / UNKNOWN). +# +# Why: the v0.8 per-date statistic turned out to be heavily attenuated by daily +# noise — an empirically perfect ladder scores ~0.05-0.11 — while the gate sat at +# a bare 0.0 with no dispersion estimate at all. Two real factors landed 0.021 +# apart across that boundary. +# -------------------------------------------------------------------------- + + +#: CI bounds only; the point is supplied too, since a real payload always carries +#: both and the reasons quote it. +def _mono(point: float, low: float, high: float) -> dict: + return { + "monotonicity_spearman_by_date": point, + "monotonicity_spearman_by_date_ci_low": low, + "monotonicity_spearman_by_date_ci_high": high, + } + + +def test_the_monotonicity_direction_gate_is_three_valued_on_the_ci(): + """The three states, each reached by moving ONLY the interval.""" + # (a) entirely above the bar -> the direction may be asserted -> PASS + holds = decide_verdict( + _inputs(**{**_STRONG, **_mono(0.08, 0.03, 0.13)}, **_OOS_OK) + ) + assert holds.predictive.verdict == AXIS_PASS + + # (b) entirely below 0 -> a MEASURED reversal -> FAIL + contradicted = decide_verdict( + _inputs(**{**_STRONG, **_mono(-0.08, -0.13, -0.03)}, **_OOS_OK) + ) + assert contradicted.predictive.verdict == AXIS_FAIL + assert any("CONTRADICTED" in x for x in contradicted.predictive.reasons) + + # (c) straddling 0 -> UNKNOWN -> neither convicts nor acquits. + # THE POINT IS POSITIVE (0.08, which v0.8 would have waved through as a + # direction that "holds"): it is the INTERVAL that withholds the PASS. + unknown = decide_verdict( + _inputs(**{**_STRONG, **_mono(0.08, -0.02, 0.18)}, **_OOS_OK) + ) + assert unknown.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert any("straddles" in x for x in unknown.predictive.reasons) + assert any( + "not a PASS and not a FAIL" in x for x in unknown.predictive.reasons + ) + + +def test_an_unknown_monotonicity_does_not_rescue_a_factor_that_fails_elsewhere(): + """THE routing rule. An unknown direction withholds a PASS; it must never + withhold a FAIL that the other evidence already earned — otherwise 'we could not + measure the ladder' would upgrade every genuinely bad factor from Reject to + INSUFFICIENT-DATA, which is the exact opposite of quick-to-reject.""" + straddling = _mono(0.08, -0.02, 0.18) + + # each of the other predictive criteria, broken ONE AT A TIME + weak_icir = {**_STRONG, "ic_ir": 0.05, "ic_ir_ci_low": -0.2, "ic_ir_ci_high": 0.3} + weak_nw_t = {**_STRONG, "ic_nw_t": 0.4} + weak_win = {**_STRONG, "ic_win_rate": 0.41} + for broken in (weak_icir, weak_nw_t, weak_win): + r = decide_verdict(_inputs(**{**broken, **straddling}, **_OOS_OK)) + assert r.predictive.verdict == AXIS_FAIL + assert r.verdict == REJECT + # and the unknown direction is DISCLOSED alongside, not swallowed + assert any("INDISTINGUISHABLE FROM 0" in x for x in r.predictive.reasons) + + # out-of-sample inconsistency is "elsewhere" too + oos_bad = decide_verdict( + _inputs( + **{**_STRONG, **straddling}, + oos_available=True, + oos_sign_consistent=False, + ) + ) + assert oos_bad.predictive.verdict == AXIS_FAIL + + # ... and with everything else intact the SAME unknown yields INSUFFICIENT_DATA + alone = decide_verdict(_inputs(**{**_STRONG, **straddling}, **_OOS_OK)) + assert alone.predictive.verdict == AXIS_INSUFFICIENT_DATA + + +def test_an_unknown_monotonicity_axis_does_not_become_a_reject_label(): + """Predictive INSUFFICIENT_DATA + nothing failing = INSUFFICIENT-DATA, never + Reject: the deployment derivation must not read a withheld PASS as a failure.""" + r = decide_verdict( + _inputs(**{**_STRONG, **_mono(0.08, -0.02, 0.18)}, **_OOS_OK) + ) + assert r.predictive.verdict == AXIS_INSUFFICIENT_DATA + assert r.verdict == INSUFFICIENT_DATA + + +def test_the_aligned_monotonicity_ci_swaps_min_and_max_at_sign_minus_one(): + """Multiplying an interval by -1 REVERSES it. A low-vol factor (sign=-1) whose + RAW per-date CI is entirely NEGATIVE is a factor whose ALIGNED CI is entirely + positive — i.e. its hypothesis holds. Getting the swap wrong reads it backwards.""" + thr = VerdictThresholds() + # raw [-0.13, -0.03] at sign=-1 -> aligned [+0.03, +0.13] -> HOLDS + state, reasons = _monotonicity_direction( + VerdictInputs(expected_ic_sign=-1, **_mono(-0.08, -0.13, -0.03)), thr + ) + assert state == MONO_HOLDS + assert "[+0.0300, +0.1300]" in reasons[0] # re-sorted, not left reversed + + # raw [+0.03, +0.13] at sign=-1 -> aligned [-0.13, -0.03] -> CONTRADICTED + state, _ = _monotonicity_direction( + VerdictInputs(expected_ic_sign=-1, **_mono(0.08, 0.03, 0.13)), thr + ) + assert state == MONO_CONTRADICTED + + # the same raw intervals at sign=+1 read the other way round + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(0.08, 0.03, 0.13)), thr + )[0] == MONO_HOLDS + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(-0.08, -0.13, -0.03)), thr + )[0] == MONO_CONTRADICTED + + # end to end: the sign=-1 PASS survives the new gate + common = dict( + expected_ic_sign=-1, **_SAMPLE_OK, **_OOS_OK, + ic_ir=-0.8, ic_ir_ci_low=-1.05, ic_ir_ci_high=-0.55, + ic_win_rate=0.7, ic_nw_t=-3.5, + ) + assert decide_verdict( + VerdictInputs(**common, **_mono(-0.08, -0.13, -0.03)) + ).predictive.verdict == AXIS_PASS + assert decide_verdict( + VerdictInputs(**common, **_mono(0.08, 0.03, 0.13)) + ).predictive.verdict == AXIS_FAIL + + +def test_the_direction_gate_is_strict_and_a_ci_touching_the_bar_is_unknown(): + """`> bar`, like every other level comparison in this module: an aligned lower + bound sitting exactly ON 0.0 has not shown the direction holds.""" + thr = VerdictThresholds() + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(0.09, 0.0, 0.18)), thr + )[0] == MONO_UNKNOWN + # and an upper bound exactly at 0 is not a demonstrated reversal either + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(-0.09, -0.18, 0.0)), thr + )[0] == MONO_UNKNOWN + + +def test_a_raised_bar_withholds_a_pass_without_calling_the_factor_reversed(): + """The PASS side moves with the (configurable) bar; the FAIL side stays pinned + at 0. A correctly-ordered-but-weak ladder is UNKNOWN under a strength bar — it + is NOT 'reversed', which is a claim about the sign.""" + strict = VerdictThresholds(min_monotonicity_spearman=0.5) + state, reasons = _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(0.08, 0.03, 0.13)), strict + ) + assert state == MONO_UNKNOWN + assert any("straddles the bar 0.5" in x for x in reasons) + # genuinely reversed is still CONTRADICTED under the same raised bar + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(-0.08, -0.13, -0.03)), strict + )[0] == MONO_CONTRADICTED + + +def test_the_monotonicity_fallback_chain_has_two_disclosed_levels(): + """CI absent -> the bare per-date point (v0.8); per-date point absent too -> + the pooled magnitude statistic (v0.7). Each level is ANNOUNCED in the reasons, + because judging on a weaker statistic while presenting a v0.9 verdict would be + the silent degradation this whole layer exists to prevent.""" + thr = VerdictThresholds() + + # level 1: point present, CI absent -> two-valued, v0.8 behaviour + holds, notes = _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, monotonicity_spearman_by_date=0.7), thr + ) + assert holds == MONO_HOLDS + assert any("REVERTED to the BARE per-date POINT" in n for n in notes) + assert not any("FELL BACK to the pooled" in n for n in notes) + contradicted, _ = _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, monotonicity_spearman_by_date=-0.3), thr + ) + # NOT unknown: without a dispersion estimate there is nothing to be uncertain + # with, and inventing an UNKNOWN would turn every pre-v0.9 FAIL into + # INSUFFICIENT_DATA behind the reader's back. + assert contradicted == MONO_CONTRADICTED + + # level 2: per-date figure absent too -> the pooled one, both notes present + pooled, notes2 = _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, monotonicity_spearman=0.9), thr + ) + assert pooled == MONO_HOLDS + assert any("FELL BACK to the pooled" in n for n in notes2) + assert any("REVERTED to the BARE per-date POINT" in n for n in notes2) + + # nothing at all: unmeasurable monotonicity still lands where v0.7/v0.8 put it + assert _monotonicity_direction(VerdictInputs(expected_ic_sign=+1), thr)[0] \ + == MONO_CONTRADICTED + + # and the levels surface through the real axis, not just the helper + axis = decide_verdict( + _inputs(**{**_STRONG, "monotonicity_spearman_by_date": 0.7}, **_OOS_OK) + ).predictive + assert axis.verdict == AXIS_PASS + assert any("REVERTED to the BARE per-date POINT" in x for x in axis.reasons) + + +def test_a_supplied_ci_is_used_even_when_it_disagrees_with_the_point(): + """The CI is the gate, not a decoration on the point. A point above the bar with + an interval that straddles 0 must NOT pass — that is the v0.9 fix — and a point + below 0 with an interval that clears must not be convicted on the point.""" + thr = VerdictThresholds() + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(0.9, -0.05, 1.85)), thr + )[0] == MONO_UNKNOWN + assert _monotonicity_direction( + VerdictInputs(expected_ic_sign=+1, **_mono(-0.4, 0.05, 0.20)), thr + )[0] == MONO_HOLDS + # a HALF-supplied CI is not a CI: fall back rather than guess the other end + half, notes = _monotonicity_direction( + VerdictInputs( + expected_ic_sign=+1, + monotonicity_spearman_by_date=0.7, + monotonicity_spearman_by_date_ci_low=0.03, + ), + thr, + ) + assert half == MONO_HOLDS + assert any("REVERTED to the BARE per-date POINT" in n for n in notes) + + +def test_the_return_risk_verdict_keys_document_the_monotonicity_ci(): + """VERDICT_KEYS is the documented contract of what the verdict reads; a gate + input missing from it is an undocumented dependency.""" + assert "monotonicity_spearman_by_date_ci_low" in VERDICT_KEYS["return_risk"] + assert "monotonicity_spearman_by_date_ci_high" in VERDICT_KEYS["return_risk"] + # every documented return_risk key is a real VerdictInputs field + fields = {f.name for f in dataclasses.fields(VerdictInputs)} + for key in VERDICT_KEYS["return_risk"]: + assert key in fields diff --git a/tests/test_factor_eval_standard.py b/tests/test_factor_eval_standard.py index 499e604..6c9aab5 100644 --- a/tests/test_factor_eval_standard.py +++ b/tests/test_factor_eval_standard.py @@ -48,10 +48,14 @@ half_life, hypothesis_win_rate, information_ratio_ci, + mean_by_date_spearman, mean_ci, newey_west_lag, newey_west_t, sortino, + spearman, + spearman_by_date, + spearman_series_by_date, ) from analytics.factor import compute_ic from factors.spec import FactorSpec @@ -1896,3 +1900,313 @@ def test_factors_layer_never_imports_analytics(): ) ] assert offenders == [] + + +# -------------------------------------------------------------------------- +# v0.8 fix #2: monotonicity_spearman_by_date — a RANK statistic for a rank axis. +# -------------------------------------------------------------------------- + + +def _quantile_frame(rows: list[list[float]]) -> pd.DataFrame: + """dates x buckets(1..N), the shape ``quantile_return_matrix`` produces.""" + return pd.DataFrame( + rows, + index=pd.bdate_range("2024-01-02", periods=len(rows), name=DATE), + columns=pd.Index(range(1, len(rows[0]) + 1), name="quantile"), + ) + + +def test_per_date_monotonicity_survives_the_outlier_days_that_flip_the_pooled_one(): + """THE proof of the v0.8 monotonicity fix. + + Every single date orders its buckets EXACTLY as the hypothesis says (low + bucket -> low return, monotone up). But on two days the market only realizes + returns in the bottom three buckets, and those returns are ~1000x the normal + scale. Those two days dominate the cross-date ARITHMETIC MEAN of buckets 1-3 + and drag them above buckets 4-5 — so the POOLED Spearman reads the factor as + REVERSED even though no individual day ever was. + + This is the PR-K / PR-G "smile" shape (one outlier bucket driving the pooled + figure) reproduced in miniature, and it is why a rank-based Predictive axis + must not be gated on a magnitude-sensitive statistic. + """ + normal = [0.001, 0.002, 0.003, 0.004, 0.005] + outlier = [1.0, 1.1, 1.2, float("nan"), float("nan")] + frame = _quantile_frame([normal] * 20 + [outlier] * 2) + + bucket_means = [frame[q].mean() for q in frame.columns] + pooled = spearman([float(q) for q in frame.columns], bucket_means) + by_date = spearman_by_date(frame) + + assert pooled == pytest.approx(-0.5) # REVERSED — the defect + assert by_date == pytest.approx(1.0) # every day was perfectly ordered + # and the direction gate (v0.7, threshold 0.0) reads them oppositely + assert pooled < 0.0 < by_date + + +def test_per_date_monotonicity_matches_the_pooled_one_on_clean_monotone_data(): + """No outliers, no gaps: the two statistics must agree, so the v0.8 field is a + ROBUSTNESS change, not a different measurement.""" + frame = _quantile_frame([[0.001, 0.002, 0.003, 0.004, 0.005]] * 12) + bucket_means = [frame[q].mean() for q in frame.columns] + assert spearman([float(q) for q in frame.columns], bucket_means) == pytest.approx(1.0) + assert spearman_by_date(frame) == pytest.approx(1.0) + # and it reads a cleanly REVERSED panel as -1.0, not as "no opinion" + assert spearman_by_date(_quantile_frame([[0.005, 0.004, 0.003, 0.002, 0.001]] * 12)) \ + == pytest.approx(-1.0) + + +def test_a_date_with_fewer_than_three_finite_buckets_is_skipped_not_counted(): + """Two points always give a perfect +/-1 rank correlation, which would be pure + noise injected at full weight. Such a date contributes NOTHING — not a zero.""" + good = [0.001, 0.002, 0.003, 0.004, 0.005] + thin = [0.005, 0.004, float("nan"), float("nan"), float("nan")] # 2 buckets, reversed + with_thin = spearman_by_date(_quantile_frame([good] * 6 + [thin] * 3)) + without_thin = spearman_by_date(_quantile_frame([good] * 6)) + assert with_thin == pytest.approx(without_thin) == pytest.approx(1.0) + # a three-bucket date, by contrast, IS counted + three = [0.005, 0.004, 0.003, float("nan"), float("nan")] # reversed + assert spearman_by_date(_quantile_frame([good] * 3 + [three] * 3)) == pytest.approx(0.0) + + +def test_per_date_monotonicity_is_unknown_rather_than_zero_when_no_date_qualifies(): + nan = float("nan") + assert math.isnan(spearman_by_date(_quantile_frame([[0.1, 0.2, nan, nan, nan]] * 5))) + # all buckets tied on every date: Spearman undefined, so UNKNOWN, not 0.0 + assert math.isnan(spearman_by_date(_quantile_frame([[0.2] * 5] * 5))) + assert math.isnan(spearman_by_date(pd.DataFrame())) + + +def test_return_risk_reports_both_monotonicity_statistics(rng): + """The pooled field is KEPT (historical comparability) alongside the new one.""" + panel, factor = planted_signal_setup(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + payload = report.by_name()["return_risk"].payload + assert payload["monotonicity_spearman"] == pytest.approx(1.0) + by_date = payload["monotonicity_spearman_by_date"] + assert math.isfinite(by_date) and -1.0 <= by_date <= 1.0 + + +# -------------------------------------------------------------------------- +# v0.9 fix: the per-date monotonicity SERIES + its N_eff CI. The point figure +# stays BIT-IDENTICAL to v0.8 — this is a refactor plus an addition, not a +# re-measurement. +# -------------------------------------------------------------------------- + + +def _v08_spearman_by_date(quantile_returns: pd.DataFrame, min_buckets: int = 3) -> float: + """The v0.8 ``spearman_by_date`` body, VERBATIM — the bit-identity oracle. + + Frozen here on purpose: v0.9 re-expresses the same reduction as the mean of a + new per-date series, and "the returned float did not move" is a claim that needs + the OLD code to compare against, not a re-reading of the new code. + """ + if quantile_returns.empty or quantile_returns.shape[1] < min_buckets: + return float("nan") + index = [float(q) for q in quantile_returns.columns] + daily: list[float] = [] + for _, row in quantile_returns.iterrows(): + values = [float(v) for v in row.to_numpy(dtype=float)] + if sum(1 for v in values if math.isfinite(v)) < min_buckets: + continue + rho = spearman(index, values) + if math.isfinite(rho): + daily.append(rho) + if not daily: + return float("nan") + return float(sum(daily) / len(daily)) + + +def _same_float(a: float, b: float) -> bool: + """Exact equality, with NaN == NaN. No tolerance: this is a bit-identity check.""" + return (math.isnan(a) and math.isnan(b)) or a == b + + +def test_spearman_by_date_is_bit_identical_to_the_v08_implementation(): + """Every v0.8 fixture in this file, plus a messy random panel, must return the + EXACT same float — not approx. A v0.9 report field that drifted in the last bit + would silently break cross-run comparability with the ten factors already + evaluated under v0.8.""" + nan = float("nan") + normal = [0.001, 0.002, 0.003, 0.004, 0.005] + fixtures = [ + # the four v0.8 test fixtures, verbatim + _quantile_frame([normal] * 20 + [[1.0, 1.1, 1.2, nan, nan]] * 2), + _quantile_frame([normal] * 12), + _quantile_frame([[0.005, 0.004, 0.003, 0.002, 0.001]] * 12), + _quantile_frame([normal] * 6 + [[0.005, 0.004, nan, nan, nan]] * 3), + _quantile_frame([normal] * 3 + [[0.005, 0.004, 0.003, nan, nan]] * 3), + _quantile_frame([[0.1, 0.2, nan, nan, nan]] * 5), + _quantile_frame([[0.2] * 5] * 5), + pd.DataFrame(), + ] + # ... and a panel whose per-date values are irrational-ish, so a changed + # summation order (Series.mean vs the sequential sum) WOULD show up. + rng = np.random.default_rng(20260721) + messy = rng.normal(size=(97, 5)) / 3.0 + messy[messy > 1.6] = np.nan + fixtures.append(_quantile_frame(messy.tolist())) + + for frame in fixtures: + assert _same_float(spearman_by_date(frame), _v08_spearman_by_date(frame)) + # and the messy one really does exercise a non-trivial average + tail = spearman_by_date(fixtures[-1]) + assert math.isfinite(tail) and tail not in (-1.0, 0.0, 1.0) + + +def test_the_per_date_series_omits_skipped_dates_rather_than_holding_them(): + """A skipped date must be ABSENT from the index, not a NaN row: mean_ci reads + N_eff off this series, and a NaN row would be dropped anyway while an invented + 0.0 would be a fabricated observation.""" + nan = float("nan") + good = [0.001, 0.002, 0.003, 0.004, 0.005] + thin = [0.005, 0.004, nan, nan, nan] # 2 finite buckets -> skipped + tied = [0.2] * 5 # all tied -> Spearman undefined -> skipped + frame = _quantile_frame([good, thin, good, tied, good]) + + series = spearman_series_by_date(frame) + assert list(series.index) == [frame.index[0], frame.index[2], frame.index[4]] + assert series.notna().all() + assert list(series) == [pytest.approx(1.0)] * 3 + assert series.index.name == frame.index.name + # and the point figure is exactly this series' mean + assert _same_float(spearman_by_date(frame), mean_by_date_spearman(series)) + + +def test_the_per_date_series_is_empty_not_all_nan_when_no_date_qualifies(): + """'No observation' and 'an unknown observation' are different facts, and only + the first is true here — an all-NaN series would let a downstream reader believe + dates were measured and came back unknown.""" + nan = float("nan") + for frame in ( + _quantile_frame([[0.1, 0.2, nan, nan, nan]] * 5), # never 3 finite buckets + _quantile_frame([[0.2] * 5] * 5), # always tied + pd.DataFrame(), # nothing at all + _quantile_frame([[0.1, 0.2]] * 5), # fewer buckets than min + ): + series = spearman_series_by_date(frame) + assert isinstance(series, pd.Series) + assert series.empty and len(series) == 0 + assert series.isna().sum() == 0 # empty, NOT a column of NaN + assert math.isnan(mean_by_date_spearman(series)) + + +def test_return_risk_attaches_an_n_eff_ci_to_the_per_date_monotonicity(rng): + """v0.9: the direction gate reads an INTERVAL, so the section must emit one — + and it must describe the very same observations as the point.""" + panel, factor = planted_signal_setup(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ) + payload = report.by_name()["return_risk"].payload + for key in ( + "monotonicity_spearman_by_date_se", + "monotonicity_spearman_by_date_ci_low", + "monotonicity_spearman_by_date_ci_high", + "monotonicity_spearman_by_date_n_eff", + ): + assert key in payload + + low = payload["monotonicity_spearman_by_date_ci_low"] + high = payload["monotonicity_spearman_by_date_ci_high"] + point = payload["monotonicity_spearman_by_date"] + assert math.isfinite(low) and math.isfinite(high) + assert low < point < high + # symmetric around the point, i.e. the CI is OF this point and not of some + # other reduction of the panel + assert (low + high) / 2.0 == pytest.approx(point) + n_eff = payload["monotonicity_spearman_by_date_n_eff"] + n_dates = payload["monotonicity_spearman_by_date_n_dates"] + assert 1.0 <= n_eff <= n_dates # N_eff can never exceed the observations + assert n_dates > 0 + + +def test_the_monotonicity_ci_is_the_shared_mean_ci_on_the_shared_series(rng): + """Not a second CI implementation: the payload must equal mean_ci() applied to + spearman_series_by_date() on the same quantile matrix.""" + panel, factor = planted_signal_setup(rng) + ir = build_eval_ir(factor, make_spec(), make_cfg(), EvalContext(price_panel=panel)) + expected = mean_ci(spearman_series_by_date(ir.quantile_returns)) + + payload = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), EvalContext(price_panel=panel) + ).by_name()["return_risk"].payload + assert payload["monotonicity_spearman_by_date_ci_low"] == expected["ci_low"] + assert payload["monotonicity_spearman_by_date_ci_high"] == expected["ci_high"] + assert payload["monotonicity_spearman_by_date_se"] == expected["se"] + assert payload["monotonicity_spearman_by_date_n_eff"] == expected["n_eff"] + + +# -------------------------------------------------------------------------- +# v0.8 fix #1 at the standard layer: the aligned spread series/CI. +# -------------------------------------------------------------------------- + + +def _aligned_mean_from_ci(payload) -> float: + """The aligned base spread's MEAN, recovered from its symmetric N_eff CI.""" + return (payload["net_long_short_base_ci_low"] + payload["net_long_short_base_ci_high"]) / 2.0 + + +def test_aligned_base_spread_subtracts_cost_after_flipping_the_legs(rng): + """sign=-1: aligned = -gross - cost, NOT -(gross - cost) = -net. + + Recovered from the payload's own numbers, so this pins the real evaluator + output rather than a re-derivation of the formula. + """ + panel, factor = planted_signal_setup(rng, sign=-1) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(expected_ic_sign=-1), make_cfg(), + EvalContext(price_panel=panel, fee_rate=0.001), + ) + payload = report.by_name()["return_risk"].payload + gross = payload["gross_long_short_mean"] + net = payload["net_long_short_by_cost"][1.0] + cost = gross - net + assert cost > 0 # trading is never free + aligned = _aligned_mean_from_ci(payload) + assert aligned == pytest.approx(-gross - cost, rel=1e-9) + # the pre-v0.8 reading handed the cost back as profit; it must be exactly + # 2 * cost better than the truth, and must NOT be what we report. + assert aligned == pytest.approx(-net - 2 * cost, rel=1e-9) + assert aligned < -net + + +def test_aligned_base_spread_is_bit_identical_to_the_net_at_positive_sign(rng): + """The sign=+1 regression lock: aligned mean == net mean, exactly.""" + panel, factor = planted_signal_setup(rng) + report = StandardFactorEvaluator().evaluate( + factor, make_spec(), make_cfg(), + EvalContext(price_panel=panel, fee_rate=0.001), + ) + payload = report.by_name()["return_risk"].payload + assert _aligned_mean_from_ci(payload) == pytest.approx( + payload["net_long_short_by_cost"][1.0], rel=1e-12 + ) + + +def test_a_higher_fee_makes_the_negative_sign_aligned_spread_worse_not_better(rng): + """The behavioural signature of the defect, at the standard layer. + + The SAME trades at a higher fee must degrade the reported aligned performance. + Under the pre-v0.8 `sign * net` the extra fee was ADDED to a sign=-1 factor's + aligned spread, so raising the fee IMPROVED its reported NAV — a factor could + be made to look better simply by charging it more to trade. + """ + panel, factor = planted_signal_setup(rng, sign=-1) + spec, cfg = make_spec(expected_ic_sign=-1), make_cfg() + cheap = StandardFactorEvaluator().evaluate( + factor, spec, cfg, EvalContext(price_panel=panel, fee_rate=0.0005) + ).by_name()["return_risk"].payload + dear = StandardFactorEvaluator().evaluate( + factor, spec, cfg, EvalContext(price_panel=panel, fee_rate=0.004) + ).by_name()["return_risk"].payload + + # same trades either way: the gross leg difference is untouched by the fee + assert dear["gross_long_short_mean"] == pytest.approx( + cheap["gross_long_short_mean"], rel=1e-12 + ) + assert _aligned_mean_from_ci(dear) < _aligned_mean_from_ci(cheap) + assert dear["aligned_spread_final_nav"] < cheap["aligned_spread_final_nav"] + assert dear["aligned_spread_annual_return"] < cheap["aligned_spread_annual_return"]