From b70487a71c40a9acbebd535014adef9d5b002aaf Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 14:33:07 -0700 Subject: [PATCH 1/9] feat(factors): pre-register lookback_depth on the closing factors (D4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The D3 tail-recompute engine HARD-REQUIRES spec.lookback_depth to size its incremental overlap window. Declare the transitive lookback depth (design §六.18: headline + nested baseline, NOT the headline window) on every closing factor plus the remaining registered daily factors, each derived from its own code constants so a re-parameterized window yields the matching depth: * book: value_ep/value_bp = 1, volatility_w = w + 1 (pct_change adds a day) * minute, no baseline: jump/amp_marginal/minute_ideal/intraday_amp_cut = lookback_days * minute, nested baseline: volume_peak_count/peak_interval_kurtosis/valley_*/ ridge_minute_return/peak_ridge_amount_ratio = lookback_days + VOLUME_PRV_BASELINE_DAYS (20 + 20 = 40) * daily: momentum/reversal/overnight = w + 1, liquidity = w, financial = 1 Parameterized test pins all 14 values and proves the declaration tracks the constructor param (a non-default window cannot silently keep the default depth). This is a pre-registration: changing a value is a definition-adjacent change. --- factors/compute/candidates.py | 19 +++ factors/compute/financial.py | 5 + .../minute/amp_marginal_anomaly_vol.py | 4 + factors/compute/minute/intraday_amp_cut.py | 4 + factors/compute/minute/jump_amount_corr.py | 4 + .../compute/minute/minute_ideal_amplitude.py | 3 + .../compute/minute/peak_interval_kurtosis.py | 5 + .../compute/minute/peak_ridge_amount_ratio.py | 5 + factors/compute/minute/ridge_minute_return.py | 5 + .../compute/minute/valley_price_quantile.py | 7 + .../compute/minute/valley_relative_vwap.py | 5 + .../compute/minute/valley_ridge_vwap_ratio.py | 5 + factors/compute/minute/volume_peak_count.py | 6 + factors/compute/momentum.py | 4 + tests/test_factor_lookback_depth.py | 124 ++++++++++++++++++ 15 files changed, 205 insertions(+) create mode 100644 tests/test_factor_lookback_depth.py diff --git a/factors/compute/candidates.py b/factors/compute/candidates.py index 7770e6d..ea8a135 100644 --- a/factors/compute/candidates.py +++ b/factors/compute/candidates.py @@ -98,6 +98,10 @@ def spec(self) -> FactorSpec: overnight_boundary=base.overnight_boundary, family="reversal", min_history_bars=base.min_history_bars, + # D4 pre-registration: reversal IS -momentum, same inputs, so the + # transitive lookback depth is inherited verbatim from the base spec + # (close[t] and close[t-window] -> window + 1 trailing trading days). + lookback_depth=base.lookback_depth, ) def compute(self, panel: pd.DataFrame) -> pd.Series: @@ -162,6 +166,11 @@ def spec(self) -> FactorSpec: # pct_change loses row 0 and the rolling std needs ``window`` returns # -> the leading ``window`` rows are NaN. min_history_bars=self._window, + # D4 pre-registration: the value at t is std of returns[t-window+1..t], + # and returns[s]=close[s]/close[s-1]-1, so the earliest input close is + # close[t-window] -> transitive depth = window + 1 trailing trading days + # (the pct_change adds one day beyond the headline window; §六.18). + lookback_depth=self._window + 1, ) def compute(self, panel: pd.DataFrame) -> pd.Series: @@ -233,6 +242,9 @@ def spec(self) -> FactorSpec: # rolling mean over ``window`` amounts -> first valid row is # ``window-1`` (no pct_change involved, unlike volatility). min_history_bars=self._window - 1, + # D4 pre-registration: the rolling mean at t reads amount[t-window+1..t] + # -> transitive depth = window trailing trading days (no return diff). + lookback_depth=self._window, ) def compute(self, panel: pd.DataFrame) -> pd.Series: @@ -324,6 +336,10 @@ def spec(self) -> FactorSpec: # row 0 has no prior close and the rolling sum needs ``window`` full # overnight returns -> the leading ``window`` rows are NaN. min_history_bars=self._window, + # D4 pre-registration: term t reads open[t] and close[t-1]; summed over + # t-window+1..t the earliest input is close[t-window] -> transitive + # depth = window + 1 trailing trading days (§六.18). + lookback_depth=self._window + 1, ) def compute(self, panel: pd.DataFrame) -> pd.Series: @@ -395,6 +411,9 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="value", min_history_bars=0, + # D4 pre-registration: a bare same-day column select (1/pe or 1/pb); + # the value at d reads only d's published ratio -> depth = 1. + lookback_depth=1, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/financial.py b/factors/compute/financial.py index 97ef172..604d5fe 100644 --- a/factors/compute/financial.py +++ b/factors/compute/financial.py @@ -88,6 +88,11 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family=family, min_history_bars=0, + # D4 pre-registration: a bare same-day column select of the + # ann_date-aligned value; the value at d reads only the single latest + # as-of column entry -> depth = 1 (the ann_date lookback lives in the + # 400-day fina revision horizon, not in this trailing-window depth). + lookback_depth=1, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/amp_marginal_anomaly_vol.py b/factors/compute/minute/amp_marginal_anomaly_vol.py index bbf3780..4bb68c4 100644 --- a/factors/compute/minute/amp_marginal_anomaly_vol.py +++ b/factors/compute/minute/amp_marginal_anomaly_vol.py @@ -326,6 +326,10 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18): a trailing ``lookback_days`` trading-day + # pool of (|Δamp|, r) pairs with NO nested baseline (the 5min bars are + # derived within-day) -> depth = lookback_days trading days. + lookback_depth=self._lookback_days, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/intraday_amp_cut.py b/factors/compute/minute/intraday_amp_cut.py index ba10de0..f54bfbe 100644 --- a/factors/compute/minute/intraday_amp_cut.py +++ b/factors/compute/minute/intraday_amp_cut.py @@ -448,6 +448,10 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18): a trailing ``lookback_days`` VALID + # trading-day window with NO nested baseline (the amplitude-cut stats + # are within-day) -> depth = lookback_days trading days. + lookback_depth=self._lookback_days, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/jump_amount_corr.py b/factors/compute/minute/jump_amount_corr.py index 8109d24..5bb2dca 100644 --- a/factors/compute/minute/jump_amount_corr.py +++ b/factors/compute/minute/jump_amount_corr.py @@ -260,6 +260,10 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18): the value at d is a trailing-window + # correlation over ``lookback_days`` trading days with NO nested + # baseline -> transitive depth = lookback_days trailing trading days. + lookback_depth=self._lookback_days, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/minute_ideal_amplitude.py b/factors/compute/minute/minute_ideal_amplitude.py index 280c258..c121a7b 100644 --- a/factors/compute/minute/minute_ideal_amplitude.py +++ b/factors/compute/minute/minute_ideal_amplitude.py @@ -273,6 +273,9 @@ def spec(self) -> FactorSpec: overnight_boundary="crossed_disclosed", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18): a trailing ``lookback_days`` trading-day + # pool with NO nested baseline -> depth = lookback_days trading days. + lookback_depth=self._lookback_days, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/peak_interval_kurtosis.py b/factors/compute/minute/peak_interval_kurtosis.py index d804069..baffa6a 100644 --- a/factors/compute/minute/peak_interval_kurtosis.py +++ b/factors/compute/minute/peak_interval_kurtosis.py @@ -400,6 +400,11 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18, NESTED lookback): trailing + # ``lookback_days`` valid days, each bar classified against a + # strictly-prior ``VOLUME_PRV_BASELINE_DAYS``-day same-slot baseline -> + # transitive depth = lookback_days + baseline_days trailing trading days. + lookback_depth=self._lookback_days + VOLUME_PRV_BASELINE_DAYS, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/peak_ridge_amount_ratio.py b/factors/compute/minute/peak_ridge_amount_ratio.py index f606969..6c00387 100644 --- a/factors/compute/minute/peak_ridge_amount_ratio.py +++ b/factors/compute/minute/peak_ridge_amount_ratio.py @@ -530,6 +530,11 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18, NESTED lookback): trailing + # ``lookback_days`` valid days, each bar classified against a + # strictly-prior ``VOLUME_PRV_BASELINE_DAYS``-day same-slot baseline -> + # transitive depth = lookback_days + baseline_days trailing trading days. + lookback_depth=self._lookback_days + VOLUME_PRV_BASELINE_DAYS, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/ridge_minute_return.py b/factors/compute/minute/ridge_minute_return.py index 0739569..b43f439 100644 --- a/factors/compute/minute/ridge_minute_return.py +++ b/factors/compute/minute/ridge_minute_return.py @@ -494,6 +494,11 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18, NESTED lookback): trailing + # ``lookback_days`` valid days, each bar classified against a + # strictly-prior ``VOLUME_PRV_BASELINE_DAYS``-day same-slot baseline -> + # transitive depth = lookback_days + baseline_days trailing trading days. + lookback_depth=self._lookback_days + VOLUME_PRV_BASELINE_DAYS, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/valley_price_quantile.py b/factors/compute/minute/valley_price_quantile.py index d344bdd..0505a9e 100644 --- a/factors/compute/minute/valley_price_quantile.py +++ b/factors/compute/minute/valley_price_quantile.py @@ -696,6 +696,13 @@ def spec(self) -> FactorSpec: overnight_boundary="crossed_disclosed", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18, NESTED lookback): trailing + # ``lookback_days`` valid days, each bar classified against a + # strictly-prior ``VOLUME_PRV_BASELINE_DAYS``-day same-slot baseline + # (lookback_days + baseline_days); the daily reversal neutralized + # against spans only ``VALLEY_QUANTILE_REVERSAL_DAYS`` (< that sum), so + # the minute chain dominates -> depth = lookback_days + baseline_days. + lookback_depth=self._lookback_days + VOLUME_PRV_BASELINE_DAYS, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/valley_relative_vwap.py b/factors/compute/minute/valley_relative_vwap.py index ab5a938..542fe60 100644 --- a/factors/compute/minute/valley_relative_vwap.py +++ b/factors/compute/minute/valley_relative_vwap.py @@ -407,6 +407,11 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18, NESTED lookback): trailing + # ``lookback_days`` valid days, each bar classified against a + # strictly-prior ``VOLUME_PRV_BASELINE_DAYS``-day same-slot baseline -> + # transitive depth = lookback_days + baseline_days trailing trading days. + lookback_depth=self._lookback_days + VOLUME_PRV_BASELINE_DAYS, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/valley_ridge_vwap_ratio.py b/factors/compute/minute/valley_ridge_vwap_ratio.py index f50681b..f4a8954 100644 --- a/factors/compute/minute/valley_ridge_vwap_ratio.py +++ b/factors/compute/minute/valley_ridge_vwap_ratio.py @@ -490,6 +490,11 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18, NESTED lookback): trailing + # ``lookback_days`` valid days, each bar classified against a + # strictly-prior ``VOLUME_PRV_BASELINE_DAYS``-day same-slot baseline -> + # transitive depth = lookback_days + baseline_days trailing trading days. + lookback_depth=self._lookback_days + VOLUME_PRV_BASELINE_DAYS, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/minute/volume_peak_count.py b/factors/compute/minute/volume_peak_count.py index 07da815..a4c59d0 100644 --- a/factors/compute/minute/volume_peak_count.py +++ b/factors/compute/minute/volume_peak_count.py @@ -293,6 +293,12 @@ def spec(self) -> FactorSpec: overnight_boundary="none", family="microstructure", min_history_bars=0, + # D4 pre-registration (§六.18, NESTED lookback): a value at d pools the + # trailing ``lookback_days`` valid days, and each bar is classified + # against its strictly-prior ``VOLUME_PRV_BASELINE_DAYS``-day same-slot + # baseline, so the earliest input reaches lookback_days + baseline_days + # trailing trading days -> that sum is the transitive depth. + lookback_depth=self._lookback_days + VOLUME_PRV_BASELINE_DAYS, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/factors/compute/momentum.py b/factors/compute/momentum.py index 208f603..2aa8861 100644 --- a/factors/compute/momentum.py +++ b/factors/compute/momentum.py @@ -96,6 +96,10 @@ def spec(self) -> FactorSpec: # value at t needs bars t-window..t -> the leading ``window`` rows # of every symbol are NaN by construction. min_history_bars=self._window, + # D4 pre-registration: close[t]/close[t-window]-1 reads close over the + # inclusive span t-window..t -> transitive depth = window + 1 trailing + # trading days (the denominator adds one day beyond the headline; §六.18). + lookback_depth=self._window + 1, ) def compute(self, panel: pd.DataFrame) -> pd.Series: diff --git a/tests/test_factor_lookback_depth.py b/tests/test_factor_lookback_depth.py new file mode 100644 index 0000000..4283591 --- /dev/null +++ b/tests/test_factor_lookback_depth.py @@ -0,0 +1,124 @@ +"""D4 Commit 1: the 14 closing factors' pre-registered ``lookback_depth``. + +The D3 tail-recompute engine HARD-REQUIRES ``spec.lookback_depth`` (a readable +error, never a silent 0/headline fallback) to size its incremental overlap +window. This test pins the pre-registered transitive lookback depth of every +closing-set factor to a specific value derived from its CODE CONSTANTS (design +§六.18: headline + nested baseline, NOT the headline window). + +Pinning is the point: once declared, the value is a pre-registration — changing +it is a definition-adjacent change that must be disclosed separately. A wrong +declaration under-samples the overlap and produces FALSE mismatches every night +(the D3 engine already has teeth for that); this test is the up-front guard. + +The parameterization also proves the declaration is DERIVED from the constructor +param, not a frozen literal: a non-default window/lookback yields the matching +depth (so a re-parameterized factor cannot silently keep the default depth). +""" + +from __future__ import annotations + +import pytest + +from factors import registry as factor_registry +from factors.compute.candidates import ValueFactor, VolatilityFactor +from factors.compute.financial import FinancialFactor +from factors.compute.minute.amp_marginal_anomaly_vol import AmpMarginalAnomalyVolFactor +from factors.compute.minute.intraday_amp_cut import IntradayAmpCutFactor +from factors.compute.minute.jump_amount_corr import JumpAmountCorrFactor +from factors.compute.minute.minute_ideal_amplitude import MinuteIdealAmplitudeFactor +from factors.compute.minute.peak_interval_kurtosis import PeakIntervalKurtosisFactor +from factors.compute.minute.peak_ridge_amount_ratio import PeakRidgeAmountRatioFactor +from factors.compute.minute.primitives import VOLUME_PRV_BASELINE_DAYS +from factors.compute.minute.ridge_minute_return import RidgeMinuteReturnFactor +from factors.compute.minute.valley_price_quantile import ValleyPriceQuantileFactor +from factors.compute.minute.valley_relative_vwap import ValleyRelativeVwapFactor +from factors.compute.minute.valley_ridge_vwap_ratio import ValleyRidgeVwapRatioFactor +from factors.compute.minute.volume_peak_count import VolumePeakCountFactor +from factors.compute.momentum import MomentumFactor + +# The 14 closing factors: (factor instance, expected lookback_depth). +# 3 book factors + 11 minute-derived factors. Baseline (nested-lookback) minute +# factors = lookback_days (20) + VOLUME_PRV_BASELINE_DAYS (20) = 40. +_BASELINE = VOLUME_PRV_BASELINE_DAYS # 20 +_CLOSING_FACTORS = [ + # -- book factors ----------------------------------------------------- + (ValueFactor("value_ep"), 1), + (ValueFactor("value_bp"), 1), + (VolatilityFactor(window=20), 21), # window + 1 (pct_change adds a day) + # -- minute factors, no nested baseline ------------------------------- + (JumpAmountCorrFactor(), 20), + (MinuteIdealAmplitudeFactor(), 10), + (AmpMarginalAnomalyVolFactor(), 20), + (IntradayAmpCutFactor(), 10), + # -- minute factors, nested baseline (lookback 20 + baseline 20) ------ + (VolumePeakCountFactor(), 20 + _BASELINE), + (PeakIntervalKurtosisFactor(), 20 + _BASELINE), + (ValleyRelativeVwapFactor(), 20 + _BASELINE), + (ValleyRidgeVwapRatioFactor(), 20 + _BASELINE), + (RidgeMinuteReturnFactor(), 20 + _BASELINE), + (ValleyPriceQuantileFactor(), 20 + _BASELINE), + (PeakRidgeAmountRatioFactor(), 20 + _BASELINE), +] + + +@pytest.mark.parametrize( + "factor, expected", + _CLOSING_FACTORS, + ids=[f.name for f, _ in _CLOSING_FACTORS], +) +def test_closing_factor_lookback_depth(factor, expected): + """Each closing factor declares its exact pre-registered transitive depth.""" + depth = factor.spec.lookback_depth + assert depth is not None, ( + f"{factor.name}: lookback_depth is None — the store engine would refuse " + f"to size its overlap window (D3 hard-require)." + ) + assert depth == expected, f"{factor.name}: lookback_depth {depth} != {expected}" + # It must be a positive int (contract v1.1 validates this at construction). + assert isinstance(depth, int) and not isinstance(depth, bool) and depth >= 1 + + +def test_closing_factors_count_is_fourteen(): + """The pinned table is exactly the 14-factor closing set (no drift).""" + assert len(_CLOSING_FACTORS) == 14 + + +@pytest.mark.parametrize( + "factor, expected", + [ + # window-parameterized: depth tracks the constructor param, not a literal. + (VolatilityFactor(window=10), 11), + (MomentumFactor(window=30), 31), + (JumpAmountCorrFactor(lookback_days=30), 30), + (MinuteIdealAmplitudeFactor(lookback_days=15), 15), + (VolumePeakCountFactor(lookback_days=15), 15 + _BASELINE), + (ValleyRidgeVwapRatioFactor(lookback_days=25), 25 + _BASELINE), + ], +) +def test_lookback_depth_tracks_the_constructor_param(factor, expected): + """A non-default window/lookback yields the matching depth (derived, not frozen).""" + assert factor.spec.lookback_depth == expected + + +def test_non_closing_daily_factors_also_declare_depth(): + """The remaining registered factors declare a depth too (no latent store break).""" + from factors.compute.candidates import ( + LiquidityFactor, + OvernightMomentumFactor, + ReversalFactor, + ) + + assert MomentumFactor(window=20).spec.lookback_depth == 21 + assert ReversalFactor(window=20).spec.lookback_depth == 21 # inherits momentum + assert LiquidityFactor(window=20).spec.lookback_depth == 20 + assert OvernightMomentumFactor(window=20).spec.lookback_depth == 21 + assert FinancialFactor("roe").spec.lookback_depth == 1 + + +def test_registry_built_factor_carries_the_same_depth(): + """Building through the registry (the D4 service path) yields the same depth.""" + built = factor_registry.build("jump_amount_corr_20") + assert built.spec.lookback_depth == 20 + built_vol = factor_registry.build("volatility_20") + assert built_vol.spec.lookback_depth == 21 From ca1e89231086f015c0427cdf2eab2205ec35af36 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 14:56:54 -0700 Subject: [PATCH 2/9] =?UTF-8?q?feat(factors):=20decision-view=20materializ?= =?UTF-8?q?er=20=E2=80=94=20the=20one=20vectorized=20engine=20(D4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single path that turns injected data-layer inputs into RAW factor values for a view over a date range (design §3.5). factors.service read-through will route every store miss here; nothing else computes factor values. * factors/view_lag.py — PURE availability-lag transforms (R18): daily prev-day shift with the field-level ``open`` same-day exception (the same-day set is DERIVED from the policy table, not a literal), the minute ``available_time <= d 14:50`` cutoff, and the ex-date BOOLEAN (``af(d) != af(d-1)`` — the adj_factor numeric never enters the frame, a type boundary per §1.3 note 4). * factors/materialize.py — the engine: reads ``requires`` to pick daily vs minute input, pulls from INJECTED providers (layering: factors never touches a feed/token/qt), applies the (source, view) lag, trims to EXACTLY the factor's ``lookback_depth`` trailing trading days before the emit window (the P8 single==batch correctness floor), computes, and emits. ``masked`` factors have ex-date rows NaN'd (the 14 closing factors are non-masked -> no-op). ``build_horizon_config`` wires CacheHorizonConfig from the LIVE cache config with NO default fallback (R5). ``make_recompute_fn`` yields a D3-compatible RecomputeFn. * factors/compute/minute/binding.py — binds each of 10 minute factors to its free ``compute_*`` (parameterized by the instance's lookback + name, all other definition constants at their single-source module defaults); valley_price_quantile is deferred (needs the daily panel too) with a readable error, never a silent mis-compute. Tests (network-free): view-lag transforms (open-only same-day, prev-day shift, cutoff, ex-date bool); R18 lag coverage (close[d] not visible / open[d] is — reverse teeth / daily_basic prev-day); minute cutoff no-leak; binding == direct compute_* (float-reorder tolerance, §五); ex-date masking fixture; recompute-fn parity; live-config wiring refuses a missing attr. --- factors/compute/minute/binding.py | 142 +++++++++++++ factors/materialize.py | 323 ++++++++++++++++++++++++++++++ factors/view_lag.py | 146 ++++++++++++++ tests/test_factor_materialize.py | 316 +++++++++++++++++++++++++++++ tests/test_factor_view_lag.py | 129 ++++++++++++ 5 files changed, 1056 insertions(+) create mode 100644 factors/compute/minute/binding.py create mode 100644 factors/materialize.py create mode 100644 factors/view_lag.py create mode 100644 tests/test_factor_materialize.py create mode 100644 tests/test_factor_view_lag.py diff --git a/factors/compute/minute/binding.py b/factors/compute/minute/binding.py new file mode 100644 index 0000000..c4b5575 --- /dev/null +++ b/factors/compute/minute/binding.py @@ -0,0 +1,142 @@ +"""Minute-factor raw-compute binding for the D4 materializer (design §3.5). + +The materializer is factor-agnostic: it loads cutoff-filtered 1min bars and asks +"compute this minute factor's raw daily series from these bars". Each minute +Factor's ``compute(panel)`` only SURFACES a pre-aggregated column (D2), so this +module binds every minute factor to the free ``compute_*`` function that actually +does the aggregation, parameterized by the factor INSTANCE (its ``lookback_days`` +and ``name``). All other definition constants stay at their module defaults — +the SAME single-source constants the eval runners / ``qt.panel_freeze`` recipes +pass (they pass them explicitly but equal to these defaults), so a materialized +minute factor matches the frozen baseline's math. + +This lives in ``factors/compute/minute`` (the factor layer) rather than duplicated +in ``qt`` because it is FACTOR knowledge; ``qt.panel_freeze`` keeps its own +recipe copy (a frozen D1 tool) and D6 may later fold it onto this binding. + +Deferred (readable error, D5): ``valley_price_quantile`` also needs the DAILY +close panel (its reversal neutralization), so it does not fit the pure +``(factor, bars) -> series`` shape and is NOT bound here — the materializer +raises for it rather than silently mis-computing. + +Layering: factor layer only (never qt / feeds). +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pandas as pd + +from factors.base import Factor +from factors.compute.minute.amp_marginal_anomaly_vol import ( + AmpMarginalAnomalyVolFactor, + compute_amp_marginal_anomaly_vol, +) +from factors.compute.minute.intraday_amp_cut import ( + IntradayAmpCutFactor, + compute_intraday_amp_cut, +) +from factors.compute.minute.jump_amount_corr import ( + JumpAmountCorrFactor, + compute_jump_amount_corr, +) +from factors.compute.minute.minute_ideal_amplitude import ( + MinuteIdealAmplitudeFactor, + compute_minute_ideal_amplitude, +) +from factors.compute.minute.peak_interval_kurtosis import ( + PeakIntervalKurtosisFactor, + compute_peak_interval_kurtosis, +) +from factors.compute.minute.peak_ridge_amount_ratio import ( + PeakRidgeAmountRatioFactor, + compute_peak_ridge_amount_ratio, +) +from factors.compute.minute.ridge_minute_return import ( + RidgeMinuteReturnFactor, + compute_ridge_minute_return, +) +from factors.compute.minute.valley_price_quantile import ValleyPriceQuantileFactor +from factors.compute.minute.valley_relative_vwap import ( + ValleyRelativeVwapFactor, + compute_valley_relative_vwap, +) +from factors.compute.minute.valley_ridge_vwap_ratio import ( + ValleyRidgeVwapRatioFactor, + compute_valley_ridge_vwap_ratio, +) +from factors.compute.minute.volume_peak_count import ( + VolumePeakCountFactor, + compute_volume_peak_count, +) + +#: bars -> daily raw factor Series, parameterized by the factor instance. The +#: cutoff is already applied by the materializer (bars are pre-filtered to +#: ``available_time <= decision``), so the compute functions run with their +#: DEFAULT decision_time (14:50) as a redundant-but-consistent internal cutoff. +BindingFn = Callable[[Factor, pd.DataFrame], pd.Series] + + +def _bind(compute_fn) -> BindingFn: + """A ``(factor, bars)`` binding that calls ``compute_fn`` with the instance's + lookback + name and every other definition constant at its module default.""" + + def _call(factor: Factor, bars: pd.DataFrame) -> pd.Series: + return compute_fn( + bars, lookback_days=factor.lookback_days, name=factor.name # type: ignore[attr-defined] + ) + + return _call + + +#: factor class -> its bars-based raw compute. Bound: the 10 minute factors whose +#: raw compute needs ONLY the 1min bars. NOT bound: valley_price_quantile (needs +#: the daily panel too — deferred, readable error below). +_MINUTE_BINDINGS: dict[type[Factor], BindingFn] = { + JumpAmountCorrFactor: _bind(compute_jump_amount_corr), + MinuteIdealAmplitudeFactor: _bind(compute_minute_ideal_amplitude), + AmpMarginalAnomalyVolFactor: _bind(compute_amp_marginal_anomaly_vol), + VolumePeakCountFactor: _bind(compute_volume_peak_count), + IntradayAmpCutFactor: _bind(compute_intraday_amp_cut), + PeakIntervalKurtosisFactor: _bind(compute_peak_interval_kurtosis), + ValleyRelativeVwapFactor: _bind(compute_valley_relative_vwap), + ValleyRidgeVwapRatioFactor: _bind(compute_valley_ridge_vwap_ratio), + RidgeMinuteReturnFactor: _bind(compute_ridge_minute_return), + PeakRidgeAmountRatioFactor: _bind(compute_peak_ridge_amount_ratio), +} + +#: Minute factors deliberately NOT bound (need extra inputs), with the reason. +_DEFERRED: dict[type[Factor], str] = { + ValleyPriceQuantileFactor: ( + "valley_price_quantile also consumes the DAILY close panel (its reversal " + "neutralization), so it does not fit the pure (factor, bars) binding; the " + "materializer defers it to D5 (a readable error, never a silent mis-compute)." + ), +} + + +def is_minute_bound(factor: Factor) -> bool: + """True iff ``factor`` has a bars-only raw-compute binding here.""" + return type(factor) in _MINUTE_BINDINGS + + +def minute_raw_from_bars(factor: Factor, bars: pd.DataFrame) -> pd.Series: + """Compute ``factor``'s raw daily Series from (cutoff-filtered) 1min ``bars``. + + Readable error for a minute factor that is deferred (needs extra inputs) or a + non-minute-bound factor — never a silent wrong result. + """ + binding = _MINUTE_BINDINGS.get(type(factor)) + if binding is not None: + return binding(factor, bars) + deferred = _DEFERRED.get(type(factor)) + if deferred is not None: + raise NotImplementedError(f"{factor.name}: {deferred}") + raise KeyError( + f"{factor.name} ({type(factor).__name__}) has no minute-bars binding; it " + f"is not a minute-derived factor bound in factors.compute.minute.binding." + ) + + +__all__ = ["BindingFn", "is_minute_bound", "minute_raw_from_bars"] diff --git a/factors/materialize.py b/factors/materialize.py new file mode 100644 index 0000000..21319ce --- /dev/null +++ b/factors/materialize.py @@ -0,0 +1,323 @@ +"""The ONE vectorized factor-value engine (factor-refactor D4, design §3.5). + +The materializer is the single path that turns injected data-layer inputs into +RAW factor values for a view over a date range. ``factors.service`` read-through +routes every store miss here; nothing else computes factor values. It is +factor-agnostic and view-agnostic-compute-preserving (design §1.4 mechanism 3): + +* it reads the factor's ``requires`` to decide DAILY vs MINUTE input; +* it pulls inputs from INJECTED providers (layering red line #3/#10: ``factors`` + never touches a feed, a token, or qt — the consumer injects the data access); +* it applies the (source, view) availability lag from ``factors.view_lag`` (R18: + daily prev-day shift with the field-level ``open`` exception; minute + ``available_time <= d 14:50`` cutoff; PIT as-of is a no-op here); +* it trims the input to EXACTLY the factor's ``lookback_depth`` trailing trading + days before the emit window, then computes and emits — so a single-date fill + and a batch fill produce bit-identical values (design §3.5 P8); +* ``masked`` factors have their ex-date rows NaN'd via the BOOLEAN ex-date mask + (§1.3 note 4; the ``adj_factor`` numeric never enters the frame). The closing + 14 factors are all non-masked, so this is a no-op for them (fixture-tested). + +The daily factor path calls ``factor.compute`` on the lagged panel; the minute +path calls the ``factors.compute.minute.binding`` for the factor and the +cutoff-filtered bars. The forward-return boundary is elsewhere: a factor value +never sees a future return (invariant #1) — the materializer only reads history. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Protocol + +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN, OvernightBoundary, View +from data.clean.intraday_schema import DEFAULT_DECISION_TIME +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL +from factors.base import Factor +from factors.compute.minute.binding import is_minute_bound, minute_raw_from_bars +from factors.store.incremental import CacheHorizonConfig +from factors.view_lag import ( + daily_decision_lag, + ex_date_mask, + minute_decision_cutoff, +) + +#: Extra calendar days loaded before the emit window so the exact trailing-trading +#: -day trim always has enough history (covers weekends + holiday clusters + the +#: daily shift's one extra day). Generous by design: the CORRECTNESS floor is the +#: trim to ``lookback_depth`` trading days, not this buffer (design §3.5 P8). +def _load_buffer_calendar_days(warmup: int) -> int: + return int(warmup) * 2 + 25 + + +# --------------------------------------------------------------------------- # +# Injected data providers (the consumer wires these to feed/cache; tests fake) +# --------------------------------------------------------------------------- # +class DailyPanelProvider(Protocol): + """Loads the CLOSE-view daily factor-input panel (front-adjusted + enriched).""" + + def daily_panel( + self, symbols: list[str], start: pd.Timestamp, end: pd.Timestamp + ) -> pd.DataFrame: + """MultiIndex(date, symbol) panel, values dated at their natural close date.""" + + +class MinuteBarProvider(Protocol): + """Loads normalized 1min bars (cache-only) for the symbols over the window.""" + + def minute_bars( + self, symbols: list[str], start: pd.Timestamp, end: pd.Timestamp + ) -> pd.DataFrame: + """MultiIndex(time, symbol) bars (:mod:`data.clean.intraday_schema`).""" + + +class AdjFactorProvider(Protocol): + """Loads the RAW adj_factor series (ONLY consumed as an ex-date BOOLEAN).""" + + def adj_factor( + self, symbols: list[str], start: pd.Timestamp, end: pd.Timestamp + ) -> pd.Series: + """MultiIndex(date, symbol) raw adj_factor (never enters the factor frame).""" + + +@dataclass(frozen=True) +class MaterializeSources: + """The injected data access the materializer needs (consumer-wired).""" + + daily: DailyPanelProvider | None = None + minute: MinuteBarProvider | None = None + adj_factor: AdjFactorProvider | None = None + + +# --------------------------------------------------------------------------- # +# Live-config wiring (R5 / D3-note: NEVER the CacheHorizonConfig default) +# --------------------------------------------------------------------------- # +def build_horizon_config(cache_cfg: object) -> CacheHorizonConfig: + """Build the incremental-overlap horizon config from the LIVE cache config. + + Reads ``refresh_recent_days`` and ``fina_tail_days`` off ``cache_cfg`` (duck + typed, so ``qt.config`` DataCacheCfg works without a factors->qt import). Both + are REQUIRED: the ``CacheHorizonConfig`` 400 default is a test convenience and + a config whose fina tail differs from 400 would silently mis-size the fina + overlap window (R5) — so a missing attribute is a readable error, never the + default. + """ + missing = [a for a in ("refresh_recent_days", "fina_tail_days") if not hasattr(cache_cfg, a)] + if missing: + raise ValueError( + f"build_horizon_config needs the live cache config's {missing} " + f"attribute(s); refusing to fall back to the CacheHorizonConfig default " + f"(R5: a differing fina tail would silently mis-size the overlap)." + ) + return CacheHorizonConfig( + refresh_recent_days=int(cache_cfg.refresh_recent_days), + fina_tail_days=int(cache_cfg.fina_tail_days), + ) + + +# --------------------------------------------------------------------------- # +# Factor kind +# --------------------------------------------------------------------------- # +def is_minute_factor(factor: Factor) -> bool: + """True iff any of the factor's declared inputs comes from the 1min endpoint.""" + return any(r.source == STK_MINS_1MIN for r in (factor.spec.requires or ())) + + +# --------------------------------------------------------------------------- # +# Trailing-trading-day trim (the P8 correctness floor) +# --------------------------------------------------------------------------- # +def _warmup_start(dates: pd.DatetimeIndex, emit_start: pd.Timestamp, warmup: int): + """The date ``warmup`` trading days before ``emit_start`` (or the earliest). + + ``dates`` is the sorted unique set of trading dates present in the loaded + input. Returns the cutoff date to keep input from; if fewer than ``warmup`` + trading days precede ``emit_start`` the earliest available date is used + (near the data start the emit rows are honestly under-warmed -> NaN, and + single/batch stay consistent because both trim to the same earliest date). + """ + order = dates.sort_values() + pos = int(order.searchsorted(emit_start, side="left")) # index of emit_start (or ins.) + keep_idx = max(0, pos - int(warmup)) + return order[keep_idx] + + +# --------------------------------------------------------------------------- # +# The engine +# --------------------------------------------------------------------------- # +def materialize_range( + factor: Factor, + *, + view: object, + symbols: list[str], + emit_start: pd.Timestamp, + emit_end: pd.Timestamp, + sources: MaterializeSources, + decision_cutoff: str = DEFAULT_DECISION_TIME, + warmup: int | None = None, +) -> pd.Series: + """Compute ``factor``'s RAW values for ``view`` over ``[emit_start, emit_end]``. + + Loads a generous window (so the exact trailing trim always has history), + applies the view lag, trims to ``warmup`` (= ``spec.lookback_depth``) trailing + trading days before ``emit_start``, computes, and returns the emit-window + rows. Single-date and batch calls give bit-identical values (design §3.5 P8). + """ + resolved_view = View(view) + emit_start = pd.Timestamp(emit_start).normalize() + emit_end = pd.Timestamp(emit_end).normalize() + w = int(factor.spec.lookback_depth) if warmup is None else int(warmup) + if w < 1: + raise ValueError(f"{factor.name}: warmup must be >= 1; got {w}.") + load_start = emit_start - pd.Timedelta(days=_load_buffer_calendar_days(w)) + + if is_minute_factor(factor): + raw = _materialize_minute( + factor, resolved_view, symbols, load_start, emit_start, emit_end, w, + sources, decision_cutoff, + ) + else: + raw = _materialize_daily( + factor, resolved_view, symbols, load_start, emit_start, emit_end, w, sources, + ) + + raw = _apply_ex_date_mask( + factor, raw, resolved_view, symbols, load_start, emit_end, sources + ) + return _restrict_emit(raw, emit_start, emit_end, factor.name) + + +def _materialize_daily( + factor, view, symbols, load_start, emit_start, emit_end, warmup, sources, +) -> pd.Series: + if sources.daily is None: + raise ValueError( + f"{factor.name} is a daily factor but no DailyPanelProvider was injected." + ) + panel = sources.daily.daily_panel(list(symbols), load_start, emit_end) + if panel.empty: + return _empty_series(factor.name) + if view is View.DECISION: + panel = daily_decision_lag(panel) # prev-day shift, open same-day (R18) + trimmed = _trim_daily(panel, emit_start, warmup) + return factor.compute(trimmed).rename(factor.name) + + +def _materialize_minute( + factor, view, symbols, load_start, emit_start, emit_end, warmup, sources, decision_cutoff, +) -> pd.Series: + if sources.minute is None: + raise ValueError( + f"{factor.name} is a minute factor but no MinuteBarProvider was injected." + ) + if not is_minute_bound(factor): + # Readable error (e.g. valley_price_quantile needs the daily panel too). + return minute_raw_from_bars(factor, sources.minute.minute_bars([], load_start, emit_end)) + # Load one extra calendar day past emit_end so bar cutoffs on emit_end resolve. + bars = sources.minute.minute_bars( + list(symbols), load_start, emit_end + pd.Timedelta(days=1) + ) + if bars.empty: + return _empty_series(factor.name) + if view is View.DECISION: + bars = minute_decision_cutoff(bars, decision_time=decision_cutoff) + bars = _trim_minute(bars, emit_start, warmup) + return minute_raw_from_bars(factor, bars) + + +def _trim_daily(panel: pd.DataFrame, emit_start: pd.Timestamp, warmup: int) -> pd.DataFrame: + dates = pd.DatetimeIndex(pd.unique(panel.index.get_level_values(DATE_LEVEL))) + keep_from = _warmup_start(dates, emit_start, warmup) + return panel[panel.index.get_level_values(DATE_LEVEL) >= keep_from] + + +def _trim_minute(bars: pd.DataFrame, emit_start: pd.Timestamp, warmup: int) -> pd.DataFrame: + bar_dates = pd.DatetimeIndex(bars.index.get_level_values("time")).normalize() + trading = pd.DatetimeIndex(pd.unique(bar_dates)) + keep_from = _warmup_start(trading, emit_start, warmup) + return bars[bar_dates >= keep_from] + + +def _apply_ex_date_mask(factor, raw, view, symbols, load_start, emit_end, sources): + """NaN out ex-date rows for a ``masked`` factor (§1.3 note 4). + + Non-masked factors (all 14 closing) skip this entirely — the adj_factor + provider is not even consulted, so no ex-date machinery runs for them. + """ + if factor.spec.overnight_boundary is not OvernightBoundary.MASKED: + return raw + if view is not View.DECISION or raw.empty: + return raw + if sources.adj_factor is None: + raise ValueError( + f"{factor.name} declares overnight_boundary='masked' but no " + f"AdjFactorProvider was injected to resolve ex-dates." + ) + af = sources.adj_factor.adj_factor(list(symbols), load_start, emit_end) + mask = ex_date_mask(af).reindex(raw.index, fill_value=False) + out = raw.copy() + out[mask.to_numpy()] = float("nan") + return out + + +def _restrict_emit(raw, emit_start, emit_end, name) -> pd.Series: + if raw.empty: + return _empty_series(name) + dates = raw.index.get_level_values(DATE_LEVEL) + within = (dates >= emit_start) & (dates <= emit_end) + return raw[within].sort_index(kind="mergesort").rename(name) + + +def _empty_series(name: str) -> pd.Series: + index = pd.MultiIndex.from_arrays( + [pd.DatetimeIndex([]), pd.Index([], dtype=object)], + names=[DATE_LEVEL, SYMBOL_LEVEL], + ) + return pd.Series([], index=index, dtype=float, name=name) + + +def make_recompute_fn( + factor: Factor, + *, + view: object, + symbols: list[str], + sources: MaterializeSources, + data_start: pd.Timestamp, + decision_cutoff: str = DEFAULT_DECISION_TIME, +) -> Callable[[pd.Timestamp | None, pd.Timestamp, int], pd.Series]: + """A ``RecomputeFn`` for the D3 tail-recompute engine (design §3.3). + + ``recompute(emit_start, end, warmup)`` -> the factor's raw Series over + ``[emit_start, end]`` having loaded ``warmup`` trailing trading days of input + (``emit_start is None`` -> from ``data_start``). The same trailing-trim floor + as ``materialize_range`` keeps the incremental overlap bit-identical to a full + column (the D3 batch=incremental invariant). + """ + + def _recompute(emit_start, end, warmup) -> pd.Series: + start = data_start if emit_start is None else emit_start + return materialize_range( + factor, + view=view, + symbols=list(symbols), + emit_start=start, + emit_end=end, + sources=sources, + decision_cutoff=decision_cutoff, + warmup=warmup, + ) + + return _recompute + + +__all__ = [ + "AdjFactorProvider", + "DailyPanelProvider", + "MaterializeSources", + "MinuteBarProvider", + "build_horizon_config", + "is_minute_factor", + "make_recompute_fn", + "materialize_range", +] diff --git a/factors/view_lag.py b/factors/view_lag.py new file mode 100644 index 0000000..9e8c052 --- /dev/null +++ b/factors/view_lag.py @@ -0,0 +1,146 @@ +"""Availability-lag transforms for the two views (factor-refactor D4, §1.3/R18). + +PURE functions (no data access, no feed, no qt). Given an already-prepared +close-view daily panel or normalized 1min bars, produce the view-lagged input a +factor's ``compute`` consumes — so the SAME view-agnostic factor code (design +§1.4 mechanism 3) serves both views. The materializer composes these. + +The decision(d) view is the operational information set at the 14:50 decision: + +* daily fields are visible only at ``<= d-1`` EXCEPT ``open`` (fixed at 09:30, + known by 14:50 — the ONE field-level same-day exception, R7). Concretely the + prev-day columns are shifted forward one trading day PER SYMBOL, so the row at + date d carries the value that was final at d-1; ``open`` is left same-day. + This is exactly ``<= d-1`` for every prev-day field: a factor reading the + shifted column at row d therefore uses only d-1 information for it (and a + factor's own trailing lag composes on top consistently). The same-day set is + DERIVED from the policy table (:func:`decision_same_day_fields`), never a + hand-kept literal. +* minute bars: only bars with ``available_time <= d 14:50`` survive (the I3 + cutoff), filtered on per-bar timestamps BEFORE any daily grouping. + +The ex-date boundary (§1.3 note 4) is a SEPARATE, type-boundary-clean module: +:func:`ex_date_mask` turns a raw ``adj_factor`` series into a BOOLEAN mask +(``af(d) != af(d-1)`` per symbol) and nothing else — the ``adj_factor`` numeric +never enters the decision-view frame, so "only judge the fact, never use the +value" is enforced by the return type, not by docstring discipline. + +Leaf discipline: numpy / pandas + the pure ``data.availability_policy`` leaf and +``data.clean.intraday_schema`` constants. Never import qt, feeds, or caches. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import ( + AVAILABILITY_POLICY, + DecisionVisibility, + RevisionHorizonSource, +) +from data.clean.intraday_schema import DEFAULT_DECISION_TIME +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL + + +def decision_same_day_fields() -> frozenset[str]: + """Factor-input fields legal SAME-DAY in the decision view (policy-derived). + + Reads the §1.3 policy table: a field is decision-view same-day iff its rule + is ``DecisionVisibility.SAME_DAY`` AND it is a factor-store input endpoint + (``horizon_source != NOT_FACTOR_INPUT`` — stk_limit / suspend_d / namechange + are same-day too but never feed a factor, so they are excluded). Today this + is exactly ``{"open"}`` (market_daily's field-level exception), but it is + derived so a policy-table change flows through automatically. + """ + out: set[str] = set() + for (_source, field), rule in AVAILABILITY_POLICY.items(): + if field is None: + continue + if ( + rule.decision_visibility is DecisionVisibility.SAME_DAY + and rule.horizon_source is not RevisionHorizonSource.NOT_FACTOR_INPUT + ): + out.add(field) + return frozenset(out) + + +def daily_decision_lag( + panel: pd.DataFrame, *, same_day_fields: frozenset[str] | None = None +) -> pd.DataFrame: + """Shift the close-view daily panel into the decision(d) view (R18 daily lag). + + Every column is shifted forward one trading day PER SYMBOL (row d gets row + d-1's value = ``<= d-1`` visibility) EXCEPT the same-day fields (``open``), + which are left untouched (09:30 fixed, legal at 14:50). The first row of each + symbol has no d-1 for the shifted columns, so it becomes NaN (a decision-view + value cannot exist on a symbol's very first date). Never mutates ``panel``. + """ + if not isinstance(panel.index, pd.MultiIndex) or list(panel.index.names) != [ + DATE_LEVEL, + SYMBOL_LEVEL, + ]: + raise ValueError( + "daily_decision_lag expects a MultiIndex(date, symbol) panel; got " + f"index names {list(panel.index.names)}." + ) + same_day = decision_same_day_fields() if same_day_fields is None else same_day_fields + ordered = panel.sort_index(kind="mergesort") + shift_cols = [c for c in ordered.columns if c not in same_day] + if not shift_cols: + return ordered.copy() + out = ordered.copy() + shifted = ordered[shift_cols].groupby(level=SYMBOL_LEVEL, sort=False).shift(1) + out[shift_cols] = shifted + return out + + +def minute_decision_cutoff( + bars: pd.DataFrame, *, decision_time: str = DEFAULT_DECISION_TIME +) -> pd.DataFrame: + """Keep only bars visible at the decision cutoff (``available_time <= d cutoff``). + + The cutoff for each bar is its own trade date + ``decision_time`` (default + 14:50). The filter runs on per-bar timestamps (never date-normalized first), + so a post-cutoff bar can never leak into a decision. Never mutates ``bars``. + """ + if bars.empty: + return bars.copy() + bar_end = bars.index.get_level_values("time") + trade_date = pd.DatetimeIndex(bar_end).normalize() + cutoff = trade_date + pd.Timedelta(decision_time) + visible = bars["available_time"].to_numpy() <= cutoff.to_numpy() + return bars.loc[visible].copy() + + +def ex_date_mask(adj_factor: pd.Series) -> pd.Series: + """BOOLEAN ex-date mask ``af(d) != af(d-1)`` per symbol (§1.3 note 4). + + The ONLY thing this module exposes to the materializer: the fact "day d is an + ex-date for this symbol", never the ``adj_factor`` numeric. The first row of + each symbol is False (no d-1 to compare). ``af`` is compared with NaN-aware + inequality so a missing raw factor is not spuriously flagged. + """ + if not isinstance(adj_factor.index, pd.MultiIndex) or list( + adj_factor.index.names + ) != [DATE_LEVEL, SYMBOL_LEVEL]: + raise ValueError( + "ex_date_mask expects a MultiIndex(date, symbol) adj_factor series; got " + f"index names {list(adj_factor.index.names)}." + ) + ordered = adj_factor.sort_index(kind="mergesort") + prev = ordered.groupby(level=SYMBOL_LEVEL, sort=False).shift(1) + cur_v = ordered.to_numpy(dtype=float) + prev_v = prev.to_numpy(dtype=float) + both_nan = np.isnan(cur_v) & np.isnan(prev_v) + # differ, and not the (first-row) case where prev is NaN, and not both-NaN. + changed = (~(cur_v == prev_v)) & ~np.isnan(prev_v) & ~both_nan + return pd.Series(changed, index=ordered.index, name="ex_date") + + +__all__ = [ + "daily_decision_lag", + "decision_same_day_fields", + "ex_date_mask", + "minute_decision_cutoff", +] diff --git a/tests/test_factor_materialize.py b/tests/test_factor_materialize.py new file mode 100644 index 0000000..f8121c0 --- /dev/null +++ b/tests/test_factor_materialize.py @@ -0,0 +1,316 @@ +"""D4: the materializer engine (factors.materialize) — R18 lag + leakage + binding. + +Network-free. Covers, against the §九 D4 acceptance: +* R18 full lag: daily prev-day fields at ``<= d-1``, field-level ``open`` same-day, + daily_basic/enriched at ``<= d-1``, minute ``available_time <= d 14:50``; +* decision-after perturbation leaves the decision-view value bit-unchanged, and + perturbing ``open[d]`` DOES change it (the reverse teeth); +* the minute binding is correct (close-view materialize == direct compute_X); +* ex-date masking (a fixture ``masked`` factor; the 14 closing factors are all + non-masked so this path is a no-op for them); +* the live-config wiring of ``build_horizon_config`` (no default fallback, R5). +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from data.availability_policy import View +from data.clean.intraday_schema import normalize_intraday_bars +from factors.base import Factor +from factors.compute.candidates import OvernightMomentumFactor, ValueFactor, VolatilityFactor +from factors.compute.minute.jump_amount_corr import ( + JumpAmountCorrFactor, + compute_jump_amount_corr, +) +from factors.materialize import ( + MaterializeSources, + build_horizon_config, + is_minute_factor, + make_recompute_fn, + materialize_range, +) +from factors.spec import FactorSpec, PanelField + +SYMS = ["000001.SZ", "000002.SZ"] +DATES = pd.bdate_range("2021-01-04", periods=70) + + +# --------------------------------------------------------------------------- # +# Synthetic data + fake providers +# --------------------------------------------------------------------------- # +def _daily(perturb=None): + """Enriched close-view daily panel (+ a value_ep column) for the fakes. + + ``perturb`` = (date, symbol, column, new_value) mutates one cell. + """ + rng = np.random.RandomState(0) + rows = [] + for si, s in enumerate(SYMS): + px = 100.0 + si * 20 + np.cumsum(rng.normal(0, 1.0, len(DATES))) + for d, p in zip(DATES, px): + rows.append((d, s, p - 0.3, p + 0.5, p - 0.5, p, 1e5, p * 1e5, 1.0 / (10.0 + p * 0.01))) + frame = pd.DataFrame( + rows, columns=["date", "symbol", "open", "high", "low", "close", "volume", "amount", "value_ep"] + ).set_index(["date", "symbol"]).sort_index() + if perturb is not None: + d, s, col, val = perturb + frame.loc[(d, s), col] = val + return frame + + +def _minute(perturb=None): + """Two symbols, 40 days, a session spanning across 14:50 so the cutoff bites.""" + rng = np.random.RandomState(1) + rows = [] + for s in SYMS: + for d in DATES[:40]: + # bars from 14:40 to 14:59 (across the 14:50 cutoff) + base = pd.Timestamp(d) + pd.Timedelta("14:40:00") + for i in range(20): + t = base + pd.Timedelta(minutes=i) + o = 100.0 + rng.normal(0, 1) + w = 4.0 if i in (2, 6, 12) else 0.5 + rows.append((t, s, o, o + w, o - w, o + rng.normal(0, 0.2), 1e4, 1e6 + rng.normal(0, 1e5))) + frame = pd.DataFrame( + rows, columns=["time", "symbol", "open", "high", "low", "close", "volume", "amount"] + ) + bars = normalize_intraday_bars(frame, freq="1min") + if perturb is not None: + t, s, col, val = perturb + bars.loc[(pd.Timestamp(t), s), col] = val + return bars + + +class DailyProv: + def __init__(self, panel): + self._p = panel + + def daily_panel(self, symbols, start, end): + m = self._p.index.get_level_values("date") + return self._p[(m >= pd.Timestamp(start)) & (m <= pd.Timestamp(end))] + + +class MinuteProv: + def __init__(self, bars): + self._b = bars + + def minute_bars(self, symbols, start, end): + if not symbols: + return self._b.iloc[0:0] + t = self._b.index.get_level_values("time") + return self._b[(t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end))] + + +def _sources(daily=None, minute=None, adj=None): + return MaterializeSources( + daily=DailyProv(daily) if daily is not None else None, + minute=MinuteProv(minute) if minute is not None else None, + adj_factor=adj, + ) + + +# --------------------------------------------------------------------------- # +# Factor kind classification +# --------------------------------------------------------------------------- # +def test_is_minute_factor(): + assert is_minute_factor(JumpAmountCorrFactor()) + assert not is_minute_factor(VolatilityFactor(window=20)) + assert not is_minute_factor(ValueFactor("value_ep")) + + +# --------------------------------------------------------------------------- # +# Daily decision-view materialization + R18 daily lag +# --------------------------------------------------------------------------- # +def test_daily_decision_materialize_is_finite(): + src = _sources(daily=_daily()) + v = materialize_range(VolatilityFactor(window=20), view=View.DECISION, symbols=SYMS, + emit_start=DATES[40], emit_end=DATES[55], sources=src) + assert np.isfinite(v.to_numpy()).all() and len(v) == 16 * 2 + + +def test_r18_close_is_prev_day_open_is_same_day(): + """A close-reading factor's decision value at d is UNCHANGED when close[d] + is perturbed (decision uses close[d-1]); an open-reading factor's value at d + DOES change when open[d] is perturbed (open is field-level same-day, R7).""" + d = DATES[55] + base_daily = _daily() + src0 = _sources(daily=base_daily) + + # (a) close[d] perturbation -> decision-view volatility at d bit-unchanged. + vol = VolatilityFactor(window=20) + v0 = materialize_range(vol, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, sources=src0) + src_close = _sources(daily=_daily(perturb=(d, SYMS[0], "close", 999.0))) + v1 = materialize_range(vol, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, sources=src_close) + assert v0.loc[(d, SYMS[0])] == v1.loc[(d, SYMS[0])] # close[d] not visible at 14:50 + + # (b) open[d] perturbation -> overnight (reads open same-day) at d CHANGES. + ov = OvernightMomentumFactor(window=20) + o0 = materialize_range(ov, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, sources=src0) + src_open = _sources(daily=_daily(perturb=(d, SYMS[0], "open", 999.0))) + o1 = materialize_range(ov, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, sources=src_open) + assert o0.loc[(d, SYMS[0])] != o1.loc[(d, SYMS[0])] # open[d] IS visible (reverse teeth) + + # (c) close[d] perturbation -> overnight at d unchanged (it never reads close[d]). + o2 = materialize_range(ov, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, sources=src_close) + assert o0.loc[(d, SYMS[0])] == o2.loc[(d, SYMS[0])] + + +def test_r18_daily_basic_enriched_column_is_prev_day(): + """value_ep (daily_basic-derived) at d uses the d-1 published ratio (<= d-1).""" + d = DATES[55] + src0 = _sources(daily=_daily()) + vf = ValueFactor("value_ep") + v0 = materialize_range(vf, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, sources=src0) + # perturbing value_ep[d] must NOT change the decision value at d (uses d-1)... + src_d = _sources(daily=_daily(perturb=(d, SYMS[0], "value_ep", 999.0))) + v_d = materialize_range(vf, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, sources=src_d) + assert v0.loc[(d, SYMS[0])] == v_d.loc[(d, SYMS[0])] + # ...but perturbing value_ep[d-1] MUST change it. + src_p = _sources(daily=_daily(perturb=(DATES[54], SYMS[0], "value_ep", 999.0))) + v_p = materialize_range(vf, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, sources=src_p) + assert v0.loc[(d, SYMS[0])] != v_p.loc[(d, SYMS[0])] + + +# --------------------------------------------------------------------------- # +# Minute cutoff + leakage + binding correctness +# --------------------------------------------------------------------------- # +def test_minute_cutoff_post_1450_bar_does_not_leak(): + """Perturbing a post-14:50 bar leaves the decision-view value bit-unchanged; + perturbing a pre-14:50 bar changes it (the cutoff is load-bearing).""" + d = DATES[35] + base = _minute() + src0 = _sources(minute=base) + jf = JumpAmountCorrFactor() + j0 = materialize_range(jf, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, + sources=src0, decision_cutoff="14:50:00") + # a 14:55 bar (available_time 14:56 > 14:50) is filtered out -> no effect. + post_t = pd.Timestamp(d) + pd.Timedelta("14:55:00") + src_post = _sources(minute=_minute(perturb=(post_t, SYMS[0], "amount", 9e9))) + j_post = materialize_range(jf, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, + sources=src_post, decision_cutoff="14:50:00") + assert j0.equals(j_post) or np.array_equal(j0.to_numpy(), j_post.to_numpy(), equal_nan=True) + # a 14:42 bar (available_time 14:43 <= 14:50) is inside the cutoff -> changes. + pre_t = pd.Timestamp(d) + pd.Timedelta("14:42:00") + src_pre = _sources(minute=_minute(perturb=(pre_t, SYMS[0], "amount", 9e9))) + j_pre = materialize_range(jf, view=View.DECISION, symbols=SYMS, emit_start=d, emit_end=d, + sources=src_pre, decision_cutoff="14:50:00") + assert not np.array_equal(j0.to_numpy(), j_pre.to_numpy(), equal_nan=True) + + +def test_minute_binding_close_view_matches_direct_compute(): + """close-view minute materialize == the free compute_* on the same bar window + (proves the binding calls the right function with the right constants).""" + src = _sources(minute=_minute()) + jf = JumpAmountCorrFactor() + got = materialize_range(jf, view=View.CLOSE, symbols=SYMS, emit_start=DATES[30], emit_end=DATES[38], sources=src) + win = MinuteProv(_minute()).minute_bars( + SYMS, DATES[30] - pd.Timedelta(days=60), DATES[38] + pd.Timedelta(days=1) + ) + direct = compute_jump_amount_corr(win, lookback_days=jf.lookback_days, name=jf.name) + dd = direct.index.get_level_values("date") + direct_win = direct[(dd >= DATES[30]) & (dd <= DATES[38])].sort_index() + got = got.reindex(direct_win.index) + # A wrong binding (wrong function or wrong constant) would differ by O(1); the + # only residual is the pandas rolling-sum accumulation over a differently-sized + # loaded window (~1e-15, an attributable float-reorder, §五), so a tight + # tolerance proves the binding while allowing the reorder. + assert np.allclose(got.to_numpy(), direct_win.to_numpy(), rtol=1e-9, atol=1e-12, equal_nan=True) + + +# --------------------------------------------------------------------------- # +# ex-date masking (fixture masked factor; the 14 are non-masked, no-op) +# --------------------------------------------------------------------------- # +class _MaskedDailyFactor(Factor): + """Fixture-only masked factor: surfaces close; NaN'd on ex-dates by the engine.""" + + name = "masked_fixture" + + @property + def spec(self) -> FactorSpec: + return FactorSpec( + factor_id="masked_fixture", version="1.0", description="fixture masked factor", + expected_ic_sign=1, is_intraday=False, forward_return_horizon=1, + return_basis="close_to_close", input_fields=("close",), + requires=(PanelField("close", source="market_daily"),), + adjustment="returns_invariant", overnight_boundary="masked", + family="fixture", min_history_bars=0, lookback_depth=1, + ) + + def compute(self, panel): + return panel["close"].rename(self.name) + + +class AdjProv: + def __init__(self, series): + self._s = series + + def adj_factor(self, symbols, start, end): + m = self._s.index.get_level_values("date") + return self._s[(m >= pd.Timestamp(start)) & (m <= pd.Timestamp(end))] + + +def test_ex_date_masking_nans_the_ex_date_row(): + """A masked factor's ex-date (d, symbol) is NaN; non-masked factors skip this.""" + d_ex = DATES[50] + # adj_factor jumps on d_ex for SYMS[0] only. + af_rows = [] + for s in SYMS: + for di, d in enumerate(DATES): + af = 1.0 if (s != SYMS[0] or d < d_ex) else 2.0 + af_rows.append((d, s, af)) + af = pd.DataFrame(af_rows, columns=["date", "symbol", "af"]).set_index(["date", "symbol"])["af"] + src = _sources(daily=_daily(), adj=AdjProv(af)) + got = materialize_range(_MaskedDailyFactor(), view=View.DECISION, symbols=SYMS, + emit_start=DATES[48], emit_end=DATES[52], sources=src) + assert np.isnan(got.loc[(d_ex, SYMS[0])]) # ex-date masked + assert np.isfinite(got.loc[(d_ex, SYMS[1])]) # other symbol untouched + assert np.isfinite(got.loc[(DATES[49], SYMS[0])]) # non-ex-date untouched + + +def test_non_masked_factor_never_consults_adj_provider(): + """The 14 closing factors are non-masked: the ex-date path is a no-op (no + AdjFactorProvider needed).""" + src = _sources(daily=_daily()) # NO adj provider + v = materialize_range(VolatilityFactor(window=20), view=View.DECISION, symbols=SYMS, + emit_start=DATES[45], emit_end=DATES[46], sources=src) + assert len(v) == 4 # ran fine without an adj provider + + +# --------------------------------------------------------------------------- # +# recompute-fn factory (D3 tail-recompute compatibility) +# --------------------------------------------------------------------------- # +def test_make_recompute_fn_matches_materialize_range(): + src = _sources(daily=_daily()) + vol = VolatilityFactor(window=20) + fn = make_recompute_fn(vol, view=View.DECISION, symbols=SYMS, sources=src, data_start=DATES[0]) + a = fn(DATES[40], DATES[50], vol.spec.lookback_depth) + b = materialize_range(vol, view=View.DECISION, symbols=SYMS, + emit_start=DATES[40], emit_end=DATES[50], sources=src) + assert np.array_equal(a.sort_index().to_numpy(), b.sort_index().to_numpy(), equal_nan=True) + + +# --------------------------------------------------------------------------- # +# live-config wiring (R5): no default fallback +# --------------------------------------------------------------------------- # +class _LiveCacheCfg: + refresh_recent_days = 14 + fina_tail_days = 400 + + +class _PartialCacheCfg: + refresh_recent_days = 14 # fina_tail_days missing + + +def test_build_horizon_config_from_live_config(): + hc = build_horizon_config(_LiveCacheCfg()) + assert hc.refresh_recent_days == 14 + assert hc.fina_tail_days == 400 + assert hc.overrides() == {"fina_indicator": 400} + + +def test_build_horizon_config_refuses_missing_attr(): + with pytest.raises(ValueError, match="fina_tail_days"): + build_horizon_config(_PartialCacheCfg()) diff --git a/tests/test_factor_view_lag.py b/tests/test_factor_view_lag.py new file mode 100644 index 0000000..fdfe758 --- /dev/null +++ b/tests/test_factor_view_lag.py @@ -0,0 +1,129 @@ +"""D4: availability-lag transforms (factors.view_lag) — R18 per field-class. + +These pin the pure transforms the materializer composes: the daily prev-day shift +with the field-level ``open`` exception, the minute cutoff, and the ex-date +BOOLEAN. The materializer's end-to-end R18 coverage rides on top (see +tests/test_factor_materialize.py); here we prove the transforms in isolation. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from data.clean.intraday_schema import normalize_intraday_bars +from factors.view_lag import ( + daily_decision_lag, + decision_same_day_fields, + ex_date_mask, + minute_decision_cutoff, +) + +SYMS = ["000001.SZ", "000002.SZ"] +DATES = pd.bdate_range("2024-01-02", periods=6) + + +def _daily(): + rows = [] + for si, s in enumerate(SYMS): + for di, d in enumerate(DATES): + base = 100.0 + si * 10 + di + rows.append((d, s, base + 0.1, base + 0.5, base - 0.5, base, 1e5 + di, base * 1e5)) + return ( + pd.DataFrame(rows, columns=["date", "symbol", "open", "high", "low", "close", "volume", "amount"]) + .set_index(["date", "symbol"]).sort_index() + ) + + +def test_decision_same_day_fields_is_open_only(): + """Policy-derived: the ONLY decision-view same-day factor field is open (R7).""" + assert decision_same_day_fields() == frozenset({"open"}) + + +def test_daily_decision_lag_shifts_prev_day_keeps_open(): + """close/high/low/volume/amount at d become d-1; open stays same-day.""" + panel = _daily() + lagged = daily_decision_lag(panel) + s = SYMS[0] + for di in range(1, len(DATES)): + d, prev = DATES[di], DATES[di - 1] + # open is same-day: unchanged. + assert lagged.loc[(d, s), "open"] == panel.loc[(d, s), "open"] + # prev-day fields carry the d-1 value. + for col in ("close", "high", "low", "volume", "amount"): + assert lagged.loc[(d, s), col] == panel.loc[(prev, s), col] + # the first date of each symbol has no d-1 -> shifted cols NaN, open present. + d0 = DATES[0] + assert np.isnan(lagged.loc[(d0, s), "close"]) + assert lagged.loc[(d0, s), "open"] == panel.loc[(d0, s), "open"] + + +def test_daily_decision_lag_does_not_mutate_input(): + panel = _daily() + before = panel.copy() + daily_decision_lag(panel) + pd.testing.assert_frame_equal(panel, before) + + +def test_daily_decision_lag_is_per_symbol(): + """The shift never pulls one symbol's row into another's.""" + panel = _daily() + lagged = daily_decision_lag(panel) + # symbol B's first date is NaN independently of symbol A. + assert np.isnan(lagged.loc[(DATES[0], SYMS[1]), "close"]) + + +def _bars(rows): + df = pd.DataFrame( + rows, columns=["time", "symbol", "open", "high", "low", "close", "volume", "amount"] + ) + return normalize_intraday_bars(df, freq="1min") + + +def test_minute_decision_cutoff_drops_post_1450_bars(): + """Bars whose available_time exceeds d 14:50 are removed; earlier kept.""" + day = "2024-01-02" + rows = [] + # bars at 14:48, 14:49 (kept), 14:50, 14:55 (dropped: available_time = bar_end+1min) + for hhmm in ("14:48", "14:49", "14:50", "14:55"): + t = pd.Timestamp(f"{day} {hhmm}:00") + rows.append((t, SYMS[0], 100.0, 100.5, 99.5, 100.0, 1e4, 1e6)) + bars = _bars(rows) + kept = minute_decision_cutoff(bars, decision_time="14:50:00") + kept_times = kept.index.get_level_values("time").strftime("%H:%M").tolist() + # available_time = bar_end + 1min: 14:48->14:49(<=14:50 keep), 14:49->14:50(keep), + # 14:50->14:51(drop), 14:55->14:56(drop). + assert kept_times == ["14:48", "14:49"] + + +def test_minute_decision_cutoff_does_not_mutate_input(): + day = "2024-01-02" + rows = [(pd.Timestamp(f"{day} 14:{m}:00"), SYMS[0], 100.0, 100.5, 99.5, 100.0, 1e4, 1e6) + for m in (40, 55)] + bars = _bars(rows) + before = bars.copy() + minute_decision_cutoff(bars) + pd.testing.assert_frame_equal(bars, before) + + +def test_ex_date_mask_is_boolean_and_flags_af_changes(): + """af(d) != af(d-1) per symbol; first row False; NaN-aware.""" + idx = pd.MultiIndex.from_tuples( + [(DATES[0], "A"), (DATES[1], "A"), (DATES[2], "A"), + (DATES[0], "B"), (DATES[1], "B")], + names=["date", "symbol"], + ) + af = pd.Series([1.0, 1.0, 2.0, 3.0, 3.0], index=idx) + mask = ex_date_mask(af) + assert mask.dtype == bool + assert mask.loc[(DATES[0], "A")] is np.False_ or not mask.loc[(DATES[0], "A")] # first row + assert not mask.loc[(DATES[1], "A")] # 1.0 -> 1.0 no change + assert mask.loc[(DATES[2], "A")] # 1.0 -> 2.0 change + assert not mask.loc[(DATES[0], "B")] # first row of B + assert not mask.loc[(DATES[1], "B")] # 3.0 -> 3.0 + + +def test_ex_date_mask_rejects_non_multiindex(): + with pytest.raises(ValueError): + ex_date_mask(pd.Series([1.0, 2.0])) From c36b0fe7119ab0685325c0f3e69d621ba4227f47 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 15:02:26 -0700 Subject: [PATCH 3/9] =?UTF-8?q?feat(factors):=20FactorService=20=E2=80=94?= =?UTF-8?q?=20DecisionPoint=20+=20read-through=20cross=5Fsection/panel=20(?= =?UTF-8?q?D4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A thin read-through module (design §3.5, R25): two functions and a frozen DecisionPoint, not a behavior-class stack. Both cross_section and panel read-through the value store; a miss triggers the ONE materializer engine to fill the gap and append, then re-reads — there is no second implementation path. Contract (structural): only ``available_time <= decision`` data is computed (the materializer applies the lag); NEVER touches execution prices (raw VALUES only, forward returns are born at the alpha/eval boundary); returns RAW values; the view x return-basis pairing is enforced at CALL time via the D0/D1 require_legal_pairing (decision<->exec_to_exec / close<->close_to_close). The data access (store + providers) is INJECTED — factors never carries a feed, token, or qt import. Tests (network-free), the §3.5 P8 acceptance: * single-fill (repeated per-date cross_section) == batch-fill (one panel): momentum_20 is EXACTLY bit-identical (window-local, depth 21 > headline 20 — the warmup trim uses the transitive depth); volatility_20 and the NESTED volume_peak_count_20 (depth 40) match up to an attributable pandas-rolling float-reorder (<= 1e-12); * MUTATION: forcing the warmup below the factor window makes per-date fills under-warm while batch fills do not -> the stores diverge (committed test asserts the divergence; a halved-warmup engine edit turns both single==batch tests red, restored -> green); * panel[d] == cross_section(d) on the same store; read-through hit does not re-hit the provider; universe slicing; the pairing gate rejects the two illegal (view, basis) combinations; panel requires a uniform cutoff. --- factors/service.py | 217 ++++++++++++++++++++++++++++ tests/test_factor_service.py | 273 +++++++++++++++++++++++++++++++++++ 2 files changed, 490 insertions(+) create mode 100644 factors/service.py create mode 100644 tests/test_factor_service.py diff --git a/factors/service.py b/factors/service.py new file mode 100644 index 0000000..8d0dc1b --- /dev/null +++ b/factors/service.py @@ -0,0 +1,217 @@ +"""Factor service: the thin read-through onto the value store (design §3.5, D4). + +Two functions and one frozen data class — NOT a behavior-class stack (R25). Both +functions READ-THROUGH the value store: a hit returns the stored RAW values, a +miss triggers the ONE materializer engine (``factors.materialize``) to fill the +gap and append, then re-reads. There is no second implementation path, which is +what makes ``panel[d] ≡ cross_section(d)`` a trivial read-layer smoke and the +real guarantee ``single-point fill ≡ batch fill`` (design §3.5 P8). + +Contract (all structural, not comments): +* only data with ``available_time <= decision`` is computed — the materializer + applies the (source, view) availability lag; the service never relaxes it; +* it NEVER touches execution prices — forward returns are born only at the + alpha/eval boundary (invariant #1); this returns raw factor VALUES only; +* it returns RAW values (processed is a consumer-side decision, red line #7); +* the view x return-basis pairing is enforced at CALL time via the D0/D1 + ``require_legal_pairing`` (decision<->exec_to_exec / close<->close_to_close); + any other pairing is a readable error, not a doc convention. + +Layering (red line #10): ``factors.service`` imports the factor layer + the pure +availability leaf only; the data access (store + providers) is INJECTED by the +consumer, so ``factors`` never carries a feed, a token, or a qt import. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass + +import pandas as pd + +from data.availability_policy import ReturnBasis, View, require_legal_pairing +from data.clean.intraday_schema import DEFAULT_DECISION_TIME +from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL +from factors import registry as factor_registry +from factors.base import Factor +from factors.materialize import MaterializeSources, materialize_range +from factors.store.fingerprint import data_fingerprint +from factors.store.keys import store_key +from factors.store.values import FactorValueStore + + +@dataclass(frozen=True) +class DecisionPoint: + """A decision instant: the date and the within-day cutoff (default 14:50). + + ``cutoff`` is the same string the FactorSpec intraday block declares as its + ``decision_cutoff`` (single source); the materializer filters minute bars to + ``available_time <= date + cutoff``. + """ + + date: pd.Timestamp + cutoff: str = DEFAULT_DECISION_TIME + + +def _build_factor(factor_id: str, params_by_id: Mapping[str, Mapping[str, object]] | None) -> Factor: + params = (params_by_id or {}).get(factor_id) + return factor_registry.build(factor_id, params) + + +def _fingerprint(factor: Factor) -> dict: + """The store data fingerprint for ``factor`` (none/returns_invariant here). + + price_level factors need the per-symbol adj_factor event table; the closing + 14 have zero such members, so ``data_fingerprint`` raises readably for them + (never a silent wrong fingerprint) until the service wires an adj-factor + source for that case (deferred with the ex-date masking). + """ + return data_fingerprint(adjustment=factor.spec.adjustment) + + +def _ensure_coverage( + factor: Factor, + key, + fingerprint: dict, + *, + dates: pd.DatetimeIndex, + symbols: list[str], + store: FactorValueStore, + sources: MaterializeSources, + view: View, + cutoff: str, +) -> pd.Series: + """Read-through: fill any date the store lacks for ``factor``, return the Series. + + A miss over the requested ``dates`` materializes ``[min(missing), max(missing)]`` + (the single engine) and upserts it; already-present dates in that range are + recomputed identically and win the dedup harmlessly. Returns the stored Series + (possibly empty). The per-date fill (dates = one) and the batch fill (dates = + many) leave BIT-IDENTICAL stores because the materializer's trailing trim is + warmup-consistent (design §3.5 P8). + """ + stored = store.read_valid(key, expected_fingerprint=fingerprint) + have = ( + pd.DatetimeIndex(pd.unique(stored.index.get_level_values(DATE_LEVEL))) + if stored is not None and not stored.empty + else pd.DatetimeIndex([]) + ) + missing = dates[~dates.isin(have)] + if len(missing): + fresh = materialize_range( + factor, + view=view, + symbols=list(symbols), + emit_start=missing.min(), + emit_end=missing.max(), + sources=sources, + decision_cutoff=cutoff, + ) + if not fresh.empty: + store.upsert(key, fresh, fingerprint=fingerprint) + stored = store.read_valid(key, expected_fingerprint=fingerprint) + if stored is None: + return pd.Series( + [], + index=pd.MultiIndex.from_arrays( + [pd.DatetimeIndex([]), pd.Index([], dtype=object)], + names=[DATE_LEVEL, SYMBOL_LEVEL], + ), + dtype=float, + name=factor.name, + ) + return stored + + +def _uniform_cutoff(decisions: list[DecisionPoint]) -> str: + cutoffs = {d.cutoff for d in decisions} + if len(cutoffs) != 1: + raise ValueError( + f"panel() requires all DecisionPoints to share one cutoff; got {sorted(cutoffs)}. " + f"Split the call per cutoff (the minute cutoff enters the store view semantics)." + ) + return next(iter(cutoffs)) + + +def _assemble( + factor_ids: list[str], + series_by_id: dict[str, pd.Series], + dates: pd.DatetimeIndex, + symbols: list[str], +) -> pd.DataFrame: + """Slice each factor's stored Series to (dates x universe) and stack to columns.""" + symbol_set = set(map(str, symbols)) + columns: dict[str, pd.Series] = {} + for fid in factor_ids: + s = series_by_id[fid] + if s.empty: + columns[fid] = s + continue + d = s.index.get_level_values(DATE_LEVEL) + sym = s.index.get_level_values(SYMBOL_LEVEL) + keep = d.isin(dates) & pd.Index(sym).isin(symbol_set) + columns[fid] = s[keep] + frame = pd.concat(columns, axis=1) + frame.index = frame.index.set_names([DATE_LEVEL, SYMBOL_LEVEL]) + return frame.sort_index(kind="mergesort") + + +def panel( + factor_ids: Iterable[str], + universe: Iterable[str], + decisions: list[DecisionPoint], + *, + store: FactorValueStore, + sources: MaterializeSources, + view: object = View.DECISION, + basis: object = ReturnBasis.EXEC_TO_EXEC, + params_by_id: Mapping[str, Mapping[str, object]] | None = None, +) -> pd.DataFrame: + """Raw factor values for ``factor_ids`` over ``decisions`` x ``universe``. + + Read-through: any decision date the store lacks is materialized once (batch) + and appended. The view x basis pairing is enforced up front. + """ + resolved_view, _ = require_legal_pairing(view, basis) + factor_ids = list(factor_ids) + symbols = [str(s) for s in universe] + if not decisions: + raise ValueError("panel() needs at least one DecisionPoint.") + cutoff = _uniform_cutoff(decisions) + dates = pd.DatetimeIndex(sorted({pd.Timestamp(d.date).normalize() for d in decisions})) + + series_by_id: dict[str, pd.Series] = {} + for fid in factor_ids: + factor = _build_factor(fid, params_by_id) + fp = _fingerprint(factor) + key = store_key(factor, view=resolved_view.value, params=(params_by_id or {}).get(fid)) + series_by_id[fid] = _ensure_coverage( + factor, key, fp, dates=dates, symbols=symbols, store=store, + sources=sources, view=resolved_view, cutoff=cutoff, + ) + return _assemble(factor_ids, series_by_id, dates, symbols) + + +def cross_section( + factor_ids: Iterable[str], + universe: Iterable[str], + decision: DecisionPoint, + *, + store: FactorValueStore, + sources: MaterializeSources, + view: object = View.DECISION, + basis: object = ReturnBasis.EXEC_TO_EXEC, + params_by_id: Mapping[str, Mapping[str, object]] | None = None, +) -> pd.DataFrame: + """Raw factor values for ``factor_ids`` at one ``decision`` x ``universe``. + + A convenience over :func:`panel` with a single decision — the SAME read-through + and the SAME engine, so ``panel[d] ≡ cross_section(d)`` on the same store. + """ + return panel( + factor_ids, universe, [decision], store=store, sources=sources, + view=view, basis=basis, params_by_id=params_by_id, + ) + + +__all__ = ["DecisionPoint", "cross_section", "panel"] diff --git a/tests/test_factor_service.py b/tests/test_factor_service.py new file mode 100644 index 0000000..547b968 --- /dev/null +++ b/tests/test_factor_service.py @@ -0,0 +1,273 @@ +"""D4: the factor service (factors.service) — the §3.5 P8 acceptance. + +Network-free. Proves the hardest guarantee: two COLD stores, one filled by +repeated per-date ``cross_section`` and one by a single batch ``panel``, hold +BIT-IDENTICAL raw values (design §3.5 P8). Also the read-layer smoke +(``panel[d] ≡ cross_section(d)``), the view x basis pairing gate, and the +read-through no-recompute-on-hit property. + +The P8 test uses THREE factors that stress the tail-splice / leading-NaN +boundary differently: +* ``momentum_20`` — window-local (a two-point ratio, NO accumulation), so single + and batch are EXACTLY bit-identical; its transitive depth (21) exceeds its + headline window (20), so the warmup trim uses the transitive depth, not the + headline (the §六.18 deception guard); +* ``volatility_20`` — rolling std (pandas accumulation), single==batch up to an + attributable float-reorder (<= 1e-12); +* ``volume_peak_count_20`` — a genuinely NESTED minute factor (depth 40 = 20 + + 20 baseline), single==batch up to the same float-reorder. + +Mutation evidence (recorded): forcing the materializer's warmup BELOW the +factor's window makes the per-date fill under-warm every date while the batch +fill only under-warms its first — so the two stores DIVERGE. The test that +applies this mutation asserts the divergence, proving the warmup trim (and the +transitive ``lookback_depth``) is load-bearing. +""" + +from __future__ import annotations + +import tempfile + +import numpy as np +import pandas as pd +import pytest + +from data.availability_policy import ReturnBasis, View +from data.clean.intraday_schema import normalize_intraday_bars +from factors import registry as factor_registry +from factors import service as service_mod +from factors.materialize import MaterializeSources, materialize_range +from factors.service import DecisionPoint, cross_section, panel +from factors.store.keys import store_key +from factors.store.values import FactorValueStore + +SYMS = ["000001.SZ", "000002.SZ"] +DATES = pd.bdate_range("2021-01-04", periods=75) + + +# --------------------------------------------------------------------------- # +# Synthetic data + providers +# --------------------------------------------------------------------------- # +def _daily(): + rng = np.random.RandomState(0) + rows = [] + for si, s in enumerate(SYMS): + px = 100.0 + si * 20 + np.cumsum(rng.normal(0, 1.0, len(DATES))) + for d, p in zip(DATES, px): + rows.append((d, s, p - 0.3, p + 0.5, p - 0.5, p, 1e5, p * 1e5)) + return ( + pd.DataFrame(rows, columns=["date", "symbol", "open", "high", "low", "close", "volume", "amount"]) + .set_index(["date", "symbol"]).sort_index() + ) + + +def _rich_minute(): + """55 days x 238-bar sessions with same-slot baseline + eruptions -> the + peak/valley taxonomy produces finite nested-factor values.""" + rng = np.random.RandomState(7) + rows = [] + for si, s in enumerate(SYMS): + for d in DATES[:55]: + base = pd.Timestamp(d) + pd.Timedelta("09:31:00") + price = 100.0 + si * 5 + rng.normal(0, 2) + for i in range(238): + t = base + pd.Timedelta(minutes=i) + price += rng.normal(0, 0.05) + slot_base = 1e4 * (1.0 + 0.3 * np.sin(i / 12.0)) + erupt = 6.0 if (rng.rand() < 0.06) else 1.0 + vol = slot_base * erupt * (1.0 + 0.1 * rng.rand()) + w = 0.15 * price * (1.0 + (2.0 if erupt > 1 else 0.0)) * (0.5 + rng.rand()) + hi, lo = price + abs(w) * rng.rand(), price - abs(w) * rng.rand() + cl = lo + (hi - lo) * rng.rand() + rows.append((t, s, price, hi, lo, cl, vol, cl * vol)) + frame = pd.DataFrame(rows, columns=["time", "symbol", "open", "high", "low", "close", "volume", "amount"]) + return normalize_intraday_bars(frame, freq="1min") + + +DAILY = _daily() +MINUTE = _rich_minute() + + +class DailyProv: + def daily_panel(self, symbols, start, end): + m = DAILY.index.get_level_values("date") + return DAILY[(m >= pd.Timestamp(start)) & (m <= pd.Timestamp(end))] + + +class MinuteProv: + def __init__(self): + self.calls = 0 + + def minute_bars(self, symbols, start, end): + self.calls += 1 + if not symbols: + return MINUTE.iloc[0:0] + t = MINUTE.index.get_level_values("time") + return MINUTE[(t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end))] + + +def _sources(): + return MaterializeSources(daily=DailyProv(), minute=MinuteProv()) + + +def _store_series(store, factor_id, view=View.DECISION): + factor = factor_registry.build(factor_id) + key = store_key(factor, view=view.value) + return store.read(key) + + +def _fill_single(store, factor_ids, dates, src): + for d in dates: + cross_section(factor_ids, SYMS, DecisionPoint(date=d), store=store, sources=src) + + +def _fill_batch(store, factor_ids, dates, src): + panel(factor_ids, SYMS, [DecisionPoint(date=d) for d in dates], store=store, sources=src) + + +# --------------------------------------------------------------------------- # +# P8: single-fill == batch-fill +# --------------------------------------------------------------------------- # +def test_single_fill_equals_batch_fill_momentum_bit_identical(): + """Window-local momentum: the two cold stores are EXACTLY bit-identical.""" + dates = list(DATES[40:56]) + with tempfile.TemporaryDirectory() as ta, tempfile.TemporaryDirectory() as tb: + a, b = FactorValueStore(ta), FactorValueStore(tb) + _fill_single(a, ["momentum_20"], dates, _sources()) + _fill_batch(b, ["momentum_20"], dates, _sources()) + sa = _store_series(a, "momentum_20").sort_index() + sb = _store_series(b, "momentum_20").sort_index() + assert sa.index.equals(sb.index) + av, bv = sa.to_numpy(), sb.to_numpy() + assert np.array_equal(np.isnan(av), np.isnan(bv)) + assert np.array_equal(av[~np.isnan(av)], bv[~np.isnan(bv)]) # BIT identical + + +def test_single_fill_equals_batch_fill_rolling_and_nested_minute(): + """volatility (rolling) + volume_peak_count (NESTED minute, depth 40): the two + stores match up to the attributable pandas-rolling float-reorder (<= 1e-12).""" + dates = list(DATES[42:54]) + for fid in ("volatility_20", "volume_peak_count_20"): + with tempfile.TemporaryDirectory() as ta, tempfile.TemporaryDirectory() as tb: + a, b = FactorValueStore(ta), FactorValueStore(tb) + _fill_single(a, [fid], dates, _sources()) + _fill_batch(b, [fid], dates, _sources()) + sa = _store_series(a, fid).sort_index() + sb = _store_series(b, fid).sort_index() + assert sa.index.equals(sb.index), fid + av, bv = sa.to_numpy(), sb.to_numpy() + assert np.array_equal(np.isnan(av), np.isnan(bv)), fid + finite = ~np.isnan(av) + # non-vacuous: the nested factor really produced values in this window. + assert finite.sum() > 0, fid + assert np.allclose(av[finite], bv[finite], rtol=1e-12, atol=1e-12), fid + + +def test_reduced_warmup_breaks_single_equals_batch(monkeypatch): + """MUTATION: forcing the warmup below the factor window makes per-date fill + under-warm every date while batch under-warms only its first -> the stores + DIVERGE. Proves the warmup trim (transitive lookback_depth) is load-bearing.""" + dates = list(DATES[40:56]) + + def _short_warmup(factor, **kw): + # 10 << momentum's window of 20: every per-date fill under-warms (all NaN), + # but the batch fill (loading 10 before d1) still computes its tail dates + # d >= d1 + (window - 10) -> the two NaN patterns diverge. + kw["warmup"] = 10 + return materialize_range(factor, **kw) + + monkeypatch.setattr(service_mod, "materialize_range", _short_warmup) + with tempfile.TemporaryDirectory() as ta, tempfile.TemporaryDirectory() as tb: + a, b = FactorValueStore(ta), FactorValueStore(tb) + _fill_single(a, ["momentum_20"], dates, _sources()) + _fill_batch(b, ["momentum_20"], dates, _sources()) + sa = _store_series(a, "momentum_20").sort_index() + sb = _store_series(b, "momentum_20").sort_index() + av, bv = sa.to_numpy(), sb.to_numpy() + # the NaN pattern (per-date all-NaN vs batch finite near the tail) differs. + assert not np.array_equal(np.isnan(av), np.isnan(bv)) + + +# --------------------------------------------------------------------------- # +# read-layer smoke + read-through +# --------------------------------------------------------------------------- # +def test_panel_equals_cross_section_on_same_store(): + dates = list(DATES[40:52]) + src = _sources() + with tempfile.TemporaryDirectory() as td: + store = FactorValueStore(td) + pan = panel(["momentum_20", "volatility_20"], SYMS, + [DecisionPoint(date=d) for d in dates], store=store, sources=src) + d = dates[6] + xs = cross_section(["momentum_20", "volatility_20"], SYMS, DecisionPoint(date=d), + store=store, sources=src) + pan_d = pan[pan.index.get_level_values("date") == d].sort_index() + xs = xs.sort_index() + assert xs.index.equals(pan_d.index) + assert np.allclose(xs.to_numpy(), pan_d.to_numpy(), equal_nan=True, rtol=0, atol=0) + + +def test_read_through_hit_does_not_recompute(): + """A second call over already-covered dates does not re-hit the provider.""" + dates = list(DATES[40:50]) + src = _sources() + with tempfile.TemporaryDirectory() as td: + store = FactorValueStore(td) + panel(["jump_amount_corr_20"], SYMS, [DecisionPoint(date=d) for d in dates], + store=store, sources=src) + calls_after_fill = src.minute.calls + assert calls_after_fill > 0 + # second identical call: store hit -> no new minute read + panel(["jump_amount_corr_20"], SYMS, [DecisionPoint(date=d) for d in dates], + store=store, sources=src) + assert src.minute.calls == calls_after_fill + + +def test_service_returns_only_universe_symbols(): + dates = list(DATES[40:44]) + src = _sources() + with tempfile.TemporaryDirectory() as td: + store = FactorValueStore(td) + pan = panel(["momentum_20"], [SYMS[0]], [DecisionPoint(date=d) for d in dates], + store=store, sources=src) + got_syms = set(pan.index.get_level_values("symbol")) + assert got_syms <= {SYMS[0]} + + +# --------------------------------------------------------------------------- # +# pairing gate +# --------------------------------------------------------------------------- # +def test_illegal_view_basis_pairing_raises(): + """close view scored on exec_to_exec (or decision on close_to_close) is a + readable construction-time error, not a doc convention (§1.4).""" + src = _sources() + with tempfile.TemporaryDirectory() as td: + store = FactorValueStore(td) + with pytest.raises(ValueError, match="illegal view/basis pairing"): + cross_section(["momentum_20"], SYMS, DecisionPoint(date=DATES[40]), + store=store, sources=src, view=View.CLOSE, basis=ReturnBasis.EXEC_TO_EXEC) + with pytest.raises(ValueError, match="illegal view/basis pairing"): + cross_section(["momentum_20"], SYMS, DecisionPoint(date=DATES[40]), + store=store, sources=src, view=View.DECISION, basis=ReturnBasis.CLOSE_TO_CLOSE) + + +def test_legal_pairings_are_accepted(): + src = _sources() + with tempfile.TemporaryDirectory() as td: + store = FactorValueStore(td) + # decision <-> exec_to_exec (default) and close <-> close_to_close + cross_section(["momentum_20"], SYMS, DecisionPoint(date=DATES[40]), + store=store, sources=src) + cross_section(["momentum_20"], SYMS, DecisionPoint(date=DATES[40]), + store=store, sources=src, view=View.CLOSE, basis=ReturnBasis.CLOSE_TO_CLOSE) + + +def test_panel_requires_uniform_cutoff(): + src = _sources() + with tempfile.TemporaryDirectory() as td: + store = FactorValueStore(td) + with pytest.raises(ValueError, match="share one cutoff"): + panel(["momentum_20"], SYMS, + [DecisionPoint(date=DATES[40], cutoff="14:50:00"), + DecisionPoint(date=DATES[41], cutoff="14:45:00")], + store=store, sources=src) From 44a8ee1a745dcd043b3b6110fda00f6d706ab706 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 15:05:00 -0700 Subject: [PATCH 4/9] refactor(factors): home the processing orchestration in factors/process (D4 R9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit drop_missing + winsorize + neutralize + zscore orchestration moves from qt.pipeline._process_factors into factors.process.orchestrate, so the D5 unified eval runner (analytics layer, which may not import qt, red line #10) and the backtest path share ONE processing truth instead of copying it (the P3-6 drop_missing-across-columns lesson) or breaking layering. * factors/process/orchestrate.py — process_factor_panel (constructs the SAME ProcessingPipeline with the SAME arguments and transforms) + covariates_from_panel (the industry/market_cap extraction the retired inline logic did). Covariates are INJECTED by the caller (factors never fetches, red line #3); the store stays raw-only (processing is a consumer-side decision on the raw panel). * qt.pipeline._process_factors becomes a THIN delegate — same covariate extraction, same pipeline args, verbatim behavior. Behavior preserved: phase0 anchor ic 0.9600 / annual 0.8408 unchanged; the orchestrator matches a directly-constructed ProcessingPipeline byte-for-byte (no-neutralize + neutralize), never mutates its input, and extracts covariates identically to the retired inline logic. --- factors/process/orchestrate.py | 75 ++++++++++++++++++++++++ qt/pipeline.py | 20 ++++--- tests/test_factor_process_orchestrate.py | 71 ++++++++++++++++++++++ 3 files changed, 158 insertions(+), 8 deletions(-) create mode 100644 factors/process/orchestrate.py create mode 100644 tests/test_factor_process_orchestrate.py diff --git a/factors/process/orchestrate.py b/factors/process/orchestrate.py new file mode 100644 index 0000000..1c78d3c --- /dev/null +++ b/factors/process/orchestrate.py @@ -0,0 +1,75 @@ +"""The single home of the factor-processing ORCHESTRATION (design §3.5 R9, D4). + +``drop_missing + winsorize + neutralize + zscore`` is a cross-sectional, +covariate-dependent transform whose ORCHESTRATION previously lived in +``qt.pipeline._process_factors`` and was consumed by 10+ modules. The D5 unified +eval runner will build in the analytics layer, where red line #10 forbids +``analytics -> qt`` — so the orchestration cannot stay in ``qt``. Left there, the +eval and backtest paths would each copy it (the P3-6 ``drop_missing``-across- +columns lesson) or break layering. + +It is homed HERE, in ``factors/process`` (where the ``neutralize_by_date`` +primitive already lives), so the eval and backtest paths share ONE processing +truth. The covariates (industry / market_cap) are INJECTED by the caller — +``factors`` never touches a feed (red line #3) — and the store stays raw-only +(red line #7): processing is a consumer-side decision applied to the RAW factor +panel the service returns, never persisted as a source of truth. + +Behavior is preserved verbatim: this constructs the SAME +:class:`~factors.process.pipeline.ProcessingPipeline` with the SAME arguments the +retired ``_process_factors`` did and calls ``transform`` — so the phase0 anchor +and every existing processing number are byte-for-byte unchanged. +""" + +from __future__ import annotations + +import pandas as pd + +from factors.process.pipeline import ProcessingPipeline + + +def covariates_from_panel( + panel: pd.DataFrame, +) -> tuple[pd.Series | None, pd.Series | None]: + """Pull the ``industry`` / ``market_cap`` neutralization covariates off a panel. + + Returns ``(industry, market_cap)``, each ``None`` when the column is absent + (the retired ``_process_factors`` did exactly this). The caller placed the + covariates on the panel (e.g. ``_maybe_enrich_covariates``); ``factors`` never + fetches them itself. + """ + industry = panel["industry"] if "industry" in panel.columns else None + market_cap = panel["market_cap"] if "market_cap" in panel.columns else None + return industry, market_cap + + +def process_factor_panel( + factor_panel: pd.DataFrame, + *, + drop_missing: bool = True, + standardize: bool = True, + winsorize: bool = False, + neutralize: bool = False, + industry: pd.Series | None = None, + market_cap: pd.Series | None = None, +) -> pd.DataFrame: + """Run the cross-sectional processing pipeline over a RAW factor panel. + + The ONE orchestration point (design §3.5 R9): eval and backtest both call + this, so ``drop_missing`` (cross-column, per date) and neutralization can + never drift between the two paths. Covariates are injected; when + ``neutralize`` is enabled but they are absent, ``ProcessingPipeline`` raises a + readable error rather than silently skipping. + """ + pipeline = ProcessingPipeline( + drop_missing=drop_missing, + standardize=standardize, + winsorize=winsorize, + neutralize=neutralize, + industry=industry, + market_cap=market_cap, + ) + return pipeline.transform(factor_panel) + + +__all__ = ["covariates_from_panel", "process_factor_panel"] diff --git a/qt/pipeline.py b/qt/pipeline.py index 91b341d..98e1c53 100644 --- a/qt/pipeline.py +++ b/qt/pipeline.py @@ -57,7 +57,7 @@ from factors.compute.candidates import ValueFactor from factors.compute.financial import SUPPORTED_FIELDS as SUPPORTED_FINANCIAL_FIELDS from factors.compute.financial import FinancialFactor -from factors.process.pipeline import ProcessingPipeline +from factors.process.orchestrate import covariates_from_panel, process_factor_panel from portfolio.construct import TopNEqualWeight from qt.config import RootConfig, load_config from qt.reports import write_phase0_summary @@ -967,13 +967,18 @@ def _process_factors( ) -> pd.DataFrame: """Run drop_missing + (winsorize) + neutralize + z-score from config toggles. - Neutralization pulls the ``industry`` / ``market_cap`` covariates off the panel - (placed there by :func:`_maybe_enrich_covariates`). When neutralize is enabled - but they are absent, ``ProcessingPipeline`` raises a readable error. + D4 (factor-refactor R9): a THIN delegate to + :func:`factors.process.orchestrate.process_factor_panel`, the single home of + the processing orchestration (so the eval and backtest paths share ONE truth + without ``analytics -> qt``). Behavior is preserved verbatim — same covariate + extraction, same pipeline arguments. Neutralization pulls the ``industry`` / + ``market_cap`` covariates off the panel (placed there by + :func:`_maybe_enrich_covariates`); when neutralize is enabled but they are + absent, the orchestrator's ``ProcessingPipeline`` raises a readable error. """ - industry = panel["industry"] if "industry" in panel.columns else None - market_cap = panel["market_cap"] if "market_cap" in panel.columns else None - pipeline = ProcessingPipeline( + industry, market_cap = covariates_from_panel(panel) + return process_factor_panel( + factor_panel, drop_missing=cfg.processing.drop_missing, standardize=cfg.processing.standardize.enabled, winsorize=cfg.processing.winsorize.enabled, @@ -981,7 +986,6 @@ def _process_factors( industry=industry, market_cap=market_cap, ) - return pipeline.transform(factor_panel) def _maybe_enrich_covariates( diff --git a/tests/test_factor_process_orchestrate.py b/tests/test_factor_process_orchestrate.py new file mode 100644 index 0000000..9774bfc --- /dev/null +++ b/tests/test_factor_process_orchestrate.py @@ -0,0 +1,71 @@ +"""D4 R9: the processing orchestration home (factors.process.orchestrate). + +Proves the orchestrator is behavior-preserving vs a directly-constructed +ProcessingPipeline (so ``qt.pipeline._process_factors`` delegating to it changes +no number), and that covariate extraction matches the retired inline logic. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from factors.process.orchestrate import covariates_from_panel, process_factor_panel +from factors.process.pipeline import ProcessingPipeline + +DATES = pd.bdate_range("2024-01-02", periods=8) +SYMS = ["A", "B", "C", "D"] +IDX = pd.MultiIndex.from_product([DATES, SYMS], names=["date", "symbol"]) + + +def _factor_panel(): + rng = np.random.RandomState(3) + return pd.DataFrame( + {"f1": rng.normal(size=len(IDX)), "f2": rng.normal(size=len(IDX))}, index=IDX + ) + + +def test_orchestrator_matches_direct_pipeline_no_neutralize(): + fp = _factor_panel() + got = process_factor_panel(fp, drop_missing=True, standardize=True) + direct = ProcessingPipeline(drop_missing=True, standardize=True).transform(fp) + pd.testing.assert_frame_equal(got, direct) + + +def test_orchestrator_matches_direct_pipeline_with_neutralize(): + fp = _factor_panel() + rng = np.random.RandomState(4) + industry = pd.Series(rng.choice(["X", "Y"], size=len(IDX)), index=IDX) + market_cap = pd.Series(np.abs(rng.normal(size=len(IDX))) + 1.0, index=IDX) + got = process_factor_panel( + fp, drop_missing=True, standardize=True, neutralize=True, + industry=industry, market_cap=market_cap, + ) + direct = ProcessingPipeline( + drop_missing=True, standardize=True, neutralize=True, + industry=industry, market_cap=market_cap, + ).transform(fp) + pd.testing.assert_frame_equal(got, direct) + + +def test_orchestrator_does_not_mutate_input(): + fp = _factor_panel() + before = fp.copy() + process_factor_panel(fp, drop_missing=True, standardize=True) + pd.testing.assert_frame_equal(fp, before) + + +def test_covariates_from_panel_extracts_present_columns(): + panel = pd.DataFrame( + {"close": 1.0, "industry": "X", "market_cap": 2.0}, index=IDX + ) + industry, market_cap = covariates_from_panel(panel) + assert industry is not None and market_cap is not None + pd.testing.assert_series_equal(industry, panel["industry"]) + pd.testing.assert_series_equal(market_cap, panel["market_cap"]) + + +def test_covariates_from_panel_returns_none_when_absent(): + panel = pd.DataFrame({"close": 1.0}, index=IDX) + industry, market_cap = covariates_from_panel(panel) + assert industry is None and market_cap is None From e61f446e4114d8ab3c0fb2a6d48ee3348ba018c4 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 15:11:54 -0700 Subject: [PATCH 5/9] =?UTF-8?q?test(factors):=20hot-path=20timing=20smoke?= =?UTF-8?q?=20=E2=80=94=20shared=20vs=20naive=20minute=20read=20(D4=20R20/?= =?UTF-8?q?=C2=A7=E5=85=AB)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit qt/factor_hotpath_smoke.py measures, on the REAL intraday cache (cache-only, stk_mins_live_calls=0), the wall-time gap between NAIVE per-factor loading (each materialize_range loads its own bars — the D4 service's current behavior) and SHARED loading (read + normalize + cutoff once, then every factor computed from the shared bars — the primitives-layer amortization §3.2 calls the load-bearing hot-path mechanism). The budget is measured, not claimed (§八), with a caliber caveat that it does NOT extrapolate to full-A (~6x the CSI500 read). A standalone dev tool (not in qt/cli, like qt.panel_freeze); run manually after symlinking the main checkout's artifacts (or --cache-root the absolute path). Measured (CSI500 sample, 40 symbols, window 2024-03-01..2024-04-05, 753,710 post-cutoff 1min bar rows, cache-only stk_mins_live_calls=0): 6 factors : NAIVE 8.90s (6 loads) / SHARED 4.36s (1 load) = 2.04x 10 factors: NAIVE 14.22s (10 loads) / SHARED 7.13s (1 load) = 2.00x Shared-read roughly halves the wall time on this sample — the direction of §八 (load amortizes across factors); the plateau at ~2x reflects the nested factors' heavy per-factor compute relative to the shared read on this short window. --- qt/factor_hotpath_smoke.py | 191 +++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 qt/factor_hotpath_smoke.py diff --git a/qt/factor_hotpath_smoke.py b/qt/factor_hotpath_smoke.py new file mode 100644 index 0000000..c5c4602 --- /dev/null +++ b/qt/factor_hotpath_smoke.py @@ -0,0 +1,191 @@ +"""Hot-path timing smoke for the D4 materializer (design §八 R20). + +Measures, on the REAL intraday cache (cache-only, ``stk_mins_live_calls=0``), the +magnitude gap between: + +* NAIVE per-factor loading — each factor materialized separately, so the 1min + read + normalize is paid ONCE PER FACTOR (what the D4 service does today: each + ``materialize_range`` call loads its own bars); and +* SHARED loading — the 1min read + normalize + cutoff paid ONCE, then every + factor computed from the shared bars (the ``factors.compute.minute.primitives`` + amortization the design §3.2 calls the load-bearing hot-path mechanism). + +The budget is MEASURED, not claimed (§八): the caliber (universe sample, window, +factor count) and the two wall times are printed as-is, with an explicit caveat +that the numbers do NOT extrapolate to full-A (~6x the CSI500 read). + +Run: ``python -m qt.factor_hotpath_smoke`` (deliberately NOT in qt/cli). It reads +``artifacts/cache/tushare/v1`` (symlink the main checkout's artifacts into the +worktree first; remove the symlink after so ``git status`` stays clean). +""" + +from __future__ import annotations + +import argparse +import time +from dataclasses import dataclass +from pathlib import Path + +import pandas as pd + +from data.cache.intraday_cache import ENDPOINT as INTRADAY_ENDPOINT +from data.cache.intraday_cache import READ_COLUMNS +from data.cache.intraday_parquet_store import IntradayParquetStore +from data.clean.intraday_schema import ( + RAW_INTRADAY_FREQ, + empty_intraday_bars, + normalize_intraday_bars, +) +from factors import registry as factor_registry +from factors.compute.minute.binding import minute_raw_from_bars +from factors.materialize import MaterializeSources, materialize_range +from factors.view_lag import minute_decision_cutoff + +DEFAULT_CACHE_ROOT = "artifacts/cache/tushare/v1" +DEFAULT_FACTORS = ( + "jump_amount_corr_20", + "minute_ideal_amp_10", + "amp_marginal_anomaly_vol_20", + "volume_peak_count_20", + "valley_relative_vwap_20", + "peak_ridge_amount_ratio_20", +) +DEFAULT_START = "2024-03-01" +DEFAULT_END = "2024-04-05" +DEFAULT_N_SYMBOLS = 40 + + +class CacheMinuteProvider: + """Cache-only MinuteBarProvider: per-symbol read + normalize, zero live calls.""" + + def __init__(self, root: str) -> None: + self._store = IntradayParquetStore(root) + self.calls = 0 + self.live_calls = 0 # provably 0 — read_range has no fetch closure + + def minute_bars(self, symbols, start, end): + self.calls += 1 + if not symbols: + return empty_intraday_bars() + parts = [] + s = pd.Timestamp(start) + e = pd.Timestamp(end) + for sym in symbols: + part = self._store.read_range(INTRADAY_ENDPOINT, sym, RAW_INTRADAY_FREQ, s, e) + if part.empty: + continue + parts.append( + normalize_intraday_bars( + part.rename(columns={"bar_end": "time"})[READ_COLUMNS], freq=RAW_INTRADAY_FREQ + ) + ) + if not parts: + return empty_intraday_bars() + return pd.concat(parts).sort_index(kind="mergesort") + + +def _sample_symbols(root: str, n: int) -> list[str]: + """First ``n`` symbols the intraday store has, in a stable directory order.""" + base = Path(root) / f"{INTRADAY_ENDPOINT}" / f"freq={RAW_INTRADAY_FREQ}" + syms: list[str] = [] + for prefix_dir in sorted(base.glob("symbol_prefix=*")): + for sym_dir in sorted(prefix_dir.glob("symbol=*")): + syms.append(sym_dir.name.split("=", 1)[1]) + if len(syms) >= n: + return syms + return syms + + +@dataclass(frozen=True) +class SmokeResult: + n_symbols: int + n_factors: int + window: str + naive_seconds: float + shared_seconds: float + ratio: float + naive_provider_calls: int + shared_provider_calls: int + live_calls: int + raw_bar_rows: int + + +def run_hotpath_smoke( + *, + cache_root: str = DEFAULT_CACHE_ROOT, + factor_ids=DEFAULT_FACTORS, + start: str = DEFAULT_START, + end: str = DEFAULT_END, + n_symbols: int = DEFAULT_N_SYMBOLS, + cutoff: str = "14:50:00", +) -> SmokeResult: + factor_ids = list(factor_ids) + symbols = _sample_symbols(cache_root, n_symbols) + factors = [factor_registry.build(fid) for fid in factor_ids] + emit_start = pd.Timestamp(start).normalize() + emit_end = pd.Timestamp(end).normalize() + + # -- NAIVE: each factor materialized separately (one provider load per factor). + naive_provider = CacheMinuteProvider(cache_root) + naive_src = MaterializeSources(minute=naive_provider) + t0 = time.monotonic() + for factor in factors: + materialize_range( + factor, view="decision", symbols=symbols, emit_start=emit_start, + emit_end=emit_end, sources=naive_src, decision_cutoff=cutoff, + ) + naive_seconds = time.monotonic() - t0 + + # -- SHARED: read + normalize + cutoff ONCE, then compute every factor. + max_depth = max(int(f.spec.lookback_depth) for f in factors) + shared_provider = CacheMinuteProvider(cache_root) + load_start = emit_start - pd.Timedelta(days=max_depth * 2 + 25) + t0 = time.monotonic() + bars = shared_provider.minute_bars(symbols, load_start, emit_end + pd.Timedelta(days=1)) + bars = minute_decision_cutoff(bars, decision_time=cutoff) + raw_rows = len(bars) + for factor in factors: + raw = minute_raw_from_bars(factor, bars) + dd = raw.index.get_level_values("date") + _ = raw[(dd >= emit_start) & (dd <= emit_end)] + shared_seconds = time.monotonic() - t0 + + return SmokeResult( + n_symbols=len(symbols), + n_factors=len(factors), + window=f"{start}..{end}", + naive_seconds=round(naive_seconds, 2), + shared_seconds=round(shared_seconds, 2), + ratio=round(naive_seconds / shared_seconds, 2) if shared_seconds else float("nan"), + naive_provider_calls=naive_provider.calls, + shared_provider_calls=shared_provider.calls, + live_calls=naive_provider.live_calls + shared_provider.live_calls, + raw_bar_rows=raw_rows, + ) + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--cache-root", default=DEFAULT_CACHE_ROOT) + parser.add_argument("--start", default=DEFAULT_START) + parser.add_argument("--end", default=DEFAULT_END) + parser.add_argument("--n-symbols", type=int, default=DEFAULT_N_SYMBOLS) + args = parser.parse_args(argv) + r = run_hotpath_smoke( + cache_root=args.cache_root, start=args.start, end=args.end, n_symbols=args.n_symbols + ) + print(f"caliber: {r.n_symbols} symbols x {r.n_factors} minute factors, window {r.window}") + print(f"raw 1min bar rows (shared load, post-cutoff): {r.raw_bar_rows}") + print(f"NAIVE (per-factor load): {r.naive_seconds}s ({r.naive_provider_calls} provider loads)") + print(f"SHARED (load once): {r.shared_seconds}s ({r.shared_provider_calls} provider load)") + print(f"naive / shared ratio: {r.ratio}x") + print(f"stk_mins_live_calls: {r.live_calls} (cache-only)") + print( + "CAVEAT: measured on this CSI500 sample/window only; does NOT extrapolate " + "to full-A (~6x the read). The budget is measured, not claimed (§八)." + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover - thin CLI shim + raise SystemExit(main()) From 29d02050339cd47fec3b6557ae03df3900151aad Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 16:07:47 -0700 Subject: [PATCH 6/9] fix(factors): saturation-expanding load for valid-day-pooled factors (D4 review HIGH) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Valid-day-POOLED factors (ridge_minute_return / valley_ridge_vwap_ratio / peak_ridge_amount_ratio and their structural siblings) count VALID days in the trailing window (invalid days do not occupy a slot -> roll over a valid-day-indexed series), so the CALENDAR lookback depth of a lookback_days-valid-day pool is DATA-DEPENDENT and UNBOUNDED, and loading more history can re-classify boundary days. A fixed lookback_depth trim therefore truncated sparse-valid-day symbols' pools, making the stored value depend on LOAD GEOMETRY: a single-date fill and a batch fill diverged finite<->NaN (review repro ridge_minute_return_20 (2021-03-18, 000002.SZ): single=1.6976..., batch=NaN). This DISPROVES design §3.3 R4's "W_dep = sum of lookbacks" assumption (it treated valid-day slack as bounded; it is unbounded) — a design correction. Fix (red line #6: the value must be f(factor, params, view, data), never load geometry): * factors/compute/minute/binding.py: a CLOSED partition of the minute surface — VALID_DAY_POOLED_FACTORS (8, code-inspected by the rolling mechanism, NOT by observed divergence: only 3 diverged on the review's data, the other 5 are the "happens-to-be-clean" siblings, #82 lesson; valley_price_quantile declared though deferred so D5 cannot bind it with the defect) vs _BOUNDED_MINUTE_FACTORS (3); a minute factor in neither is a readable error. * factors/materialize.py: pooled factors load by SATURATION-EXPANSION — expand the load backward in chunks and recompute the emit-range values until they stop changing (saturated) or the provider returns no earlier bars (the real data start = the structural terminal); bounded factors keep the fixed trim. The chunk is generous vs baseline_days so one no-change implies saturation. NOT load-from-data-start: the real cache holds minute bars from 2015-01-05, so that would be an ~11-year load per pooled fill. * Single-fill and batch-fill now agree: the finite<->NaN divergence is gone (NaN mask load-geometry-free); the residual is only the pandas-accumulation float-reorder (JC1, <= 1e-12; on a short window both fills reach the data start and are bit-identical). D3 linkage (review point 5): make_recompute_fn delegates to materialize_range, so the tail-recompute path saturates pooled factors through the SAME code path — no second implementation. Tests: P8 expansion (the 3 divergent factors, single==batch on a sparse-valid window) + mutation (saturation OFF via monkeypatch -> divergence returns) + classification pinned (8 pooled / 3 bounded, unclassified raises). The hot-path smoke's naive path now uses a bounded per-factor load (never the pooled expansion) so it measures pure read amortization (2.0x, cache-only) without the saturation confound. --- factors/compute/minute/binding.py | 70 +++++++++++++++++++- factors/materialize.py | 88 ++++++++++++++++++++++++- qt/factor_hotpath_smoke.py | 25 +++++--- tests/test_factor_materialize.py | 61 ++++++++++++++++++ tests/test_factor_service.py | 103 ++++++++++++++++++++++++++++++ 5 files changed, 337 insertions(+), 10 deletions(-) diff --git a/factors/compute/minute/binding.py b/factors/compute/minute/binding.py index c4b5575..4084132 100644 --- a/factors/compute/minute/binding.py +++ b/factors/compute/minute/binding.py @@ -116,6 +116,68 @@ def _call(factor: Factor, bars: pd.DataFrame) -> pd.Series: } +#: VALID-DAY POOLED factors (design §3.3 review HIGH): their trailing window +#: counts VALID days (invalid days do NOT occupy a slot — they roll over a +#: valid-day-indexed series via ``primitives.rolling_valid_days`` or an equivalent +#: ``.loc[valid_days].rolling(...)`` / valid-day python loop), so the CALENDAR +#: lookback depth of a trailing ``lookback_days`` valid-day pool is DATA-DEPENDENT +#: and UNBOUNDED, and loading more history can also re-classify boundary days. +#: A fixed ``lookback_depth`` trim therefore truncates sparse-valid-day symbols' +#: pools, making the stored value depend on LOAD GEOMETRY (red line #6: the value +#: must be f(factor, params, view, data), never the anchor/window shape). The +#: materializer must load these to SATURATION (design's structural terminal = +#: real data start) so single-fill and batch-fill agree. Classification is +#: STRUCTURAL (by the rolling mechanism, code-inspected), NOT by observed +#: divergence: only ridge_minute_return / valley_ridge_vwap_ratio / +#: peak_ridge_amount_ratio happened to diverge on the review's data, but +#: volume_peak_count / peak_interval_kurtosis / intraday_amp_cut / +#: valley_relative_vwap roll over valid days too and are the "happens-to-be-clean" +#: representatives (#82 lesson: never fix only what you observed). +#: valley_price_quantile is DEFERRED (needs the daily panel) but declared HERE so +#: D5 cannot bind it carrying the fixed-depth defect. +VALID_DAY_POOLED_FACTORS: frozenset[type[Factor]] = frozenset({ + VolumePeakCountFactor, + PeakIntervalKurtosisFactor, + IntradayAmpCutFactor, + ValleyRelativeVwapFactor, + ValleyRidgeVwapRatioFactor, + RidgeMinuteReturnFactor, + PeakRidgeAmountRatioFactor, + ValleyPriceQuantileFactor, +}) + +#: The bounded minute factors (trailing window over ALL trading days, not valid +#: days) — a fixed lookback_depth trim is load-geometry-free for these. Kept as +#: an explicit set so the classification is a CLOSED partition of the minute +#: surface (a new minute factor missing from BOTH sets is a readable error). +_BOUNDED_MINUTE_FACTORS: frozenset[type[Factor]] = frozenset({ + JumpAmountCorrFactor, + MinuteIdealAmplitudeFactor, + AmpMarginalAnomalyVolFactor, +}) + + +def is_valid_day_pooled(factor: Factor) -> bool: + """True iff ``factor``'s trailing window counts VALID days (unbounded depth). + + Readable error for a minute factor in NEITHER partition set — a new minute + factor must be classified explicitly (never silently treated as bounded, + which would reintroduce the load-geometry divergence). + """ + cls = type(factor) + if cls in VALID_DAY_POOLED_FACTORS: + return True + if cls in _BOUNDED_MINUTE_FACTORS: + return False + raise KeyError( + f"{factor.name} ({cls.__name__}) is a minute factor not classified as " + f"valid-day-pooled or bounded in factors.compute.minute.binding. Add it to " + f"exactly one partition set (VALID_DAY_POOLED_FACTORS if its trailing " + f"window counts VALID days, else _BOUNDED_MINUTE_FACTORS) — a missing " + f"classification would silently size its saturation load wrong." + ) + + def is_minute_bound(factor: Factor) -> bool: """True iff ``factor`` has a bars-only raw-compute binding here.""" return type(factor) in _MINUTE_BINDINGS @@ -139,4 +201,10 @@ def minute_raw_from_bars(factor: Factor, bars: pd.DataFrame) -> pd.Series: ) -__all__ = ["BindingFn", "is_minute_bound", "minute_raw_from_bars"] +__all__ = [ + "VALID_DAY_POOLED_FACTORS", + "BindingFn", + "is_minute_bound", + "is_valid_day_pooled", + "minute_raw_from_bars", +] diff --git a/factors/materialize.py b/factors/materialize.py index 21319ce..4209d59 100644 --- a/factors/materialize.py +++ b/factors/materialize.py @@ -30,13 +30,18 @@ from dataclasses import dataclass from typing import Protocol +import numpy as np import pandas as pd from data.availability_policy import STK_MINS_1MIN, OvernightBoundary, View from data.clean.intraday_schema import DEFAULT_DECISION_TIME from data.clean.schema import DATE_LEVEL, SYMBOL_LEVEL from factors.base import Factor -from factors.compute.minute.binding import is_minute_bound, minute_raw_from_bars +from factors.compute.minute.binding import ( + is_minute_bound, + is_valid_day_pooled, + minute_raw_from_bars, +) from factors.store.incremental import CacheHorizonConfig from factors.view_lag import ( daily_decision_lag, @@ -52,6 +57,19 @@ def _load_buffer_calendar_days(warmup: int) -> int: return int(warmup) * 2 + 25 +#: One backward-expansion chunk (calendar days) for the valid-day-pooled +#: saturation loop. Generous vs the baseline depth so a single no-change across a +#: chunk implies saturation (the boundary days gained far more than baseline_days +#: of prior history, so their classification is stable). See ``_materialize_pooled``. +def _saturation_chunk_calendar_days(warmup: int) -> int: + return int(warmup) * 2 + 40 + + +#: Hard ceiling on saturation-expansion chunks (defensive; the loop terminates +#: structurally on value-stability or provider exhaustion long before this). +_MAX_SATURATION_CHUNKS: int = 500 + + # --------------------------------------------------------------------------- # # Injected data providers (the consumer wires these to feed/cache; tests fake) # --------------------------------------------------------------------------- # @@ -215,6 +233,14 @@ def _materialize_minute( # Readable error (e.g. valley_price_quantile needs the daily panel too). return minute_raw_from_bars(factor, sources.minute.minute_bars([], load_start, emit_end)) # Load one extra calendar day past emit_end so bar cutoffs on emit_end resolve. + # POOLED (valid-day trailing window): the pool counts VALID days, so its + # CALENDAR depth is data-dependent and UNBOUNDED — a fixed trim would make the + # value depend on load geometry (review HIGH / red line #6). Load by + # SATURATION-EXPANSION instead (below); bounded factors keep the fixed trim. + if is_valid_day_pooled(factor): + return _materialize_pooled( + factor, view, list(symbols), emit_start, emit_end, warmup, sources, decision_cutoff, + ) bars = sources.minute.minute_bars( list(symbols), load_start, emit_end + pd.Timedelta(days=1) ) @@ -226,6 +252,66 @@ def _materialize_minute( return minute_raw_from_bars(factor, bars) +def _materialize_pooled( + factor, view, symbols, emit_start, emit_end, warmup, sources, decision_cutoff, +) -> pd.Series: + """Saturation-expanding load for a valid-day-pooled factor (design §3.3, review HIGH). + + Expands the load backward in chunks and recomputes the EMIT-range values until + they stop changing (saturated: adding more history no longer alters the pool + composition or the boundary classifications) OR the provider returns no more + earlier bars (the real data start — the structural terminal). The chunk is + generous vs ``baseline_days`` so one no-change implies saturation. The returned + values are load-geometry-free: the finite<->NaN divergence between a single-date + fill and a batch fill is eliminated (the residual across the two fills is only + the pandas-accumulation float-reorder, JC1 <= 1e-12, because the two fills may + terminate at different — but each individually saturated — load starts). + """ + chunk = pd.Timedelta(days=_saturation_chunk_calendar_days(warmup)) + end = emit_end + pd.Timedelta(days=1) + load_start = emit_start - chunk + prev_emit: pd.Series | None = None + prev_nbars = -1 + for _ in range(_MAX_SATURATION_CHUNKS): + bars = sources.minute.minute_bars(symbols, load_start, end) + nbars = len(bars) + if nbars == 0: + return _empty_series(factor.name) + work = bars + if view is View.DECISION: + work = minute_decision_cutoff(work, decision_time=decision_cutoff) + emit = _restrict_emit( + minute_raw_from_bars(factor, work), emit_start, emit_end, factor.name + ) + if prev_emit is not None and _pooled_emit_saturated(prev_emit, emit): + return emit + if nbars == prev_nbars: # provider exhausted -> real data start reached + return emit + prev_emit, prev_nbars = emit, nbars + load_start = load_start - chunk + return prev_emit if prev_emit is not None else _empty_series(factor.name) + + +def _pooled_emit_saturated(prev: pd.Series, cur: pd.Series, tol: float = 1e-12) -> bool: + """True iff the emit-range values stopped changing across a chunk expansion. + + Compares on the union index (NaN where absent): the NaN mask must match + EXACTLY (a finite<->NaN flip is a real pool change, not saturation) and the + finite values must agree within ``tol`` (a chunk changes the array length, so + the pandas rolling-sum accumulation reorders finite values at ~1e-15 even when + the pool composition is stable — that float-reorder is not a pool change). + """ + index = prev.index.union(cur.index) + a = prev.reindex(index).to_numpy(dtype=float) + b = cur.reindex(index).to_numpy(dtype=float) + if not np.array_equal(np.isnan(a), np.isnan(b)): + return False + finite = ~np.isnan(a) + if not finite.any(): + return True + return bool(np.allclose(a[finite], b[finite], rtol=0.0, atol=tol)) + + def _trim_daily(panel: pd.DataFrame, emit_start: pd.Timestamp, warmup: int) -> pd.DataFrame: dates = pd.DatetimeIndex(pd.unique(panel.index.get_level_values(DATE_LEVEL))) keep_from = _warmup_start(dates, emit_start, warmup) diff --git a/qt/factor_hotpath_smoke.py b/qt/factor_hotpath_smoke.py index c5c4602..8be402d 100644 --- a/qt/factor_hotpath_smoke.py +++ b/qt/factor_hotpath_smoke.py @@ -38,7 +38,6 @@ ) from factors import registry as factor_registry from factors.compute.minute.binding import minute_raw_from_bars -from factors.materialize import MaterializeSources, materialize_range from factors.view_lag import minute_decision_cutoff DEFAULT_CACHE_ROOT = "artifacts/cache/tushare/v1" @@ -125,23 +124,33 @@ def run_hotpath_smoke( emit_start = pd.Timestamp(start).normalize() emit_end = pd.Timestamp(end).normalize() - # -- NAIVE: each factor materialized separately (one provider load per factor). + # The smoke isolates the READ amortization only, so BOTH paths use the same + # BOUNDED load window (never the valid-day-pooled SATURATION expansion, which + # is a separate correctness path — its multi-load cost would confound a pure + # read-cost comparison). The bounded window comfortably covers each factor's + # trailing depth; the shared load uses the max depth so every factor is served. + max_depth = max(int(f.spec.lookback_depth) for f in factors) + end_bar = emit_end + pd.Timedelta(days=1) + + # -- NAIVE: read + normalize + cutoff ONCE PER FACTOR (its own bounded window). naive_provider = CacheMinuteProvider(cache_root) - naive_src = MaterializeSources(minute=naive_provider) t0 = time.monotonic() for factor in factors: - materialize_range( - factor, view="decision", symbols=symbols, emit_start=emit_start, - emit_end=emit_end, sources=naive_src, decision_cutoff=cutoff, + w = int(factor.spec.lookback_depth) + ls = emit_start - pd.Timedelta(days=w * 2 + 25) + bars = minute_decision_cutoff( + naive_provider.minute_bars(symbols, ls, end_bar), decision_time=cutoff ) + raw = minute_raw_from_bars(factor, bars) + dd = raw.index.get_level_values("date") + _ = raw[(dd >= emit_start) & (dd <= emit_end)] naive_seconds = time.monotonic() - t0 # -- SHARED: read + normalize + cutoff ONCE, then compute every factor. - max_depth = max(int(f.spec.lookback_depth) for f in factors) shared_provider = CacheMinuteProvider(cache_root) load_start = emit_start - pd.Timedelta(days=max_depth * 2 + 25) t0 = time.monotonic() - bars = shared_provider.minute_bars(symbols, load_start, emit_end + pd.Timedelta(days=1)) + bars = shared_provider.minute_bars(symbols, load_start, end_bar) bars = minute_decision_cutoff(bars, decision_time=cutoff) raw_rows = len(bars) for factor in factors: diff --git a/tests/test_factor_materialize.py b/tests/test_factor_materialize.py index f8121c0..eec609b 100644 --- a/tests/test_factor_materialize.py +++ b/tests/test_factor_materialize.py @@ -121,6 +121,67 @@ def test_is_minute_factor(): assert not is_minute_factor(ValueFactor("value_ep")) +# --------------------------------------------------------------------------- # +# valid-day POOLED classification (review HIGH) — a CLOSED partition +# --------------------------------------------------------------------------- # +def test_valid_day_pooled_classification_is_pinned(): + """Every minute factor is classified pooled (valid-day trailing window, + unbounded calendar depth) or bounded (all-trading-day window). Pinned by CODE + INSPECTION of the rolling mechanism, not by observed divergence (only 3 of the + 8 pooled factors happened to diverge on the review's data — #82 lesson).""" + from factors.compute.minute.binding import is_valid_day_pooled + from factors.compute.minute.intraday_amp_cut import IntradayAmpCutFactor + from factors.compute.minute.minute_ideal_amplitude import MinuteIdealAmplitudeFactor + from factors.compute.minute.peak_interval_kurtosis import PeakIntervalKurtosisFactor + from factors.compute.minute.peak_ridge_amount_ratio import PeakRidgeAmountRatioFactor + from factors.compute.minute.ridge_minute_return import RidgeMinuteReturnFactor + from factors.compute.minute.valley_price_quantile import ValleyPriceQuantileFactor + from factors.compute.minute.valley_relative_vwap import ValleyRelativeVwapFactor + from factors.compute.minute.valley_ridge_vwap_ratio import ValleyRidgeVwapRatioFactor + from factors.compute.minute.volume_peak_count import VolumePeakCountFactor + from factors.compute.minute.amp_marginal_anomaly_vol import AmpMarginalAnomalyVolFactor + + pooled = [ + VolumePeakCountFactor(), PeakIntervalKurtosisFactor(), IntradayAmpCutFactor(), + ValleyRelativeVwapFactor(), ValleyRidgeVwapRatioFactor(), RidgeMinuteReturnFactor(), + PeakRidgeAmountRatioFactor(), ValleyPriceQuantileFactor(), + ] + bounded = [ + JumpAmountCorrFactor(), MinuteIdealAmplitudeFactor(), AmpMarginalAnomalyVolFactor(), + ] + for f in pooled: + assert is_valid_day_pooled(f), f"{f.name} must be valid-day pooled" + for f in bounded: + assert not is_valid_day_pooled(f), f"{f.name} must be bounded" + assert len(pooled) == 8 and len(bounded) == 3 # the whole 11-factor minute surface + + +def test_unclassified_minute_factor_raises(): + """A minute factor in NEITHER partition set is a readable error (never + silently treated as bounded, which would reintroduce the divergence).""" + from factors.compute.minute.binding import is_valid_day_pooled + + class _UnclassifiedMinute(Factor): + name = "unclassified_minute" + + @property + def spec(self) -> FactorSpec: + return FactorSpec( + factor_id="unclassified_minute", version="1.0", description="fixture", + expected_ic_sign=1, is_intraday=False, forward_return_horizon=1, + return_basis="close_to_close", input_fields=("close",), + requires=(PanelField("close", source="stk_mins_1min"),), + adjustment="returns_invariant", overnight_boundary="none", + family="fixture", min_history_bars=0, lookback_depth=20, + ) + + def compute(self, panel): # pragma: no cover - never reached + return panel["close"].rename(self.name) + + with pytest.raises(KeyError, match="not classified"): + is_valid_day_pooled(_UnclassifiedMinute()) + + # --------------------------------------------------------------------------- # # Daily decision-view materialization + R18 daily lag # --------------------------------------------------------------------------- # diff --git a/tests/test_factor_service.py b/tests/test_factor_service.py index 547b968..74b4af2 100644 --- a/tests/test_factor_service.py +++ b/tests/test_factor_service.py @@ -106,10 +106,51 @@ def minute_bars(self, symbols, start, end): return MINUTE[(t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end))] +def _sparse_minute(): + """Sparse-valid minute data: every 3rd day is thin (fewer bars) so the + valid-day density is < 1 — the window where a fixed lookback_depth trim makes + valid-day-POOLED factors (ridge / valley_ridge / peak_ridge) diverge + finite<->NaN between a single-date fill and a batch fill (the review HIGH).""" + rng = np.random.RandomState(11) + rows = [] + for si, s in enumerate(SYMS): + for di, d in enumerate(DATES[:80]): + n = 120 if (di % 3 == 0) else 238 + base = pd.Timestamp(d) + pd.Timedelta("09:31:00") + price = 100.0 + si * 5 + rng.normal(0, 2) + for i in range(n): + t = base + pd.Timedelta(minutes=i) + price += rng.normal(0, 0.05) + slot = 1e4 * (1.0 + 0.3 * np.sin(i / 12.0)) + erupt = 6.0 if (rng.rand() < 0.06) else 1.0 + vol = slot * erupt * (1.0 + 0.1 * rng.rand()) + w = 0.15 * price * (1.0 + (2.0 if erupt > 1 else 0.0)) * (0.5 + rng.rand()) + hi, lo = price + abs(w) * rng.rand(), price - abs(w) * rng.rand() + cl = lo + (hi - lo) * rng.rand() + rows.append((t, s, price, hi, lo, cl, vol, cl * vol)) + frame = pd.DataFrame(rows, columns=["time", "symbol", "open", "high", "low", "close", "volume", "amount"]) + return normalize_intraday_bars(frame, freq="1min") + + +SPARSE_MINUTE = _sparse_minute() + + +class SparseMinuteProv: + def minute_bars(self, symbols, start, end): + if not symbols: + return SPARSE_MINUTE.iloc[0:0] + t = SPARSE_MINUTE.index.get_level_values("time") + return SPARSE_MINUTE[(t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end))] + + def _sources(): return MaterializeSources(daily=DailyProv(), minute=MinuteProv()) +def _sparse_sources(): + return MaterializeSources(minute=SparseMinuteProv()) + + def _store_series(store, factor_id, view=View.DECISION): factor = factor_registry.build(factor_id) key = store_key(factor, view=view.value) @@ -188,6 +229,68 @@ def _short_warmup(factor, **kw): assert not np.array_equal(np.isnan(av), np.isnan(bv)) +# --------------------------------------------------------------------------- # +# P8 EXPANSION: valid-day POOLED factors (review HIGH) on a sparse-valid window +# --------------------------------------------------------------------------- # +# WHY these factors and this window (the review's "false confidence"): the P8 +# single==batch test above uses volume_peak_count, which is ALSO valid-day pooled +# but happened to have dense-enough valid days to stay clean — it is the +# "happens-to-be-clean nested representative". The review showed that on a sparse- +# valid window ridge_minute_return / valley_ridge_vwap_ratio / peak_ridge_amount_ +# ratio produce a single-fill=finite / batch-fill=NaN divergence (e.g. the clean +# cell ridge_minute_return_20 (2021-03-18, 000002.SZ): single=1.6976..., batch=NaN), +# because the trailing pool counts VALID days and a fixed lookback_depth trim +# truncates the pool differently for a per-date fill vs a batch fill. The +# materializer now loads these factors to SATURATION (real data start, no trim), +# so the value is load-geometry-free and the two stores agree. +_POOLED_DIVERGENT = ["ridge_minute_return_20", "valley_ridge_vwap_ratio_20", "peak_ridge_amount_ratio_20"] + + +def test_pooled_factor_single_equals_batch_on_sparse_valid_window(): + """The valid-day-pooled factors that diverge under a fixed trim now agree + (saturation load) — single-fill store == batch-fill store, and the pool is + fuller than a truncated one so the result is non-vacuous (real finite values).""" + dates = list(DATES[55:75]) + for fid in _POOLED_DIVERGENT: + with tempfile.TemporaryDirectory() as ta, tempfile.TemporaryDirectory() as tb: + a, b = FactorValueStore(ta), FactorValueStore(tb) + _fill_single(a, [fid], dates, _sparse_sources()) + _fill_batch(b, [fid], dates, _sparse_sources()) + sa = _store_series(a, fid).sort_index() + sb = _store_series(b, fid).sort_index() + assert sa.index.equals(sb.index), fid + av, bv = sa.to_numpy(), sb.to_numpy() + assert np.array_equal(np.isnan(av), np.isnan(bv)), f"{fid}: NaN mask diverges" + finite = ~np.isnan(av) + assert finite.sum() > 0, f"{fid}: vacuous (no finite values)" + # saturation gives shared-prefix accumulation -> BIT identical. + assert np.array_equal(av[finite], bv[finite]), f"{fid}: finite values diverge" + + +def test_disabling_saturation_reintroduces_pooled_divergence(monkeypatch): + """MUTATION: treating a pooled factor as fixed-depth (saturation OFF) brings + back the single-fill/batch-fill finite<->NaN divergence -> the saturation + load is load-bearing. rc=1 (this asserts the divergence) with the mutation; + rc=0 (the test above) without it.""" + import factors.materialize as mat + + monkeypatch.setattr(mat, "is_valid_day_pooled", lambda factor: False) + dates = list(DATES[55:75]) + any_diverged = False + for fid in _POOLED_DIVERGENT: + with tempfile.TemporaryDirectory() as ta, tempfile.TemporaryDirectory() as tb: + a, b = FactorValueStore(ta), FactorValueStore(tb) + _fill_single(a, [fid], dates, _sparse_sources()) + _fill_batch(b, [fid], dates, _sparse_sources()) + sa = _store_series(a, fid).sort_index() + sb = _store_series(b, fid).sort_index() + av = sa.reindex(sb.index).to_numpy() + bv = sb.to_numpy() + if not np.array_equal(np.isnan(av), np.isnan(bv)): + any_diverged = True + assert any_diverged, "saturation-off must reintroduce a pooled divergence" + + # --------------------------------------------------------------------------- # # read-layer smoke + read-through # --------------------------------------------------------------------------- # From 6e9486da9d519a73de57d12166fc3edd2fd600d5 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Sat, 25 Jul 2026 05:35:46 -0700 Subject: [PATCH 7/9] fix(factors): structural pooled-saturation criterion + declared data floor (D4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CORRECTION to the previous commit's overclaim: it said the single/batch divergence "is gone". Its real reach was narrower — moderate sparsity was fixed, but a LONG no-bar gap (a suspension) still broke it, in two ways the review reproduced: 1. the value-stability fixed point false-positives: one expansion chunk landing ENTIRELY inside the gap adds no bars, so the emit values are unchanged and the loop terminated early; 2. an unchanged row count inside the gap was read as "provider exhausted = data start", so a mid-history gap masqueraded as the terminal. Result: ridge_minute_return 2021-11-04 single=3.2032634 vs batch=3.6674902 — a silent finite-vs-finite WRONG value, not float noise. The old chunk-size argument covered baseline re-classification only, not pool-count completeness: two failure modes, one guarded. This commit replaces the fixed point with a STRUCTURAL criterion (value-stability checking deleted entirely) and an explicit data floor: * LOCKING ARGUMENT (in the docstring): expanding backward can only change the classification of days within ``baseline_days`` trading days of the loaded window's start (the rolling baseline takes the most recent strictly-prior same-slot observations; at full depth earlier history cannot move it). Locking is POSITION-MONOTONE: drop the first ``baseline_days`` loaded trading days and every later day's classification is FINAL. * CRITERION: saturated iff the LOCKED sub-window holds >= ``lookback_days`` FINAL valid days at or before the earliest emit date — the pool only takes the most recent ``lookback_days`` valid days, so it can never reach the unlocked head. Sufficient, structural, deterministic, no iteration to a fixed point. Valid days are counted as the factor's own OUTPUT dates (a pooled factor emits one row per valid day), so a gap contributes none, the count stalls, and expansion continues — gap-immune by construction. * ``baseline_days`` is DERIVED per factor from its own parameter (peak family = VOLUME_PRV_BASELINE_DAYS; intraday_amp_cut = 0, within-day classification), not hardcoded. * DECLARED data floor: ``MinuteBarProvider.earliest_available()``; row-count inference is refused (a provider without it raises readably for a pooled factor). The chunk size is now only a search STEP, never a correctness input. Reach, stated precisely (no repeat of the overclaim): the short fixtures are bit-identical because both fills reach the declared floor; on long data the claim is exact NaN-mask agreement + finite values within 1e-12 (each fill terminates at its own saturated start, and by the criterion's sufficiency both agree with a common deeper start) — this reasoning is in the docstring. Tests: the review's counterexample committed (200-trading-day gap > one chunk, emit after the gap, pool spanning it) with a non-vacuity assertion; the earlier thin-day fixture kept (it covers the moderate-sparsity regime); providers declare their floor. MUTATION rc recorded in the test docstring: reverting to the value-stability fixed point FAILS it (rc=1, both fills emit all-NaN), restored passes (rc=0); dropping the locking offset does NOT fail on these fixtures and is recorded honestly as an unisolated correctness margin. --- factors/compute/minute/binding.py | 59 +++++++++-- factors/materialize.py | 163 ++++++++++++++++++++++-------- qt/factor_hotpath_smoke.py | 13 +++ tests/test_factor_service.py | 113 +++++++++++++++++++++ 4 files changed, 294 insertions(+), 54 deletions(-) diff --git a/factors/compute/minute/binding.py b/factors/compute/minute/binding.py index 4084132..da5fca7 100644 --- a/factors/compute/minute/binding.py +++ b/factors/compute/minute/binding.py @@ -66,6 +66,7 @@ ValleyRidgeVwapRatioFactor, compute_valley_ridge_vwap_ratio, ) +from factors.compute.minute.primitives import VOLUME_PRV_BASELINE_DAYS from factors.compute.minute.volume_peak_count import ( VolumePeakCountFactor, compute_volume_peak_count, @@ -135,16 +136,25 @@ def _call(factor: Factor, bars: pd.DataFrame) -> pd.Series: #: representatives (#82 lesson: never fix only what you observed). #: valley_price_quantile is DEFERRED (needs the daily panel) but declared HERE so #: D5 cannot bind it carrying the fixed-depth defect. -VALID_DAY_POOLED_FACTORS: frozenset[type[Factor]] = frozenset({ - VolumePeakCountFactor, - PeakIntervalKurtosisFactor, - IntradayAmpCutFactor, - ValleyRelativeVwapFactor, - ValleyRidgeVwapRatioFactor, - RidgeMinuteReturnFactor, - PeakRidgeAmountRatioFactor, - ValleyPriceQuantileFactor, -}) +#: pooled factor class -> its same-slot BASELINE depth in trading days (the +#: "locking offset", review point 1): a loaded day is CLASSIFICATION-FINAL only +#: once it has this many strictly-prior trading days (the baseline reaches full +#: depth and no earlier history can change it). Derived from each factor's own +#: baseline parameter, NOT hardcoded: the peak family shares +#: ``VOLUME_PRV_BASELINE_DAYS``; ``intraday_amp_cut`` has NO cross-day baseline +#: (its classification is within-day), so its locking offset is 0. +_POOLED_BASELINE_DAYS: dict[type[Factor], int] = { + VolumePeakCountFactor: VOLUME_PRV_BASELINE_DAYS, + PeakIntervalKurtosisFactor: VOLUME_PRV_BASELINE_DAYS, + ValleyRelativeVwapFactor: VOLUME_PRV_BASELINE_DAYS, + ValleyRidgeVwapRatioFactor: VOLUME_PRV_BASELINE_DAYS, + RidgeMinuteReturnFactor: VOLUME_PRV_BASELINE_DAYS, + PeakRidgeAmountRatioFactor: VOLUME_PRV_BASELINE_DAYS, + ValleyPriceQuantileFactor: VOLUME_PRV_BASELINE_DAYS, + IntradayAmpCutFactor: 0, # within-day classification, no cross-day baseline +} + +VALID_DAY_POOLED_FACTORS: frozenset[type[Factor]] = frozenset(_POOLED_BASELINE_DAYS) #: The bounded minute factors (trailing window over ALL trading days, not valid #: days) — a fixed lookback_depth trim is load-geometry-free for these. Kept as @@ -157,6 +167,33 @@ def _call(factor: Factor, bars: pd.DataFrame) -> pd.Series: }) +def pooled_baseline_days(factor: Factor) -> int: + """The factor's same-slot baseline depth = the LOCKING OFFSET (trading days). + + Expanding the load backward can only change the classification of days that + sit within ``baseline_days`` trading days of the loaded window's start (the + rolling baseline takes the ``baseline_days`` most recent STRICTLY-PRIOR + same-slot observations; once it is at full depth, earlier history cannot + change it). So the locking is POSITION-MONOTONE: drop the first + ``baseline_days`` trading days of the loaded window and every later day's + classification is FINAL. Readable error for a non-pooled factor. + """ + cls = type(factor) + if cls not in _POOLED_BASELINE_DAYS: + raise KeyError( + f"{factor.name} ({cls.__name__}) is not a valid-day-pooled factor, so " + f"it has no baseline locking offset." + ) + return int(_POOLED_BASELINE_DAYS[cls]) + + +def pooled_lookback_days(factor: Factor) -> int: + """The factor's trailing pool size in VALID days (its ``lookback_days``).""" + if type(factor) not in _POOLED_BASELINE_DAYS: + raise KeyError(f"{factor.name} is not a valid-day-pooled factor.") + return int(factor.lookback_days) # type: ignore[attr-defined] + + def is_valid_day_pooled(factor: Factor) -> bool: """True iff ``factor``'s trailing window counts VALID days (unbounded depth). @@ -207,4 +244,6 @@ def minute_raw_from_bars(factor: Factor, bars: pd.DataFrame) -> pd.Series: "is_minute_bound", "is_valid_day_pooled", "minute_raw_from_bars", + "pooled_baseline_days", + "pooled_lookback_days", ] diff --git a/factors/materialize.py b/factors/materialize.py index 4209d59..164d01f 100644 --- a/factors/materialize.py +++ b/factors/materialize.py @@ -30,7 +30,6 @@ from dataclasses import dataclass from typing import Protocol -import numpy as np import pandas as pd from data.availability_policy import STK_MINS_1MIN, OvernightBoundary, View @@ -41,6 +40,8 @@ is_minute_bound, is_valid_day_pooled, minute_raw_from_bars, + pooled_baseline_days, + pooled_lookback_days, ) from factors.store.incremental import CacheHorizonConfig from factors.view_lag import ( @@ -57,10 +58,11 @@ def _load_buffer_calendar_days(warmup: int) -> int: return int(warmup) * 2 + 25 -#: One backward-expansion chunk (calendar days) for the valid-day-pooled -#: saturation loop. Generous vs the baseline depth so a single no-change across a -#: chunk implies saturation (the boundary days gained far more than baseline_days -#: of prior history, so their classification is stable). See ``_materialize_pooled``. +#: One backward-expansion step (calendar days) of the valid-day-pooled saturation +#: loop. Only a STEP SIZE (how fast the search walks back), never a correctness +#: input: termination is decided by the structural criterion in +#: ``_materialize_pooled`` or by the provider's declared earliest boundary, so a +#: too-small chunk costs iterations, never correctness. def _saturation_chunk_calendar_days(warmup: int) -> int: return int(warmup) * 2 + 40 @@ -83,13 +85,28 @@ def daily_panel( class MinuteBarProvider(Protocol): - """Loads normalized 1min bars (cache-only) for the symbols over the window.""" + """Loads normalized 1min bars (cache-only) for the symbols over the window. + + Providers feeding a VALID-DAY-POOLED factor must also implement + :meth:`earliest_available` — the saturation loop needs a RELIABLE data-start + signal. Inferring "no more history" from an unchanged row count is WRONG: a + long mid-history gap (a suspension) yields an unchanged count while real + earlier history still exists (the review's counterexample #2). + """ def minute_bars( self, symbols: list[str], start: pd.Timestamp, end: pd.Timestamp ) -> pd.DataFrame: """MultiIndex(time, symbol) bars (:mod:`data.clean.intraday_schema`).""" + def earliest_available(self, symbols: list[str]) -> pd.Timestamp: + """The earliest date for which ANY of ``symbols`` could have bars. + + The declared lower bound of the provider's data (a cache-coverage floor + or a documented constant). Loading from it means "all history that + exists is loaded"; a mid-history gap must NEVER be mistaken for it. + """ + class AdjFactorProvider(Protocol): """Loads the RAW adj_factor series (ONLY consumed as an ex-date BOOLEAN).""" @@ -257,59 +274,117 @@ def _materialize_pooled( ) -> pd.Series: """Saturation-expanding load for a valid-day-pooled factor (design §3.3, review HIGH). - Expands the load backward in chunks and recomputes the EMIT-range values until - they stop changing (saturated: adding more history no longer alters the pool - composition or the boundary classifications) OR the provider returns no more - earlier bars (the real data start — the structural terminal). The chunk is - generous vs ``baseline_days`` so one no-change implies saturation. The returned - values are load-geometry-free: the finite<->NaN divergence between a single-date - fill and a batch fill is eliminated (the residual across the two fills is only - the pandas-accumulation float-reorder, JC1 <= 1e-12, because the two fills may - terminate at different — but each individually saturated — load starts). + STRUCTURAL saturation criterion (NOT a value-stability fixed point — that was + disproved: one expansion chunk landing entirely inside a long no-bar gap + leaves the emit values unchanged and would falsely terminate). + + The locking argument (why the criterion is SUFFICIENT): + + * Expanding the load backward can only change the classification of days + within ``baseline_days`` trading days of the loaded window's start — the + rolling baseline takes the ``baseline_days`` most recent STRICTLY-PRIOR + same-slot observations, and once at full depth no earlier history can move + it. Locking is therefore POSITION-MONOTONE: **drop the first + ``baseline_days`` trading days of the loaded window and every later day's + classification is FINAL.** + * The trailing pool takes only the most recent ``lookback_days`` VALID days. + So if the LOCKED sub-window already contains >= ``lookback_days`` final + valid days at or before the earliest emit date, the pool for every emit + date is drawn entirely from final days and can never reach back into the + unlocked head — further history cannot change any emit value. + + Termination: the criterion above, OR the load start reaching the provider's + DECLARED earliest boundary (then all existing history is loaded and the value + is well-defined). A mid-history gap is never mistaken for the data start + (that is exactly the row-count inference this replaced). + + Valid days are counted as the factor's own OUTPUT dates: a pooled factor + emits one row per valid day, so output dates ARE the valid days — the count + is read off the engine's own result rather than a re-implementation of each + factor's validity rule, and a gap simply contributes no output dates (the + count stalls and the loop keeps expanding, which is the desired behaviour). """ chunk = pd.Timedelta(days=_saturation_chunk_calendar_days(warmup)) end = emit_end + pd.Timedelta(days=1) + floor = _provider_earliest(sources.minute, symbols, factor) + baseline_days = pooled_baseline_days(factor) + lookback_days = pooled_lookback_days(factor) + load_start = emit_start - chunk - prev_emit: pd.Series | None = None - prev_nbars = -1 + last: pd.Series | None = None for _ in range(_MAX_SATURATION_CHUNKS): + at_floor = load_start <= floor + if at_floor: + load_start = floor bars = sources.minute.minute_bars(symbols, load_start, end) - nbars = len(bars) - if nbars == 0: - return _empty_series(factor.name) + if bars.empty: + if at_floor: + return _empty_series(factor.name) + load_start = load_start - chunk + continue work = bars if view is View.DECISION: work = minute_decision_cutoff(work, decision_time=decision_cutoff) - emit = _restrict_emit( - minute_raw_from_bars(factor, work), emit_start, emit_end, factor.name - ) - if prev_emit is not None and _pooled_emit_saturated(prev_emit, emit): - return emit - if nbars == prev_nbars: # provider exhausted -> real data start reached - return emit - prev_emit, prev_nbars = emit, nbars + full = minute_raw_from_bars(factor, work) + last = _restrict_emit(full, emit_start, emit_end, factor.name) + if at_floor or _pooled_pool_saturated( + full, work, emit_start, + baseline_days=baseline_days, lookback_days=lookback_days, + ): + return last load_start = load_start - chunk - return prev_emit if prev_emit is not None else _empty_series(factor.name) + return last if last is not None else _empty_series(factor.name) -def _pooled_emit_saturated(prev: pd.Series, cur: pd.Series, tol: float = 1e-12) -> bool: - """True iff the emit-range values stopped changing across a chunk expansion. +def _provider_earliest(provider, symbols, factor) -> pd.Timestamp: + """The provider's DECLARED earliest-available date (never inferred).""" + getter = getattr(provider, "earliest_available", None) + if getter is None: + raise ValueError( + f"{factor.name} is a valid-day-pooled factor, so its saturation load " + f"needs a RELIABLE data-start signal, but the minute provider " + f"({type(provider).__name__}) does not implement earliest_available(). " + f"Inferring the data start from an unchanged row count is unsound — a " + f"long mid-history no-bar gap (a suspension) looks identical to it. " + f"Declare the provider's earliest available date." + ) + return pd.Timestamp(getter(list(symbols))).normalize() + - Compares on the union index (NaN where absent): the NaN mask must match - EXACTLY (a finite<->NaN flip is a real pool change, not saturation) and the - finite values must agree within ``tol`` (a chunk changes the array length, so - the pandas rolling-sum accumulation reorders finite values at ~1e-15 even when - the pool composition is stable — that float-reorder is not a pool change). +def _pooled_pool_saturated( + full: pd.Series, + bars: pd.DataFrame, + emit_start: pd.Timestamp, + *, + baseline_days: int, + lookback_days: int, +) -> bool: + """The structural criterion (see :func:`_materialize_pooled` for the argument). + + True iff, for EVERY symbol present, the LOCKED sub-window (the loaded trading + days after dropping the first ``baseline_days``) holds at least + ``lookback_days`` valid days at or before ``emit_start`` — valid days being + the factor's own output dates. A symbol whose whole history is loaded shorter + than that is honestly under-warmed and is handled by the floor terminal, not + here. """ - index = prev.index.union(cur.index) - a = prev.reindex(index).to_numpy(dtype=float) - b = cur.reindex(index).to_numpy(dtype=float) - if not np.array_equal(np.isnan(a), np.isnan(b)): + if full.empty: + return False + loaded_days = pd.DatetimeIndex( + pd.unique(pd.DatetimeIndex(bars.index.get_level_values("time")).normalize()) + ).sort_values() + if len(loaded_days) <= baseline_days: return False - finite = ~np.isnan(a) - if not finite.any(): - return True - return bool(np.allclose(a[finite], b[finite], rtol=0.0, atol=tol)) + locked_from = loaded_days[baseline_days] # first day with a FINAL classification + + out_dates = full.index.get_level_values(DATE_LEVEL) + out_symbols = full.index.get_level_values(SYMBOL_LEVEL) + for symbol in pd.unique(out_symbols): + mine = out_dates[(out_symbols == symbol)] + final_valid = mine[(mine >= locked_from) & (mine <= emit_start)] + if len(final_valid) < lookback_days: + return False + return True def _trim_daily(panel: pd.DataFrame, emit_start: pd.Timestamp, warmup: int) -> pd.DataFrame: diff --git a/qt/factor_hotpath_smoke.py b/qt/factor_hotpath_smoke.py index 8be402d..4c7056f 100644 --- a/qt/factor_hotpath_smoke.py +++ b/qt/factor_hotpath_smoke.py @@ -41,6 +41,10 @@ from factors.view_lag import minute_decision_cutoff DEFAULT_CACHE_ROOT = "artifacts/cache/tushare/v1" +#: The intraday cache's DECLARED earliest bar date (measured on the real cache: +#: several CSI500 names carry 1min bars from 2015-01-05). The pooled saturation +#: loop needs a declared floor; it must never infer one from row counts. +CACHE_MINUTE_DATA_START = "2015-01-05" DEFAULT_FACTORS = ( "jump_amount_corr_20", "minute_ideal_amp_10", @@ -62,6 +66,15 @@ def __init__(self, root: str) -> None: self.calls = 0 self.live_calls = 0 # provably 0 — read_range has no fetch closure + def earliest_available(self, symbols): + """The cache's DECLARED minute-data floor (measured: bars from 2015-01-05). + + Declared, never inferred from row counts — a long mid-history no-bar gap + (a suspension) is indistinguishable from exhaustion by row count, which + is exactly the unsound signal the pooled saturation loop refuses. + """ + return pd.Timestamp(CACHE_MINUTE_DATA_START) + def minute_bars(self, symbols, start, end): self.calls += 1 if not symbols: diff --git a/tests/test_factor_service.py b/tests/test_factor_service.py index 74b4af2..24b3028 100644 --- a/tests/test_factor_service.py +++ b/tests/test_factor_service.py @@ -105,6 +105,10 @@ def minute_bars(self, symbols, start, end): t = MINUTE.index.get_level_values("time") return MINUTE[(t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end))] + def earliest_available(self, symbols): + """DECLARED data start of this fixture (never inferred from row counts).""" + return DATES[0] + def _sparse_minute(): """Sparse-valid minute data: every 3rd day is thin (fewer bars) so the @@ -142,6 +146,72 @@ def minute_bars(self, symbols, start, end): t = SPARSE_MINUTE.index.get_level_values("time") return SPARSE_MINUTE[(t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end))] + def earliest_available(self, symbols): + return DATES[0] + + +#: Gap geometry (the review counterexample, sized to actually exercise it). +#: One expansion chunk is ``lookback_depth*2 + 40`` CALENDAR days ~= 85 trading +#: days at depth 40. The gap must be wide enough that a WHOLE expansion step +#: lands INSIDE it (adding no bars at all -> emit values unchanged -> a +#: value-stability fixed point terminates falsely) while REAL earlier history +#: still exists beyond it: 200 trading days spans more than two chunks. +_GAP_TRADING_DAYS = 200 +_PRE_GAP_DAYS = 60 +_POST_GAP_DAYS = 30 +_GAP_BARS_PER_DAY = 238 # a full session: the pooled factors need enough + # classifiable + ridge/valley bars to yield valid days +GAP_DATES = pd.bdate_range( + "2021-01-04", periods=_PRE_GAP_DAYS + _GAP_TRADING_DAYS + _POST_GAP_DAYS +) + + +def _gapped_minute(): + """The REVIEW COUNTEREXAMPLE shape: rich bars, then a LONG no-bar gap + (a suspension) wider than one expansion chunk, then rich bars again. + + The emit window sits AFTER the gap and its trailing valid-day pool must reach + back ACROSS the gap. A value-stability fixed point terminates falsely here + (one chunk lands entirely inside the gap -> emit values unchanged), and an + unchanged row count inside the gap looks exactly like the data start — the + two failure modes the structural criterion replaces. + """ + rng = np.random.RandomState(23) + rows = [] + pre = GAP_DATES[:_PRE_GAP_DAYS] + post = GAP_DATES[_PRE_GAP_DAYS + _GAP_TRADING_DAYS:] + for si, s in enumerate(SYMS): + for d in list(pre) + list(post): + base = pd.Timestamp(d) + pd.Timedelta("09:31:00") + price = 100.0 + si * 5 + rng.normal(0, 2) + for i in range(_GAP_BARS_PER_DAY): + t = base + pd.Timedelta(minutes=i) + price += rng.normal(0, 0.05) + slot = 1e4 * (1.0 + 0.3 * np.sin(i / 12.0)) + erupt = 6.0 if (rng.rand() < 0.06) else 1.0 + vol = slot * erupt * (1.0 + 0.1 * rng.rand()) + w = 0.15 * price * (1.0 + (2.0 if erupt > 1 else 0.0)) * (0.5 + rng.rand()) + hi, lo = price + abs(w) * rng.rand(), price - abs(w) * rng.rand() + cl = lo + (hi - lo) * rng.rand() + rows.append((t, s, price, hi, lo, cl, vol, cl * vol)) + frame = pd.DataFrame(rows, columns=["time", "symbol", "open", "high", "low", "close", "volume", "amount"]) + return normalize_intraday_bars(frame, freq="1min"), list(post) + + +GAPPED_MINUTE, GAPPED_POST_DATES = _gapped_minute() + + +class GappedMinuteProv: + def minute_bars(self, symbols, start, end): + if not symbols: + return GAPPED_MINUTE.iloc[0:0] + t = GAPPED_MINUTE.index.get_level_values("time") + return GAPPED_MINUTE[(t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end))] + + def earliest_available(self, symbols): + """DECLARED start — the mid-history gap must never be mistaken for it.""" + return GAP_DATES[0] + def _sources(): return MaterializeSources(daily=DailyProv(), minute=MinuteProv()) @@ -151,6 +221,10 @@ def _sparse_sources(): return MaterializeSources(minute=SparseMinuteProv()) +def _gapped_sources(): + return MaterializeSources(minute=GappedMinuteProv()) + + def _store_series(store, factor_id, view=View.DECISION): factor = factor_registry.build(factor_id) key = store_key(factor, view=view.value) @@ -267,6 +341,45 @@ def test_pooled_factor_single_equals_batch_on_sparse_valid_window(): assert np.array_equal(av[finite], bv[finite]), f"{fid}: finite values diverge" +def test_pooled_single_equals_batch_across_a_long_no_bar_gap(): + """THE REVIEW COUNTEREXAMPLE, committed: a no-bar gap WIDER than one expansion + chunk (95 trading days = a suspension), emit AFTER the gap, and the trailing + valid-day pool must reach back ACROSS it. + + This is what defeated the value-stability fixed point (a chunk landing wholly + inside the gap leaves the emit values unchanged -> false 'stable') and the + row-count data-start inference (an unchanged count inside the gap looks like + exhaustion). The structural criterion counts FINAL valid days in the locked + sub-window, so a gap contributes none, the count stalls, and expansion + continues to the declared floor. + + MUTATION EVIDENCE (run for this commit): + * reverting the criterion to the value-stability fixed point -> this test + FAILS (rc=1): both fills terminate inside the gap and emit all-NaN, caught + by the non-vacuity assertion below; restored -> passes (rc=0). + * dropping the baseline LOCKING OFFSET (``loaded_days[baseline_days]`` -> + ``loaded_days[0]``) does NOT fail on these fixtures — recorded honestly: + the offset is a correctness margin these fixtures do not isolate (the + valid-day count criterion alone already forces enough history here). It is + kept because the locking argument requires it in general; a fixture that + isolates it would need boundary days whose classification actually flips. + """ + dates = GAPPED_POST_DATES[10:22] # after the gap, pool still spans it + for fid in _POOLED_DIVERGENT: + with tempfile.TemporaryDirectory() as ta, tempfile.TemporaryDirectory() as tb: + a, b = FactorValueStore(ta), FactorValueStore(tb) + _fill_single(a, [fid], dates, _gapped_sources()) + _fill_batch(b, [fid], dates, _gapped_sources()) + sa = _store_series(a, fid).sort_index() + sb = _store_series(b, fid).sort_index() + assert sa.index.equals(sb.index), fid + av, bv = sa.to_numpy(), sb.to_numpy() + assert np.array_equal(np.isnan(av), np.isnan(bv)), f"{fid}: NaN mask diverges" + finite = ~np.isnan(av) + assert finite.sum() > 0, f"{fid}: vacuous (no finite values across the gap)" + assert np.allclose(av[finite], bv[finite], rtol=0.0, atol=1e-12), fid + + def test_disabling_saturation_reintroduces_pooled_divergence(monkeypatch): """MUTATION: treating a pooled factor as fixed-depth (saturation OFF) brings back the single-fill/batch-fill finite<->NaN divergence -> the saturation From f1934d9a910ab4dfb1066488e8278ef4b5b877a3 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Sat, 25 Jul 2026 06:05:44 -0700 Subject: [PATCH 8/9] fix(factors): pooled saturation must count REQUESTED symbols (D4 review HIGH) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The structural criterion iterated the symbols PRESENT IN THE OUTPUT, so a symbol producing ZERO rows in the current load window (a long suspension, thin coverage) had NO VETO over termination — and it is precisely the symbol a deeper load would unlock. The consequence was worse than a wrong value: a single-date fill dropped such a name from the cross-section entirely while a batch fill kept it. Measured on the review's 2-symbol fixture (A with bars throughout; B = an ancient block + a recent block with a long suspension between): volume_peak_count_20 single-fill ``600519.SH rows=0`` vs batch-fill ``rows=10 finite=6``; the criterion's per-symbol counts were ``{'600000.SH': 67}`` with 600519 absent entirely. Silent name-dropping is a sample-coverage bias the I5a red line forbids. Fix: iterate the REQUESTED universe; an absent symbol counts 0 valid days, vetoes termination, and the loop expands until it too saturates or the declared floor is reached. After the fix both fills give ``rows=50`` with identical symbol sets. COST, stated in the docstring rather than discovered later: any requested symbol that cannot accumulate ``lookback_days`` valid days before ``emit_start`` (recent listing, long suspension, coverage hole) drags the WHOLE universe's load to the declared floor. That cost already existed for symbols with SOME output but too few valid days; this only extends the same conservative behaviour to zero-output symbols. A per-symbol saturation start is the D5 optimization — and NOT a pure one: ``intraday_amp_cut`` z-scores across the loaded cross-section, so changing per-symbol load depth changes its cross-section composition (noted for D5). Also in this commit: * LOCKING-OFFSET isolation test (the review's construction): a sparse recent block of 33 every-other-day bar-days puts an UNLOCKED band (classifiable but not yet final) inside the criterion's margin. With the offset the loop expands and reproduces the whole-history truth; without it the criterion stops a chunk early and returns a silently wrong value. This replaces last commit's honest note that the offset was not isolated by the gap fixtures. * LOW: the gap test's docstring said "95 trading days" while the fixture uses ``_GAP_TRADING_DAYS`` = 200 (stale after the geometry correction, #76 shape) — now references the constant instead of restating a number. * HARDENING: exhausting ``_MAX_SATURATION_CHUNKS`` silently returned an UNSATURATED result; it now raises with the factor, the depth reached, the symbol count and the declared floor (red line #9 — no silent degradation). Unreachable in practice (500 chunks is >160 years), which is exactly why a silent path there was indefensible. MUTATION rc: restricting the criterion to output symbols FAILS the new test (rc=1, "600519.SH silently dropped by the single-date fill"), restored passes (rc=0); forcing ``baseline_days=0`` FAILS the locking-offset test (rc=1), restored passes (rc=0). --- factors/materialize.py | 51 ++++++-- tests/test_factor_service.py | 243 +++++++++++++++++++++++++++++++++-- 2 files changed, 272 insertions(+), 22 deletions(-) diff --git a/factors/materialize.py b/factors/materialize.py index 164d01f..e7d8a12 100644 --- a/factors/materialize.py +++ b/factors/materialize.py @@ -330,10 +330,20 @@ def _materialize_pooled( if at_floor or _pooled_pool_saturated( full, work, emit_start, baseline_days=baseline_days, lookback_days=lookback_days, + symbols=symbols, ): return last load_start = load_start - chunk - return last if last is not None else _empty_series(factor.name) + # Never silently return an UNSATURATED result (red line #9: no silent + # degradation). Unreachable in practice — 500 chunks is >160 years of history + # — so hitting it means the loop is not converging, which must be loud. + raise RuntimeError( + f"{factor.name}: pooled saturation did not converge after " + f"{_MAX_SATURATION_CHUNKS} expansion chunks (expanded back to " + f"{load_start.date()} for {len(symbols)} symbol(s), declared floor " + f"{floor.date()}). Refusing to return an unsaturated value, which would " + f"depend on load geometry." + ) def _provider_earliest(provider, symbols, factor) -> pd.Timestamp: @@ -358,18 +368,37 @@ def _pooled_pool_saturated( *, baseline_days: int, lookback_days: int, + symbols: list[str], ) -> bool: """The structural criterion (see :func:`_materialize_pooled` for the argument). - True iff, for EVERY symbol present, the LOCKED sub-window (the loaded trading - days after dropping the first ``baseline_days``) holds at least + True iff, for EVERY REQUESTED symbol, the LOCKED sub-window (the loaded + trading days after dropping the first ``baseline_days``) holds at least ``lookback_days`` valid days at or before ``emit_start`` — valid days being - the factor's own output dates. A symbol whose whole history is loaded shorter - than that is honestly under-warmed and is handled by the floor terminal, not - here. + the factor's own output dates. + + ITERATING THE REQUESTED SYMBOLS, NOT THE OUTPUT SYMBOLS, IS LOAD-BEARING + (review HIGH): a symbol that produces ZERO rows in the current load window (a + long suspension, thin coverage) is absent from the output, so iterating the + output would give it NO VETO — yet it is exactly the symbol that a deeper load + would unlock. The consequence was worse than a wrong value: a single-date fill + dropped such a name from the cross-section entirely while a batch fill kept it + (measured: volume_peak_count_20 single ``rows=0`` vs batch ``rows=10``), which + is the silent name-dropping the I5a red line forbids (a missing symbol is a + sample-coverage bias, and must be loud). Counting an absent symbol as 0 valid + days vetoes termination, so the loop expands until it too is saturated or the + declared floor is reached. + + COST, stated plainly: any requested symbol that cannot accumulate + ``lookback_days`` valid days before ``emit_start`` (a recent listing, a long + suspension, a coverage hole) drags the WHOLE universe's load back to the + declared floor. That cost already existed for symbols with SOME output but too + few valid days; this fix only extends the same conservative behaviour to + zero-output symbols. A per-symbol saturation start is the D5 optimization — + and NOT a pure one: ``intraday_amp_cut`` z-scores across the loaded + cross-section, so changing per-symbol load depth changes its cross-section + composition. """ - if full.empty: - return False loaded_days = pd.DatetimeIndex( pd.unique(pd.DatetimeIndex(bars.index.get_level_values("time")).normalize()) ).sort_values() @@ -378,9 +407,9 @@ def _pooled_pool_saturated( locked_from = loaded_days[baseline_days] # first day with a FINAL classification out_dates = full.index.get_level_values(DATE_LEVEL) - out_symbols = full.index.get_level_values(SYMBOL_LEVEL) - for symbol in pd.unique(out_symbols): - mine = out_dates[(out_symbols == symbol)] + out_symbols = pd.Index(full.index.get_level_values(SYMBOL_LEVEL)) + for symbol in symbols: # the REQUESTED universe, not just what produced rows + mine = out_dates[out_symbols == str(symbol)] final_valid = mine[(mine >= locked_from) & (mine <= emit_start)] if len(final_valid) < lookback_days: return False diff --git a/tests/test_factor_service.py b/tests/test_factor_service.py index 24b3028..1e76d6f 100644 --- a/tests/test_factor_service.py +++ b/tests/test_factor_service.py @@ -231,13 +231,15 @@ def _store_series(store, factor_id, view=View.DECISION): return store.read(key) -def _fill_single(store, factor_ids, dates, src): +def _fill_single(store, factor_ids, dates, src, universe=None): for d in dates: - cross_section(factor_ids, SYMS, DecisionPoint(date=d), store=store, sources=src) + cross_section(factor_ids, universe or SYMS, DecisionPoint(date=d), + store=store, sources=src) -def _fill_batch(store, factor_ids, dates, src): - panel(factor_ids, SYMS, [DecisionPoint(date=d) for d in dates], store=store, sources=src) +def _fill_batch(store, factor_ids, dates, src, universe=None): + panel(factor_ids, universe or SYMS, [DecisionPoint(date=d) for d in dates], + store=store, sources=src) # --------------------------------------------------------------------------- # @@ -343,7 +345,7 @@ def test_pooled_factor_single_equals_batch_on_sparse_valid_window(): def test_pooled_single_equals_batch_across_a_long_no_bar_gap(): """THE REVIEW COUNTEREXAMPLE, committed: a no-bar gap WIDER than one expansion - chunk (95 trading days = a suspension), emit AFTER the gap, and the trailing + chunk (``_GAP_TRADING_DAYS`` = a suspension), emit AFTER the gap, and the trailing valid-day pool must reach back ACROSS it. This is what defeated the value-stability fixed point (a chunk landing wholly @@ -357,12 +359,10 @@ def test_pooled_single_equals_batch_across_a_long_no_bar_gap(): * reverting the criterion to the value-stability fixed point -> this test FAILS (rc=1): both fills terminate inside the gap and emit all-NaN, caught by the non-vacuity assertion below; restored -> passes (rc=0). - * dropping the baseline LOCKING OFFSET (``loaded_days[baseline_days]`` -> - ``loaded_days[0]``) does NOT fail on these fixtures — recorded honestly: - the offset is a correctness margin these fixtures do not isolate (the - valid-day count criterion alone already forces enough history here). It is - kept because the locking argument requires it in general; a fixture that - isolates it would need boundary days whose classification actually flips. + * dropping the baseline LOCKING OFFSET does NOT fail on THIS fixture (the + valid-day count alone already forces enough history here); it IS isolated + by ``test_dropping_the_locking_offset_yields_a_wrong_value``, whose sparse + recent block puts an unlocked band inside the criterion's margin. """ dates = GAPPED_POST_DATES[10:22] # after the gap, pool still spans it for fid in _POOLED_DIVERGENT: @@ -380,6 +380,227 @@ def test_pooled_single_equals_batch_across_a_long_no_bar_gap(): assert np.allclose(av[finite], bv[finite], rtol=0.0, atol=1e-12), fid +#: Zero-output fixture (review HIGH): symbol A has bars throughout; symbol B has +#: an ancient block and a recent block with a LONG suspension between, so in a +#: shallow load window B produces ZERO factor rows. +_ZO_DATES = pd.bdate_range("2021-01-04", periods=130) +_ZO_A, _ZO_B = "600000.SH", "600519.SH" +_ZO_SYMS = [_ZO_A, _ZO_B] + + +def _zero_output_minute(): + rng = np.random.RandomState(99) + rows = [] + plan = ((_ZO_A, list(range(130))), (_ZO_B, list(range(10, 25)) + list(range(120, 130)))) + for si, (s, days) in enumerate(plan): + for di in days: + base = pd.Timestamp(_ZO_DATES[di]) + pd.Timedelta("09:31:00") + price = 100.0 + si * 5 + rng.normal(0, 2) + for i in range(238): + t = base + pd.Timedelta(minutes=i) + price += rng.normal(0, 0.05) + slot = 1e4 * (1.0 + 0.3 * np.sin(i / 12.0)) + erupt = 6.0 if (rng.rand() < 0.06) else 1.0 + vol = slot * erupt * (1.0 + 0.1 * rng.rand()) + w = 0.15 * price * (1.0 + (2.0 if erupt > 1 else 0.0)) * (0.5 + rng.rand()) + hi, lo = price + abs(w) * rng.rand(), price - abs(w) * rng.rand() + cl = lo + (hi - lo) * rng.rand() + rows.append((t, s, price, hi, lo, cl, vol, cl * vol)) + frame = pd.DataFrame(rows, columns=["time", "symbol", "open", "high", "low", "close", "volume", "amount"]) + return normalize_intraday_bars(frame, freq="1min") + + +ZERO_OUTPUT_MINUTE = _zero_output_minute() + + +class ZeroOutputMinuteProv: + def minute_bars(self, symbols, start, end): + if not symbols: + return ZERO_OUTPUT_MINUTE.iloc[0:0] + t = ZERO_OUTPUT_MINUTE.index.get_level_values("time") + sym = ZERO_OUTPUT_MINUTE.index.get_level_values("symbol") + keep = (t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end)) & sym.isin(list(symbols)) + return ZERO_OUTPUT_MINUTE[keep] + + def earliest_available(self, symbols): + return _ZO_DATES[0] + + +def test_zero_output_symbol_is_not_silently_dropped(): + """REVIEW HIGH: a symbol producing ZERO rows in the current load window must + still VETO saturation — otherwise a single-date fill drops it from the + cross-section entirely while a batch fill keeps it (silent name-dropping = + sample-coverage bias, forbidden by the I5a red line). + + Measured before the fix: volume_peak_count_20 single-fill ``600519.SH rows=0`` + vs batch-fill ``rows=10 finite=6`` — the criterion's per-symbol counts were + ``{'600000.SH': 67}`` with 600519 absent entirely. After the fix both fills + carry the same symbols and the same rows. + """ + dates = list(_ZO_DATES[90:130]) # the review's emit window (indices 90..129) + fid = "volume_peak_count_20" + with tempfile.TemporaryDirectory() as ta, tempfile.TemporaryDirectory() as tb: + a, b = FactorValueStore(ta), FactorValueStore(tb) + _fill_single(a, [fid], dates, MaterializeSources(minute=ZeroOutputMinuteProv()), + universe=_ZO_SYMS) + _fill_batch(b, [fid], dates, MaterializeSources(minute=ZeroOutputMinuteProv()), + universe=_ZO_SYMS) + sa = _store_series(a, fid).sort_index() + sb = _store_series(b, fid).sort_index() + # the thin symbol must be present in BOTH stores (never silently dropped) + syms_a = set(map(str, sa.index.get_level_values("symbol"))) + syms_b = set(map(str, sb.index.get_level_values("symbol"))) + assert _ZO_B in syms_a, f"{_ZO_B} silently dropped by the single-date fill" + assert syms_a == syms_b, f"symbol sets diverge: single={syms_a} batch={syms_b}" + assert sa.index.equals(sb.index) + av, bv = sa.to_numpy(), sb.to_numpy() + assert np.array_equal(np.isnan(av), np.isnan(bv)) + finite = ~np.isnan(av) + assert finite.sum() > 0 + assert np.allclose(av[finite], bv[finite], rtol=0.0, atol=1e-12) + + +def test_zero_output_symbol_veto_mutation(monkeypatch): + """MUTATION for the HIGH: restricting the criterion to OUTPUT symbols (the + pre-fix behaviour) reintroduces the silent drop -> this test's assertion of + the drop proves the requested-symbol iteration is load-bearing.""" + import factors.materialize as mat + + real = mat._pooled_pool_saturated + + def output_only(full, bars, emit_start, *, baseline_days, lookback_days, symbols): + present = [s for s in symbols if str(s) in set(map(str, full.index.get_level_values("symbol")))] + return real(full, bars, emit_start, baseline_days=baseline_days, + lookback_days=lookback_days, symbols=present) + + monkeypatch.setattr(mat, "_pooled_pool_saturated", output_only) + dates = list(_ZO_DATES[90:130]) # the review's emit window (indices 90..129) + fid = "volume_peak_count_20" + with tempfile.TemporaryDirectory() as ta, tempfile.TemporaryDirectory() as tb: + a, b = FactorValueStore(ta), FactorValueStore(tb) + _fill_single(a, [fid], dates, MaterializeSources(minute=ZeroOutputMinuteProv()), + universe=_ZO_SYMS) + _fill_batch(b, [fid], dates, MaterializeSources(minute=ZeroOutputMinuteProv()), + universe=_ZO_SYMS) + sa, sb = _store_series(a, fid), _store_series(b, fid) + syms_a = set(map(str, sa.index.get_level_values("symbol"))) if sa is not None else set() + syms_b = set(map(str, sb.index.get_level_values("symbol"))) if sb is not None else set() + assert syms_a != syms_b or _ZO_B not in syms_a, ( + "the output-symbols-only criterion should drop the thin symbol from " + "the single-date fill" + ) + + +#: Locking-offset isolation fixture (review task 3(b), adopted verbatim in shape). +_LO_DATES = pd.bdate_range("2020-01-01", periods=210) +_LO_SYM = "600000.SH" +_LO_E_IDX = 200 +#: deep history (reachable only by a second expansion) + a SPARSE recent block of +#: 33 every-other-day bar-days ending exactly at the emit date. In the shallow +#: window the sparse block is all there is, and its local days 10..19 are +#: classifiable but NOT yet final (baseline below full depth) — the UNLOCKED band +#: the offset exists to exclude. +_LO_BAR_DAYS = list(range(40, 100)) + [_LO_E_IDX - 2 * j for j in range(32, -1, -1)] + + +def _locking_offset_minute(): + rng = np.random.RandomState(2026) + rows = [] + for di in _LO_BAR_DAYS: + base = pd.Timestamp(_LO_DATES[di]) + pd.Timedelta("09:31:00") + price = 100.0 + rng.normal(0, 2) + for i in range(238): + t = base + pd.Timedelta(minutes=i) + price += rng.normal(0, 0.05) + slot = 1e4 * (1.0 + 0.3 * np.sin(i / 12.0)) + erupt = 6.0 if (rng.rand() < 0.06) else 1.0 + vol = slot * erupt * (1.0 + 0.1 * rng.rand()) + w = 0.15 * price * (1.0 + (2.0 if erupt > 1 else 0.0)) * (0.5 + rng.rand()) + hi, lo = price + abs(w) * rng.rand(), price - abs(w) * rng.rand() + cl = lo + (hi - lo) * rng.rand() + rows.append((t, _LO_SYM, price, hi, lo, cl, vol, cl * vol)) + frame = pd.DataFrame(rows, columns=["time", "symbol", "open", "high", "low", "close", "volume", "amount"]) + return normalize_intraday_bars(frame, freq="1min") + + +LOCKING_OFFSET_MINUTE = _locking_offset_minute() + + +class LockingOffsetProv: + def minute_bars(self, symbols, start, end): + if not symbols: + return LOCKING_OFFSET_MINUTE.iloc[0:0] + t = LOCKING_OFFSET_MINUTE.index.get_level_values("time") + return LOCKING_OFFSET_MINUTE[(t >= pd.Timestamp(start)) & (t <= pd.Timestamp(end))] + + def earliest_available(self, symbols): + return _LO_DATES[0] + + +def test_locking_offset_is_load_bearing(): + """The BASELINE LOCKING OFFSET produces the load-geometry-free value. + + Ground truth = computing on the WHOLE history at once. With the offset the + saturation loop expands past the unlocked band and reproduces it exactly; + dropping the offset (``loaded_days[baseline_days]`` -> ``loaded_days[0]``) + lets the criterion stop one chunk early with the pool still drawing on days + whose classification is not final, yielding a silently WRONG value. + + MUTATION rc: with ``baseline_days=0`` forced this test FAILS (rc=1, the + mutated value != truth); unmutated it passes (rc=0). This replaces the + earlier honest note that the offset was not isolated by the gap fixtures. + """ + import factors.materialize as mat + from factors.compute.minute.binding import minute_raw_from_bars + from factors.view_lag import minute_decision_cutoff + + factor = factor_registry.build("volume_peak_count_20") + emit = _LO_DATES[_LO_E_IDX] + truth_full = minute_raw_from_bars( + factor, minute_decision_cutoff(LOCKING_OFFSET_MINUTE, decision_time="14:50:00") + ) + truth = float(truth_full.loc[(emit, _LO_SYM)]) + + got = mat.materialize_range( + factor, view=View.DECISION, symbols=[_LO_SYM], emit_start=emit, emit_end=emit, + sources=MaterializeSources(minute=LockingOffsetProv()), + ) + assert float(got.iloc[0]) == truth, ( + "the saturated value must equal the whole-history (load-geometry-free) value" + ) + + +def test_dropping_the_locking_offset_yields_a_wrong_value(monkeypatch): + """MUTATION evidence for the offset: forcing ``baseline_days=0`` makes the + criterion accept the unlocked band and return a value that differs from the + whole-history truth (measured on this fixture: 438.0 vs the true 416.0).""" + import factors.materialize as mat + from factors.compute.minute.binding import minute_raw_from_bars + from factors.view_lag import minute_decision_cutoff + + factor = factor_registry.build("volume_peak_count_20") + emit = _LO_DATES[_LO_E_IDX] + truth = float( + minute_raw_from_bars( + factor, minute_decision_cutoff(LOCKING_OFFSET_MINUTE, decision_time="14:50:00") + ).loc[(emit, _LO_SYM)] + ) + real = mat._pooled_pool_saturated + + def no_offset(full, bars, emit_start, *, baseline_days, lookback_days, symbols): + return real(full, bars, emit_start, baseline_days=0, + lookback_days=lookback_days, symbols=symbols) + + monkeypatch.setattr(mat, "_pooled_pool_saturated", no_offset) + mutated = mat.materialize_range( + factor, view=View.DECISION, symbols=[_LO_SYM], emit_start=emit, emit_end=emit, + sources=MaterializeSources(minute=LockingOffsetProv()), + ) + assert float(mutated.iloc[0]) != truth, ( + "dropping the locking offset must produce a load-geometry-dependent value" + ) + + def test_disabling_saturation_reintroduces_pooled_divergence(monkeypatch): """MUTATION: treating a pooled factor as fixed-depth (saturation OFF) brings back the single-fill/batch-fill finite<->NaN divergence -> the saturation From decbdc9f554ab04cd0f8b82237d0a70ba8a07813 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Sat, 25 Jul 2026 06:20:50 -0700 Subject: [PATCH 9/9] test(factors): cover the saturation-exhaustion raise + de-vacuify the veto mutation (D4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review NITs, test-file only (materialize.py untouched). NIT 1 — the ``_MAX_SATURATION_CHUNKS`` exhaustion RuntimeError had no test. It is unreachable in practice (500 chunks is >160 years), which is exactly why it needs one: a raise that never executes is the next AttributeError's host. Recipe: cap the budget at 2 chunks, use the thin symbol that never accumulates lookback_days valid days, and declare a floor too distant to reach — so neither termination condition fires and the loop must fall through to the raise. A second test asserts the message is actionable (names the factor, the declared floor, and the chunk count) so an operator can tell WHICH factor stalled and how deep it looked. NIT 2 — the veto mutation test's disjunction ``syms_a != syms_b or _ZO_B not in syms_a`` is trivially true when BOTH stores are empty, so it could pass without exercising the mutation. Added non-vacuity guards: both stores must be non-empty and the batch fill must still carry the thin symbol before the contrast is read. MUTATION rc: reverting the raise to a silent return FAILS both new tests (rc=1, "DID NOT RAISE"); restored passes (rc=0). --- tests/test_factor_service.py | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/test_factor_service.py b/tests/test_factor_service.py index 1e76d6f..5c76102 100644 --- a/tests/test_factor_service.py +++ b/tests/test_factor_service.py @@ -485,12 +485,72 @@ def output_only(full, bars, emit_start, *, baseline_days, lookback_days, symbols sa, sb = _store_series(a, fid), _store_series(b, fid) syms_a = set(map(str, sa.index.get_level_values("symbol"))) if sa is not None else set() syms_b = set(map(str, sb.index.get_level_values("symbol"))) if sb is not None else set() + # NON-VACUITY GUARD: the disjunction below is trivially true when BOTH + # stores are empty (nothing filled at all), which would make this mutation + # test pass without exercising the mutation. Require real content first. + assert syms_a, "single-fill store is empty — the mutation test would be vacuous" + assert syms_b, "batch-fill store is empty — the mutation test would be vacuous" + assert _ZO_B in syms_b, ( + "the batch fill must still carry the thin symbol; otherwise the " + "single-vs-batch contrast below proves nothing" + ) assert syms_a != syms_b or _ZO_B not in syms_a, ( "the output-symbols-only criterion should drop the thin symbol from " "the single-date fill" ) +class DistantFloorMinuteProv(ZeroOutputMinuteProv): + """Same bars, but a DECLARED floor far in the past — so the saturation loop + cannot terminate by reaching the floor within a bounded number of chunks.""" + + def earliest_available(self, symbols): + return pd.Timestamp("1990-01-01") + + +def test_saturation_exhaustion_raises_loudly(monkeypatch): + """Exhausting the chunk budget must RAISE, never return an unsaturated value. + + Red line #9 (no silent degradation): the branch is unreachable in practice + (500 chunks is >160 years), which is exactly why it needs a test — a raise + that never executes is the next AttributeError's host. Recipe: cap the budget + at 2 chunks, use a symbol that never accumulates ``lookback_days`` valid days, + and declare a floor too distant to reach — so neither termination condition + fires and the loop must fall through to the raise. + """ + import factors.materialize as mat + + monkeypatch.setattr(mat, "_MAX_SATURATION_CHUNKS", 2) + factor = factor_registry.build("volume_peak_count_20") + emit = _ZO_DATES[128] + with pytest.raises(RuntimeError, match="pooled saturation did not converge"): + mat.materialize_range( + factor, view=View.DECISION, symbols=_ZO_SYMS, + emit_start=emit, emit_end=emit, + sources=MaterializeSources(minute=DistantFloorMinuteProv()), + ) + + +def test_saturation_exhaustion_message_is_actionable(monkeypatch): + """The raise names the factor and the declared floor (an operator must be able + to tell WHICH factor stalled and how deep the search went).""" + import factors.materialize as mat + + monkeypatch.setattr(mat, "_MAX_SATURATION_CHUNKS", 2) + factor = factor_registry.build("volume_peak_count_20") + emit = _ZO_DATES[128] + with pytest.raises(RuntimeError) as excinfo: + mat.materialize_range( + factor, view=View.DECISION, symbols=_ZO_SYMS, + emit_start=emit, emit_end=emit, + sources=MaterializeSources(minute=DistantFloorMinuteProv()), + ) + message = str(excinfo.value) + assert factor.name in message # which factor stalled + assert "1990-01-01" in message # the declared floor + assert "expansion chunks" in message # how far the search went + + #: Locking-offset isolation fixture (review task 3(b), adopted verbatim in shape). _LO_DATES = pd.bdate_range("2020-01-01", periods=210) _LO_SYM = "600000.SH"