From 7861fba0dd9f3f9bd089f5e8a73533d0c6845cde Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 01:55:10 -0700 Subject: [PATCH 1/8] feat(factors): add factors/ops time-series operators and rewrite daily factor rolling on them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D2 commit A (design v3.2 §3.2): factors/ops/timeseries.py is the one home of the per-symbol lag/rolling machinery the daily factors used to inline. Three conventions locked by tests: strictly per-symbol grouping, full-window leading-NaN semantics, purity. Only the operators the rewrite consumes exist (ts_lag / ts_window_return / ts_pct_change / ts_std / ts_mean / ts_sum / log_positive) — no speculative cs_rank/median, no string-dispatch registry. momentum / volatility / liquidity / overnight_mom computes now call the ops; reversal still delegates to momentum. Definitions unchanged: every rewritten factor is locked bit-identical to an independent naive oracle in tests/test_factors_ops.py, and the pre-existing factor tests pass unchanged. pytest 1917 passed (baseline 1902 + 15), ruff clean. --- factors/compute/candidates.py | 35 +++-- factors/compute/momentum.py | 11 +- factors/ops/__init__.py | 27 ++++ factors/ops/timeseries.py | 106 +++++++++++++++ tests/test_factors_ops.py | 235 ++++++++++++++++++++++++++++++++++ 5 files changed, 389 insertions(+), 25 deletions(-) create mode 100644 factors/ops/__init__.py create mode 100644 factors/ops/timeseries.py create mode 100644 tests/test_factors_ops.py diff --git a/factors/compute/candidates.py b/factors/compute/candidates.py index 0c5244e..7770e6d 100644 --- a/factors/compute/candidates.py +++ b/factors/compute/candidates.py @@ -30,6 +30,7 @@ from data.availability_policy import DAILY_BASIC, MARKET_DAILY from factors.base import Factor from factors.compute.momentum import MomentumFactor +from factors.ops import log_positive, ts_lag, ts_mean, ts_pct_change, ts_std, ts_sum from factors.spec import FactorSpec, PanelField # daily_basic-derived value fields the pipeline can enrich + surface (P3-5). @@ -170,14 +171,10 @@ def compute(self, panel: pd.DataFrame) -> pd.Series: f"{list(panel.columns)}." ) price = panel[self._price_col] - grouped = price.groupby(level="symbol", group_keys=False) - # pct_change within each symbol (never across); rolling std needs a FULL - # window of returns -> leading rows are NaN, all inputs are <= t. - vol = grouped.apply( - lambda s: s.pct_change().rolling( - self._window, min_periods=self._window - ).std(ddof=1) - ) + # D2 (factors.ops): pct_change within each symbol (never across); the + # rolling std needs a FULL window of returns -> leading rows are NaN, + # all inputs are <= t (ops conventions #1/#2). + vol = ts_std(ts_pct_change(price), self._window) return vol.reindex(panel.index).rename(self.name) @@ -245,13 +242,11 @@ def compute(self, panel: pd.DataFrame) -> pd.Series: f"amount from the bar feed); panel has {list(panel.columns)}." ) amount = panel[self._amount_col] - grouped = amount.groupby(level="symbol", group_keys=False) - mean_amt = grouped.apply( - lambda s: s.rolling(self._window, min_periods=self._window).mean() - ) - # log of a non-positive mean is undefined -> NaN (degenerate liquidity). - safe = mean_amt.where(mean_amt > 0) - return np.log(safe).reindex(panel.index).rename(self.name) + # D2 (factors.ops): per-symbol full-window rolling mean; the log of a + # non-positive mean is undefined -> NaN (degenerate liquidity), never a + # silent ``-inf`` (``log_positive``). + mean_amt = ts_mean(amount, self._window) + return log_positive(mean_amt).reindex(panel.index).rename(self.name) class OvernightMomentumFactor(Factor): @@ -340,13 +335,13 @@ def compute(self, panel: pd.DataFrame) -> pd.Series: ) open_px = panel[self._open_col] close_px = panel[self._close_col] - # strictly-lagged close within each symbol; never crosses symbols. - prev_close = close_px.groupby(level="symbol").shift(1) + # D2 (factors.ops): strictly-lagged close within each symbol (``ts_lag`` + # never crosses symbols); the full-window rolling sum keeps the leading + # window NaN (ops convention #2). + prev_close = ts_lag(close_px, 1) ratio = (open_px / prev_close).where((open_px > 0) & (prev_close > 0)) overnight = np.log(ratio) - mom = overnight.groupby(level="symbol", group_keys=False).apply( - lambda s: s.rolling(self._window, min_periods=self._window).sum() - ) + mom = ts_sum(overnight, self._window) return mom.reindex(panel.index).rename(self.name) diff --git a/factors/compute/momentum.py b/factors/compute/momentum.py index 00de5f6..208f603 100644 --- a/factors/compute/momentum.py +++ b/factors/compute/momentum.py @@ -23,6 +23,7 @@ from data.availability_policy import MARKET_DAILY from factors.base import Factor +from factors.ops import ts_window_return from factors.spec import FactorSpec, PanelField @@ -117,11 +118,11 @@ def compute(self, panel: pd.DataFrame) -> pd.Series: ) price = panel[self._price_col] - # Group on the symbol index level so the ratio never crosses symbols. - # ``shift(window)`` looks strictly backward within each symbol, so the - # value at t uses only bars at t and t-window (both <= t): no lookahead. - prev = price.groupby(level="symbol").shift(self._window) - momentum = price / prev - 1.0 + # D2: the per-symbol strictly-backward ratio lives in factors.ops — + # ``ts_window_return`` groups on the symbol level and lags ``window`` + # rows, so the value at t uses only bars at t and t-window (both <= t): + # no lookahead, no cross-symbol leakage (ops convention #1). + momentum = ts_window_return(price, self._window) # Preserve the exact panel index/order; name it for the factor panel. return momentum.reindex(panel.index).rename(self.name) diff --git a/factors/ops/__init__.py b/factors/ops/__init__.py new file mode 100644 index 0000000..1680123 --- /dev/null +++ b/factors/ops/__init__.py @@ -0,0 +1,27 @@ +"""``factors.ops``: the daily-panel time-series operator library (D2, §3.2). + +Pure functions over ``MultiIndex(date, symbol)`` Series with ONE shared +convention set (see :mod:`factors.ops.timeseries`): strictly per-symbol +grouping, full-window leading-NaN semantics, plain Python imports (no string +dispatch, no operator registry — design §3.2 "明确不做"). +""" + +from factors.ops.timeseries import ( + log_positive, + ts_lag, + ts_mean, + ts_pct_change, + ts_std, + ts_sum, + ts_window_return, +) + +__all__ = [ + "log_positive", + "ts_lag", + "ts_mean", + "ts_pct_change", + "ts_std", + "ts_sum", + "ts_window_return", +] diff --git a/factors/ops/timeseries.py b/factors/ops/timeseries.py new file mode 100644 index 0000000..2ddee35 --- /dev/null +++ b/factors/ops/timeseries.py @@ -0,0 +1,106 @@ +"""Per-symbol time-series operators for daily ``MultiIndex(date, symbol)`` panels. + +D2 (design v3.2 §3.2): the ONE home of the rolling/lag machinery the daily +factors used to inline. Three conventions, shared by every operator and locked +by tests so no factor can silently deviate: + +1. **Per-symbol grouping.** Every operator groups on the ``symbol`` index level + before any temporal step, so one symbol's history can never leak into + another's value (red line: cross-symbol isolation). +2. **Leading-NaN semantics.** Window operators default ``min_periods`` to the + FULL window: a warm-up row is an honest NaN, never a partial-window estimate + that silently changes meaning across the panel head. +3. **Purity.** Operators never mutate their input and never reindex/reorder it + beyond what the per-symbol groupby implies; callers align the result to + their panel (``reindex``) themselves, exactly as the pre-D2 factor code did. + +Scope discipline (§六.12, 不过度设计): only the operators the D2 daily-factor +rewrite actually consumes exist here. Speculative operators (``cs_rank``, +``rolling_median``, …) are added when a factor needs them, not before. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.clean.schema import SYMBOL_LEVEL + + +def _grouped(series: pd.Series): + """The shared per-symbol groupby (convention #1).""" + return series.groupby(level=SYMBOL_LEVEL, group_keys=False) + + +def ts_lag(series: pd.Series, periods: int = 1) -> pd.Series: + """Per-symbol backward shift: value at ``t`` becomes the value at ``t - periods``. + + Strictly backward for ``periods > 0`` (the only direction a factor may look); + the leading ``periods`` rows of every symbol are NaN. + """ + if not isinstance(periods, int) or periods < 1: + raise ValueError(f"ts_lag periods must be a positive integer; got {periods!r}.") + return series.groupby(level=SYMBOL_LEVEL).shift(periods) + + +def ts_window_return(series: pd.Series, window: int) -> pd.Series: + """``series[t] / series[t - window] - 1`` per symbol (the momentum ratio). + + The leading ``window`` rows of every symbol are NaN (no full denominator). + """ + if not isinstance(window, int) or window < 1: + raise ValueError( + f"ts_window_return window must be a positive integer; got {window!r}." + ) + return series / ts_lag(series, window) - 1.0 + + +def ts_pct_change(series: pd.Series) -> pd.Series: + """Per-symbol one-step simple return; each symbol's first row is NaN.""" + return _grouped(series).apply(lambda s: s.pct_change()) + + +def ts_std(series: pd.Series, window: int, *, min_periods: int | None = None) -> pd.Series: + """Per-symbol rolling std (ddof=1) over ``window`` rows; full window by default.""" + if not isinstance(window, int) or window < 2: + raise ValueError(f"ts_std window must be an integer >= 2; got {window!r}.") + mp = window if min_periods is None else min_periods + return _grouped(series).apply( + lambda s: s.rolling(window, min_periods=mp).std(ddof=1) + ) + + +def ts_mean(series: pd.Series, window: int, *, min_periods: int | None = None) -> pd.Series: + """Per-symbol rolling mean over ``window`` rows; full window by default.""" + if not isinstance(window, int) or window < 1: + raise ValueError(f"ts_mean window must be a positive integer; got {window!r}.") + mp = window if min_periods is None else min_periods + return _grouped(series).apply( + lambda s: s.rolling(window, min_periods=mp).mean() + ) + + +def ts_sum(series: pd.Series, window: int, *, min_periods: int | None = None) -> pd.Series: + """Per-symbol rolling sum over ``window`` rows; full window by default.""" + if not isinstance(window, int) or window < 1: + raise ValueError(f"ts_sum window must be a positive integer; got {window!r}.") + mp = window if min_periods is None else min_periods + return _grouped(series).apply( + lambda s: s.rolling(window, min_periods=mp).sum() + ) + + +def log_positive(series: pd.Series) -> pd.Series: + """``log(series)`` where ``series > 0``; anything else is NaN, never ``-inf``.""" + return np.log(series.where(series > 0)) + + +__all__ = [ + "log_positive", + "ts_lag", + "ts_mean", + "ts_pct_change", + "ts_std", + "ts_sum", + "ts_window_return", +] diff --git a/tests/test_factors_ops.py b/tests/test_factors_ops.py new file mode 100644 index 0000000..734bb00 --- /dev/null +++ b/tests/test_factors_ops.py @@ -0,0 +1,235 @@ +"""``factors.ops`` operator library tests (D2). + +Locks the three shared conventions (per-symbol grouping / full-window leading +NaN / purity) AND the daily-factor rewrite equivalence: every factor that moved +its inline rolling onto ``factors.ops`` must produce bit-identical values to a +naive per-symbol loop written independently here (plain pandas arithmetic, no +ops import in the oracle). + +Mutation-evidence notes (design §五.1: every invariance claim must be breakable): + +* the leading-NaN tests fail if an operator's ``min_periods`` default is relaxed + to 1 (run: edit ``ts_std`` to ``min_periods=1`` -> test_ts_std_full_window_ + leading_nan fails); +* the isolation tests fail if the groupby level is dropped (run: replace the + per-symbol groupby in ``ts_lag`` with a bare ``shift`` -> test_ts_lag_never_ + crosses_symbols fails); +* the equivalence tests fail if any rewritten factor drifts from its pre-D2 + math (they ARE the semantic lock for commit A). +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from factors.compute.candidates import ( + LiquidityFactor, + OvernightMomentumFactor, + VolatilityFactor, +) +from factors.compute.momentum import MomentumFactor +from factors.ops import ( + log_positive, + ts_lag, + ts_mean, + ts_pct_change, + ts_std, + ts_sum, + ts_window_return, +) + + +def _panel(n_days: int = 30, symbols: tuple[str, ...] = ("AAA", "BBB")) -> pd.DataFrame: + """Deterministic MultiIndex(date, symbol) panel with close/open/amount.""" + rng = np.random.RandomState(7) + dates = pd.bdate_range("2024-01-02", periods=n_days) + index = pd.MultiIndex.from_product([dates, list(symbols)], names=["date", "symbol"]) + n = len(index) + close = pd.Series(100.0 + np.cumsum(rng.randn(n)) * 0.5, index=index) + return pd.DataFrame( + { + "close": close, + "open": close * (1.0 + rng.randn(n) * 0.001), + "amount": np.abs(rng.randn(n)) * 1e6 + 1e5, + } + ) + + +def _series(panel: pd.DataFrame, col: str = "close") -> pd.Series: + return panel[col] + + +# --------------------------------------------------------------------------- # +# ts_lag +# --------------------------------------------------------------------------- # +def test_ts_lag_is_strictly_backward_with_leading_nan(): + s = _series(_panel()) + lagged = ts_lag(s, 3) + for sym in ("AAA", "BBB"): + sub = s.xs(sym, level="symbol") + lag_sub = lagged.xs(sym, level="symbol") + assert lag_sub.iloc[:3].isna().all() + np.testing.assert_array_equal( + lag_sub.iloc[3:].to_numpy(), sub.iloc[:-3].to_numpy() + ) + + +def test_ts_lag_never_crosses_symbols(): + panel = _panel() + s = _series(panel) + base = ts_lag(s, 1).xs("BBB", level="symbol") + # Perturb the OTHER symbol violently; BBB's lag must be bit-identical. + poisoned = s.copy() + mask = poisoned.index.get_level_values("symbol") == "AAA" + poisoned[mask] = 1e12 + after = ts_lag(poisoned, 1).xs("BBB", level="symbol") + pd.testing.assert_series_equal(base, after) + + +def test_ts_lag_rejects_non_positive_periods(): + s = _series(_panel()) + with pytest.raises(ValueError, match="positive integer"): + ts_lag(s, 0) + + +# --------------------------------------------------------------------------- # +# ts_window_return / ts_pct_change +# --------------------------------------------------------------------------- # +def test_ts_window_return_matches_naive_per_symbol_loop(): + s = _series(_panel()) + out = ts_window_return(s, 5) + for sym in ("AAA", "BBB"): + sub = s.xs(sym, level="symbol") + naive = sub / sub.shift(5) - 1.0 + pd.testing.assert_series_equal( + out.xs(sym, level="symbol"), naive, check_names=False + ) + + +def test_ts_pct_change_first_row_per_symbol_is_nan(): + s = _series(_panel()) + out = ts_pct_change(s) + for sym in ("AAA", "BBB"): + sub = out.xs(sym, level="symbol") + assert np.isnan(sub.iloc[0]) + naive = s.xs(sym, level="symbol").pct_change() + pd.testing.assert_series_equal(sub, naive, check_names=False) + + +# --------------------------------------------------------------------------- # +# ts_std / ts_mean / ts_sum: full-window leading NaN + naive equivalence +# --------------------------------------------------------------------------- # +def test_ts_std_full_window_leading_nan(): + s = _series(_panel()) + out = ts_std(s, 10) + for sym in ("AAA", "BBB"): + sub = out.xs(sym, level="symbol") + assert sub.iloc[:9].isna().all() + assert sub.iloc[9:].notna().all() + + +def test_ts_std_matches_naive_ddof1(): + s = _series(_panel()) + out = ts_std(s, 10) + sub = s.xs("AAA", level="symbol") + naive = sub.rolling(10, min_periods=10).std(ddof=1) + pd.testing.assert_series_equal( + out.xs("AAA", level="symbol"), naive, check_names=False + ) + + +def test_ts_mean_and_sum_match_naive_full_window(): + s = _series(_panel()) + for op, agg in ((ts_mean, "mean"), (ts_sum, "sum")): + out = op(s, 7) + sub = s.xs("BBB", level="symbol") + naive = getattr(sub.rolling(7, min_periods=7), agg)() + pd.testing.assert_series_equal( + out.xs("BBB", level="symbol"), naive, check_names=False + ) + + +def test_window_ops_reject_degenerate_windows(): + s = _series(_panel()) + with pytest.raises(ValueError): + ts_std(s, 1) # a ddof=1 std needs >= 2 observations + with pytest.raises(ValueError): + ts_mean(s, 0) + with pytest.raises(ValueError): + ts_sum(s, -3) + with pytest.raises(ValueError): + ts_window_return(s, 0) + + +# --------------------------------------------------------------------------- # +# log_positive +# --------------------------------------------------------------------------- # +def test_log_positive_maps_non_positive_to_nan_never_inf(): + s = pd.Series([2.0, 0.0, -1.0, np.nan, 5.0]) + out = log_positive(s) + assert np.isclose(out.iloc[0], np.log(2.0)) + assert np.isnan(out.iloc[1]) and np.isnan(out.iloc[2]) and np.isnan(out.iloc[3]) + assert np.isclose(out.iloc[4], np.log(5.0)) + assert not np.isinf(out.to_numpy()).any() + + +# --------------------------------------------------------------------------- # +# purity: operators never mutate their input +# --------------------------------------------------------------------------- # +def test_ops_do_not_mutate_input(): + s = _series(_panel()) + frozen = s.copy(deep=True) + ts_lag(s, 2) + ts_window_return(s, 5) + ts_pct_change(s) + ts_std(s, 5) + ts_mean(s, 5) + ts_sum(s, 5) + log_positive(s) + pd.testing.assert_series_equal(s, frozen) + + +# --------------------------------------------------------------------------- # +# Daily-factor rewrite equivalence: class output == independent naive oracle +# --------------------------------------------------------------------------- # +def test_momentum_factor_matches_naive_oracle(): + panel = _panel() + out = MomentumFactor(window=5).compute(panel) + price = panel["close"] + naive = price / price.groupby(level="symbol").shift(5) - 1.0 + pd.testing.assert_series_equal(out, naive.rename("momentum_5")) + + +def test_volatility_factor_matches_naive_oracle(): + panel = _panel(n_days=40) + out = VolatilityFactor(window=10).compute(panel) + naive = panel["close"].groupby(level="symbol", group_keys=False).apply( + lambda s: s.pct_change().rolling(10, min_periods=10).std(ddof=1) + ) + pd.testing.assert_series_equal(out, naive.reindex(panel.index).rename("volatility_10")) + + +def test_liquidity_factor_matches_naive_oracle(): + panel = _panel(n_days=40) + out = LiquidityFactor(window=10).compute(panel) + mean_amt = panel["amount"].groupby(level="symbol", group_keys=False).apply( + lambda s: s.rolling(10, min_periods=10).mean() + ) + naive = np.log(mean_amt.where(mean_amt > 0)) + pd.testing.assert_series_equal(out, naive.reindex(panel.index).rename("liquidity_10")) + + +def test_overnight_momentum_factor_matches_naive_oracle(): + panel = _panel(n_days=40) + out = OvernightMomentumFactor(window=10).compute(panel) + open_px, close_px = panel["open"], panel["close"] + prev_close = close_px.groupby(level="symbol").shift(1) + ratio = (open_px / prev_close).where((open_px > 0) & (prev_close > 0)) + naive = np.log(ratio).groupby(level="symbol", group_keys=False).apply( + lambda s: s.rolling(10, min_periods=10).sum() + ) + pd.testing.assert_series_equal( + out, naive.reindex(panel.index).rename("overnight_mom_10") + ) From c196219650b56f709ca2a5d3f35bbdc230ed0e1d Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 02:32:00 -0700 Subject: [PATCH 2/8] feat(factors): migrate the 11 minute factors + MMP/jump onto factors/compute/minute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D2 commits B+C (design v3.2 §3.2 / §6.4 / R14): the single definition point of every minute factor's math moves to factors/compute/minute — one factor per file, each file carrying BOTH the compute_* math (rewritten on the shared primitives layer) and its D1 Factor surface class (spec declarations carried over unchanged). - factors/compute/minute/primitives.py: peak_mask_for_symbol + prepare_visible_minute_bars move house VERBATIM (they were already author-once; R1 — relocation, not repair), plus the shared PIT front half (visible_minute_frame), amplitude guard, positive-trade mask, valid-day rolling and pooled-trailing-reduce forms, per-symbol split, and the VOLUME_PRV_* classification constants. Per-factor definition parameters (min_valley_bars=20 / min_ridge_bars=10 / counted-AFTER-the-guard, ...) stay explicit parameters — parameterized, never unified. - data/clean/intraday_schema.py: DATE_LEVEL / DAILY_INDEX_NAMES / DEFAULT_DECISION_TIME / DEFAULT_SESSION_OPEN move here (single home both the data-layer generic core and factors.compute.minute can import without a cycle); intraday_aggregate re-exports them unchanged. - data/clean/intraday_aggregate.py: MIXED module per R14 — generic core (asof_daily_features / resample_intraday_bars / feature keys) keeps real code; the MMP and jump factor math migrated to factors.compute.minute.{mmp, jump_amount_corr} and is re-exported (qt/intraday_tail_framework.py:38 and the eval runners keep working with ZERO changes). - 10 pure data/clean factor modules + factors/compute/intraday_derived.py are now import-only re-export shims (deleted in D6d); locked by an AST purity test scoped EXACTLY to the fully-migratable set (R14) plus re-export identity tests (shim name IS the new object). - factors/registry/builtin.py imports the classes/constants from their new homes; dispatch semantics unchanged. - The 11 per-factor property-test files migrate their imports to the new modules (性质测试搬迁 — same properties, now proven directly on the new engine; the PR-M teeth tests keep their monkeypatch seams, now patching factors.compute.minute.peak_ridge_amount_ratio). Factor definitions and float operation order preserved verbatim; the D1 frozen-baseline cell-by-cell reconciliation follows in this branch. pytest 1941 passed (1917 + 24 shim tests), ruff clean, validate-config 31/31, phase0 anchor ic 0.9600 / annual 0.8408 unchanged. --- data/clean/intraday_aggregate.py | 386 +---- data/clean/intraday_amount_ratio.py | 426 +---- data/clean/intraday_amp_anomaly.py | 258 +-- data/clean/intraday_amp_cut.py | 385 +---- data/clean/intraday_amplitude.py | 202 +-- data/clean/intraday_peak_interval.py | 311 +--- data/clean/intraday_ridge_return.py | 375 +---- data/clean/intraday_schema.py | 15 + data/clean/intraday_valley_quantile.py | 566 +------ data/clean/intraday_valley_ridge_vwap.py | 380 +---- data/clean/intraday_valley_vwap.py | 306 +--- data/clean/intraday_volume_prv.py | 414 +---- factors/compute/intraday_derived.py | 1488 +---------------- factors/compute/minute/__init__.py | 8 + .../minute/amp_marginal_anomaly_vol.py | 356 ++++ factors/compute/minute/intraday_amp_cut.py | 484 ++++++ factors/compute/minute/jump_amount_corr.py | 287 ++++ .../compute/minute/minute_ideal_amplitude.py | 293 ++++ factors/compute/minute/mmp.py | 194 +++ .../compute/minute/peak_interval_kurtosis.py | 429 +++++ .../compute/minute/peak_ridge_amount_ratio.py | 560 +++++++ factors/compute/minute/primitives.py | 405 +++++ factors/compute/minute/ridge_minute_return.py | 522 ++++++ .../compute/minute/valley_price_quantile.py | 730 ++++++++ .../compute/minute/valley_relative_vwap.py | 434 +++++ .../compute/minute/valley_ridge_vwap_ratio.py | 520 ++++++ factors/compute/minute/volume_peak_count.py | 318 ++++ factors/registry/builtin.py | 44 +- tests/test_amp_marginal_anomaly_vol_factor.py | 6 +- tests/test_intraday_amp_cut_factor.py | 6 +- tests/test_jump_amount_corr_factor.py | 6 +- tests/test_minute_ideal_amplitude_factor.py | 6 +- tests/test_minute_shims.py | 163 ++ tests/test_peak_interval_kurtosis_factor.py | 10 +- tests/test_peak_ridge_amount_ratio_factor.py | 10 +- tests/test_ridge_minute_return_factor.py | 24 +- tests/test_valley_price_quantile_factor.py | 28 +- tests/test_valley_relative_vwap_factor.py | 20 +- tests/test_valley_ridge_vwap_ratio_factor.py | 24 +- tests/test_volume_peak_count_factor.py | 10 +- 40 files changed, 6036 insertions(+), 5373 deletions(-) create mode 100644 factors/compute/minute/__init__.py create mode 100644 factors/compute/minute/amp_marginal_anomaly_vol.py create mode 100644 factors/compute/minute/intraday_amp_cut.py create mode 100644 factors/compute/minute/jump_amount_corr.py create mode 100644 factors/compute/minute/minute_ideal_amplitude.py create mode 100644 factors/compute/minute/mmp.py create mode 100644 factors/compute/minute/peak_interval_kurtosis.py create mode 100644 factors/compute/minute/peak_ridge_amount_ratio.py create mode 100644 factors/compute/minute/primitives.py create mode 100644 factors/compute/minute/ridge_minute_return.py create mode 100644 factors/compute/minute/valley_price_quantile.py create mode 100644 factors/compute/minute/valley_relative_vwap.py create mode 100644 factors/compute/minute/valley_ridge_vwap_ratio.py create mode 100644 factors/compute/minute/volume_peak_count.py create mode 100644 tests/test_minute_shims.py diff --git a/data/clean/intraday_aggregate.py b/data/clean/intraday_aggregate.py index ece083e..4a2dacd 100644 --- a/data/clean/intraday_aggregate.py +++ b/data/clean/intraday_aggregate.py @@ -25,10 +25,20 @@ intraday_vwap_0930_1450 sum(amount) / sum(volume) intraday_last30m_ret_1420_1450 return over the last 30m before the cutoff -This module is DATA-layer only: it does not fetch, does not touch factors / alpha -/ portfolio / runtime, and never sees a token. Coarser intraday bars, if needed, +This module is DATA-layer only: it does not fetch, does not touch alpha / +portfolio / runtime, and never sees a token. Coarser intraday bars, if needed, are DERIVED here from 1min via :func:`resample_intraday_bars` (never raw-fetched), and a derived bar inherits ``available_time = max(source_1min.available_time)``. + +D2 MIXED-MODULE NOTE (design v3.2 §6.4 / R14): this module is the GENERIC CORE +that stays real code — ``asof_daily_features`` / ``resample_intraday_bars`` and +the feature-key machinery. The FACTOR math that used to live here migrated to +``factors.compute.minute`` and is RE-EXPORTED below for the existing importers +(deleted in D6d): the MMP per-bar math (``factors.compute.minute.mmp``) and the +jump-amount-corr factor (``factors.compute.minute.jump_amount_corr``). The +shared index/session constants moved one level down to +``data.clean.intraday_schema`` (single home importable from BOTH sides without +a cycle) and are re-exported here unchanged. """ from __future__ import annotations @@ -37,6 +47,10 @@ import pandas as pd from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DATE_LEVEL, + DEFAULT_DECISION_TIME, + DEFAULT_SESSION_OPEN, INTRADAY_CORE_COLUMNS, INTRADAY_INDEX_NAMES, SYMBOL_LEVEL, @@ -44,8 +58,24 @@ validate_intraday_bars, ) -DATE_LEVEL = "date" -DAILY_INDEX_NAMES: list[str] = [DATE_LEVEL, SYMBOL_LEVEL] +# D2 re-exports: factor math now defined in factors.compute.minute (single +# definition point from D2 on; these aliases are deleted with the shims in D6d). +# The aggregate module may import factors.compute.minute.{mmp,jump_amount_corr} +# because NEITHER of those modules imports this one (they depend only on +# intraday_schema + primitives) — no import cycle. +from factors.compute.minute.jump_amount_corr import ( # noqa: F401 (re-export) + JUMP_LOOKBACK_DAYS, + JUMP_MIN_PAIRS, + JUMP_Z, + compute_jump_amount_corr, +) +from factors.compute.minute.mmp import ( # noqa: F401 (re-export + feature hook) + DEFAULT_EPSILON, + MMP_LOOKBACK, + compute_minute_mmp, + mmp_ew_daily as _mmp_ew_daily_impl, + mmp_valid_minute_counts, +) # All selectable feature keys (validated against the ``features`` argument and # mirrored by qt.config.IntradayCfg.score_feature). Column NAMES below encode the @@ -67,82 +97,7 @@ "last30m_ret", ) -DEFAULT_DECISION_TIME = "14:50:00" -DEFAULT_SESSION_OPEN = "09:30:00" DEFAULT_LAST_WINDOW_MINUTES = 30 -# Minute Microstructure Pressure (MMP, I5c): rolling baseline window (prior bars -# t-MMP_LOOKBACK..t-1) and the default denominator epsilon. EXPLORATORY factor; -# the window is part of the factor definition, not a tuned parameter. -MMP_LOOKBACK = 20 -DEFAULT_EPSILON = 1e-6 - -# Price-jump turnover-correlation factor (PR-C, Kaiyuan report §6). The daily -# value is a trailing-``JUMP_LOOKBACK_DAYS``-trading-day lagged correlation between -# the traded ``amount`` at price-JUMP minutes and the amount at the STRICTLY-next -# minute; a jump minute is one whose within-(symbol, day) amplitude z-score exceeds -# ``JUMP_Z``. These are part of the FACTOR DEFINITION (reproduced from the report), -# not tuned knobs. Requires at least ``JUMP_MIN_PAIRS`` jump-pairs in the window. -JUMP_LOOKBACK_DAYS = 20 -JUMP_MIN_PAIRS = 10 -JUMP_Z = 1.0 -# One minute in seconds: the "strictly next minute" test. A gap != 60s (the lunch -# break, the session close, a missing bar) is NOT the next minute, so that jump -# contributes no pair — exactly the report's within-session adjacency rule. -_ONE_MINUTE_SECONDS = 60.0 - - -def compute_minute_mmp( - open_: np.ndarray, - high: np.ndarray, - low: np.ndarray, - close: np.ndarray, - volume: np.ndarray, - *, - lookback: int = MMP_LOOKBACK, - epsilon: float = DEFAULT_EPSILON, -) -> np.ndarray: - """Per-bar Minute Microstructure Pressure ``MMP_t`` for ONE symbol/day session. - - Inputs are equal-length 1D arrays ORDERED by ``bar_end`` ascending and - belonging to a SINGLE ``(symbol, trade_date)`` session. The rolling baselines - use ONLY the prior ``lookback`` bars (``t-lookback..t-1``) — never bar ``t`` - itself, never a later bar, never the prior day's tail — so the first - ``lookback`` bars have NaN ``MMP``. - - mid_t = (high_t + low_t) / 2 - S_t = (close_t - mid_t) / mid_t (NaN if mid_t <= 0) - V_t = sqrt(volume_t / median(volume[t-lookback:t])) - (NaN if baseline <= 0 / NaN) - B_t = |close_t - open_t| / (high_t - low_t + epsilon) - R_t = (high_t - low_t) / (mean(hl[t-lookback:t]) + epsilon) - (NaN if baseline is NaN) - MMP_t = S_t * V_t * B_t * R_t - - Invalid denominators yield NaN, never ``inf``. Pure: reads no returns / no - future bars / no token. - """ - open_ = np.asarray(open_, dtype=float) - high = np.asarray(high, dtype=float) - low = np.asarray(low, dtype=float) - close = np.asarray(close, dtype=float) - volume = np.asarray(volume, dtype=float) - - hl = high - low - mid = (high + low) / 2.0 - - # Prior-`lookback` baselines: rolling over t-lookback+1..t THEN shift(1) so - # position t holds the statistic of bars t-lookback..t-1 (excludes bar t). - med_vol = pd.Series(volume).rolling(lookback).median().shift(1).to_numpy() - ma_hl = pd.Series(hl).rolling(lookback).mean().shift(1).to_numpy() - - with np.errstate(divide="ignore", invalid="ignore"): - s_t = np.where(mid > 0.0, (close - mid) / mid, np.nan) - ratio = np.where(med_vol > 0.0, volume / med_vol, np.nan) - v_t = np.sqrt(ratio) - b_t = np.abs(close - open_) / (hl + epsilon) - r_t = hl / (ma_hl + epsilon) - mmp = s_t * v_t * b_t * r_t - return mmp def _hhmm(time_str: str) -> str: @@ -242,48 +197,10 @@ def _compute_group( } # MMP is the only rolling feature; compute it ONLY when requested (I5c). if "mmp_ew" in keys: - out["mmp_ew"] = _mmp_ew_daily(g, epsilon, trade_date, session_open) + out["mmp_ew"] = _mmp_ew_daily_impl(g, epsilon, trade_date, session_open) return out -def _in_session(g: pd.DataFrame, trade_date: pd.Timestamp, session_open: str) -> pd.DataFrame: - """Bars whose ``bar_end`` is on/after ``session_open`` (the MMP window lower bound). - - The MMP daily score aggregates over ``[session_open, decision_time]`` (the upper - bound is the available_time cutoff already applied upstream). Restricting to the - in-session bars keeps PRE-session bars out of BOTH the rolling baseline and the - daily mean, so the first in-session bar correctly has no prior-20 baseline. - """ - session_start = trade_date + pd.Timedelta(session_open) - return g[g["bar_end"] >= session_start] - - -def _mmp_ew_daily( - g: pd.DataFrame, epsilon: float, trade_date: pd.Timestamp, session_open: str -) -> float: - """Equal-weight mean of valid per-minute ``MMP_t`` over one PIT-filtered group. - - ``g`` is one ``(date, symbol)`` session's visible bars, sorted by ``bar_end``; - only the in-session bars (``bar_end >= session_open``) enter, so the rolling - baseline starts at the session open and the first 20 in-session bars are NaN. - Every valid minute ``MMP_t`` gets EQUAL weight (no extra volume weighting — the - volume term already lives inside ``MMP_t``). No valid minute -> NaN. - """ - gs = _in_session(g, trade_date, session_open) - if gs.empty: - return float("nan") - mmp = compute_minute_mmp( - gs["open"].to_numpy(dtype=float), - gs["high"].to_numpy(dtype=float), - gs["low"].to_numpy(dtype=float), - gs["close"].to_numpy(dtype=float), - gs["volume"].to_numpy(dtype=float), - epsilon=epsilon, - ) - valid = mmp[~np.isnan(mmp)] - return float(np.mean(valid)) if valid.size else float("nan") - - def asof_daily_features( bars: pd.DataFrame, *, @@ -334,218 +251,6 @@ def asof_daily_features( return pd.DataFrame(data, index=index)[colnames].sort_index() -def mmp_valid_minute_counts( - bars: pd.DataFrame, - *, - decision_time: str = DEFAULT_DECISION_TIME, - session_open: str = DEFAULT_SESSION_OPEN, - epsilon: float = DEFAULT_EPSILON, -) -> pd.Series: - """Per-``(date, symbol)`` count of valid (non-NaN) ``MMP_t`` minutes (I5c report). - - Report-only diagnostic: applies the SAME window as the daily MMP score — - ``available_time <= trade_date + decision_time`` (upper bound) AND - ``bar_end >= trade_date + session_open`` (lower bound) — then counts the - in-session minutes that yielded a valid ``MMP_t`` (the first ``MMP_LOOKBACK`` - in-session bars never do). Reuses :func:`compute_minute_mmp` so there is a - single MMP source of truth. - """ - validate_intraday_bars(bars) - empty = pd.Series( - [], dtype=int, - index=pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ), - ) - if len(bars) == 0: - return empty - work = bars.reset_index() - work["trade_date"] = work["bar_end"].dt.normalize() - cutoff = work["trade_date"] + pd.Timedelta(decision_time) - visible = work.loc[work["available_time"] <= cutoff].copy() - if visible.empty: - return empty - visible = visible.sort_values([SYMBOL_LEVEL, "bar_end"]) - index_tuples: list[tuple] = [] - counts: list[int] = [] - for (date, sym), g in visible.groupby(["trade_date", SYMBOL_LEVEL], sort=True): - gs = _in_session(g, pd.Timestamp(date).normalize(), session_open) - if gs.empty: - index_tuples.append((date, str(sym))) - counts.append(0) - continue - mmp = compute_minute_mmp( - gs["open"].to_numpy(dtype=float), - gs["high"].to_numpy(dtype=float), - gs["low"].to_numpy(dtype=float), - gs["close"].to_numpy(dtype=float), - gs["volume"].to_numpy(dtype=float), - epsilon=epsilon, - ) - index_tuples.append((date, str(sym))) - counts.append(int(np.count_nonzero(~np.isnan(mmp)))) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(counts, index=index, dtype=int).sort_index() - - -def _empty_jump_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` jump-factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def compute_jump_amount_corr( - bars: pd.DataFrame, - *, - lookback_days: int = JUMP_LOOKBACK_DAYS, - min_pairs: int = JUMP_MIN_PAIRS, - jump_z: float = JUMP_Z, - name: str = "jump_amount_corr", -) -> pd.Series: - """PIT-safe daily "price-jump turnover correlation" factor from 1min ``bars``. - - Reproduces the Kaiyuan report §6 factor (``价格跳跃成交额相关性``, full-A RankIC - -10.23%): the trailing-``lookback_days``-trading-day lagged Pearson correlation - between the traded ``amount`` at price-JUMP minutes and the amount at the - STRICTLY-next minute. The daily value at ``(date d, symbol s)`` uses ONLY bars - at dates ``<= d`` (the trailing window ending at d), so a factor value never - sees a future bar (invariant #1); it is meant to trade close-to-close from d+1. - - Definition (LOCKED, per bar of one (symbol, day) session): - * ``amplitude = (high - low) / open`` (guard open > 0, amount finite); - * ``jump`` = within-(symbol, day) amplitude z-score (ddof=1) ``> jump_z``; - * pair each jump minute ``t`` with the STRICTLY-next minute (same session, - ``bar_end`` gap exactly 60s — this excludes the lunch break AND the close); - * ``factor(s, d)`` = Pearson corr(amount[jump t], amount[t+1]) over ALL - jump-pairs whose date is in the trailing ``lookback_days`` TRADING DAYS - (the symbol's own minute-trading days) ending at d; NaN when fewer than - ``min_pairs`` pairs fall in the window (or the correlation is undefined). - - Vectorized (no per-rebalance-date python loop): the trailing-window correlation - is a rolling sum of per-day sufficient statistics (n, sum x, sum y, sum x^2, - sum y^2, sum xy) over the trading-day axis, then Pearson's closed form. Rolling - over ROWS of the per-day-sorted stats == a trailing window of trading days, - because consecutive rows are consecutive trading days. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the - grouping is strictly per symbol (no cross-symbol leakage). - lookback_days: trailing trading-day window length (part of the definition). - min_pairs: minimum jump-pairs in the window for a finite value. - jump_z: within-day amplitude z-score threshold defining a jump minute. - name: the returned Series name (the factor-panel column name). - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily - factor value, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if min_pairs < 2: - # Pearson correlation needs at least 2 points; below that it is undefined. - raise ValueError(f"min_pairs must be >= 2; got {min_pairs!r}.") - if len(bars) == 0: - return _empty_jump_series(name) - - work = bars.reset_index()[ - [SYMBOL_LEVEL, "bar_end", "open", "high", "low", "amount"] - ].copy() - # Guard bad rows BEFORE anything else: a non-positive open makes the amplitude - # meaningless and a non-finite amount would poison the correlation. - work = work[(work["open"] > 0.0) & np.isfinite(work["amount"].to_numpy(dtype=float))] - if work.empty: - return _empty_jump_series(name) - work[DATE_LEVEL] = work["bar_end"].dt.normalize() - # Sort so the "strictly next minute" shift and the per-symbol trading-day - # rolling both see bars in chronological order within each (symbol, day). - work = work.sort_values([SYMBOL_LEVEL, "bar_end"], kind="mergesort") - - work["amp"] = (work["high"] - work["low"]) / work["open"] - by_session = work.groupby([SYMBOL_LEVEL, DATE_LEVEL], sort=False) - mean_amp = by_session["amp"].transform("mean") - std_amp = by_session["amp"].transform("std") # ddof=1 (pandas default) - zscore = (work["amp"] - mean_amp) / std_amp - next_bar_end = by_session["bar_end"].shift(-1) - amt_next = by_session["amount"].shift(-1) - gap = (next_bar_end - work["bar_end"]).dt.total_seconds() - is_jump = (zscore > jump_z) & (gap == _ONE_MINUTE_SECONDS) - - pairs = pd.DataFrame( - { - SYMBOL_LEVEL: work[SYMBOL_LEVEL].to_numpy(), - DATE_LEVEL: work[DATE_LEVEL].to_numpy(), - "x": work["amount"].to_numpy(dtype=float), - "y": amt_next.to_numpy(dtype=float), - } - ).loc[is_jump.to_numpy()] - - # Per-(symbol, day) sufficient statistics of the jump-pairs. - if pairs.empty: - stats = pd.DataFrame( - columns=["cnt", "sx", "sy", "sxx", "syy", "sxy"], dtype=float - ) - stats.index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=[SYMBOL_LEVEL, DATE_LEVEL], - ) - else: - pairs = pairs.assign( - xx=pairs["x"] * pairs["x"], - yy=pairs["y"] * pairs["y"], - xy=pairs["x"] * pairs["y"], - ) - stats = pairs.groupby([SYMBOL_LEVEL, DATE_LEVEL], sort=True).agg( - cnt=("x", "size"), - sx=("x", "sum"), - sy=("y", "sum"), - sxx=("xx", "sum"), - syy=("yy", "sum"), - sxy=("xy", "sum"), - ) - - # Trading-day axis = every (symbol, day) the symbol has minute bars on, so the - # rolling window counts TRADING DAYS (days with no jump still occupy a row, as - # zeros — they consume one of the trailing ``lookback_days`` slots). - axis = ( - work[[SYMBOL_LEVEL, DATE_LEVEL]] - .drop_duplicates() - .sort_values([SYMBOL_LEVEL, DATE_LEVEL], kind="mergesort") - ) - full_index = pd.MultiIndex.from_arrays( - [axis[SYMBOL_LEVEL].to_numpy(), axis[DATE_LEVEL].to_numpy()], - names=[SYMBOL_LEVEL, DATE_LEVEL], - ) - dense = stats.reindex(full_index).fillna(0.0) - rolled = ( - dense.groupby(level=SYMBOL_LEVEL, sort=False) - .rolling(lookback_days, min_periods=1) - .sum() - ) - # groupby.rolling prepends the group key -> drop it, keep (symbol, date). - rolled.index = rolled.index.droplevel(0) - - n = rolled["cnt"].to_numpy(dtype=float) - sx = rolled["sx"].to_numpy(dtype=float) - sy = rolled["sy"].to_numpy(dtype=float) - with np.errstate(invalid="ignore", divide="ignore"): - cov = n * rolled["sxy"].to_numpy(dtype=float) - sx * sy - var_x = n * rolled["sxx"].to_numpy(dtype=float) - sx * sx - var_y = n * rolled["syy"].to_numpy(dtype=float) - sy * sy - den = np.sqrt(var_x * var_y) - corr = np.where((n >= min_pairs) & (den > 0.0), cov / den, np.nan) - corr = np.clip(corr, -1.0, 1.0) - - out = pd.Series(corr, index=rolled.index, name=name) - out.index = out.index.set_names([SYMBOL_LEVEL, DATE_LEVEL]) - return out.reorder_levels([DATE_LEVEL, SYMBOL_LEVEL]).sort_index() - - def resample_intraday_bars(bars: pd.DataFrame, freq: str) -> pd.DataFrame: """Derive coarser intraday bars from normalized 1min ``bars``. @@ -594,3 +299,24 @@ def resample_intraday_bars(bars: pd.DataFrame, freq: str) -> pd.DataFrame: out = out.sort_index() validate_intraday_bars(out) return out + + +__all__ = [ + "DAILY_INDEX_NAMES", + "DATE_LEVEL", + "DEFAULT_DECISION_TIME", + "DEFAULT_EPSILON", + "DEFAULT_FEATURE_KEYS", + "DEFAULT_LAST_WINDOW_MINUTES", + "DEFAULT_SESSION_OPEN", + "INTRADAY_FEATURE_KEYS", + "JUMP_LOOKBACK_DAYS", + "JUMP_MIN_PAIRS", + "JUMP_Z", + "MMP_LOOKBACK", + "asof_daily_features", + "compute_jump_amount_corr", + "compute_minute_mmp", + "mmp_valid_minute_counts", + "resample_intraday_bars", +] diff --git a/data/clean/intraday_amount_ratio.py b/data/clean/intraday_amount_ratio.py index e4aeeea..e42e2ea 100644 --- a/data/clean/intraday_amount_ratio.py +++ b/data/clean/intraday_amount_ratio.py @@ -1,421 +1,21 @@ -"""PEAK/RIDGE AMOUNT-RATIO factor (PR-M). +"""D2 re-export shim — the PR-M factor math moved to ``factors.compute.minute``. -Reproduces the SEVENTH factor of the Kaiyuan market-microstructure series #27 (开源证券 -《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, -§7.2 「峰岭成交比因子多空年化收益 27.13%」): "本小节通过计算 20 日量峰总成交额与量岭总成交额， -二者做比作为峰岭成交比因子，衡量知情交易相对个人投资者交易的相对参与程度". - -This is the last untested FAMILY of the reproduction loop. The nine prior factors covered a -COUNT (PR-F volume_peak_count, weak), a TIMING moment (PR-H peak_interval_kurtosis, null), -two price RATIOS (PR-I valley_relative_vwap, PR-J valley_ridge_vwap_ratio, both PASSING), a -price POSITION (PR-L valley_price_quantile, PASSING and strongest) and a RETURN (PR-K -ridge_minute_return, sign transferred but Reject). Every signal found so far has been a -PRICE signal. This factor carries NO price information at all — it is a pure TRADED-VALUE -mix between the two eruptive groups — so it is the one test that separates "only price -information survives in this taxonomy" from "the peak/ridge split itself carries alpha". - -The classification is REUSED verbatim from :mod:`data.clean.intraday_volume_prv` -(``prepare_visible_minute_bars`` + ``peak_mask_for_symbol``) — same same-slot strictly-prior -μ+kσ eruptive test, same classifiable rule. Nothing about the taxonomy is re-implemented -here, so the seven factors can never drift apart. - -AGGREGATION — THE REPORT'S FORM, WHICH IS *NOT* THE MEAN OF DAILY RATIOS ------------------------------------------------------------------------- -The report says "计算 20 日量峰总成交额与量岭总成交额，二者做比": sum the peak amount over -20 days, sum the ridge amount over 20 days, THEN divide. That is a RATIO OF SUMS. Contrast -§7.1 (PR-J), which explicitly says "计算 20 日价格比均值" — a MEAN OF RATIOS. The report -draws the distinction itself in adjacent sections, so it is a deliberate difference in the -source and is followed here. - -The two forms are genuinely different, not a rounding detail: a ratio of sums is -AMOUNT-WEIGHTED (busy days dominate) while a mean of ratios weights every valid day -equally. The ratio of sums is also structurally far better behaved for THIS quantity — -``peak_amt / ridge_amt`` on a single day has a small, noisy denominator and a heavy right -tail, and a single day whose ridge amount nearly vanishes would dominate a 20-day mean of -ratios. Pooling both legs first is the natural estimator of a 20-day participation MIX, -which is exactly what the report says the factor measures. - -PINNED choices (deliberate and DISCLOSED, not tuned knobs; reproduced on the factor spec): - - 1. PEAK IS THE NUMERATOR, RIDGE THE DENOMINATOR — "峰岭成交比", peak first, and the - report's stated semantics ("衡量知情交易相对个人投资者交易的相对参与程度", informed - trading RELATIVE TO retail) fix the direction independently of the name's word order. - 2. THE RIDGE MASK IS ``eruptive & ~peak`` and the PEAK MASK is PR-F's isolated-eruption - test — the identical masks PR-J used, so ``valley | peak | ridge == classifiable`` - stays an exact partition and no eruptive bar is silently counted on both sides. A - VALLEY bar enters NEITHER leg: this factor reads only the two ERUPTIVE groups. - 3. POSITIVE-TRADE GUARD. A bar with non-finite or non-positive ``amount`` carries no - traded value, so it is dropped from both legs and from both bar counts. The guard runs - at the summation step only — never before classification — because the same-slot μ/σ - baseline is PR-F's and must stay bit-identical. Unlike PR-I/PR-J this factor never - divides by volume, so ``volume`` is NOT part of the guard: a bar with a real amount - contributes its amount regardless of how its volume is recorded. - 4. RAW (UNADJUSTED) AMOUNTS. ``amount`` is traded VALUE in RMB, which no split or - dividend adjustment factor rescales — the adjustment moves prices and share counts in - compensating directions. Both legs are therefore free of the ex-date caveat PR-L had - to disclose, and free even of PR-I/PR-J's weaker "cancels within the day" argument: - there is nothing to cancel. - 5. BOTH LEGS COVER THE PIT-VISIBLE WINDOW ONLY (09:31–14:50), not the full session. A - NECESSARY DEVIATION from the report, which uses the whole day: the standing 14:50 - decision cutoff truncates history days and the signal day identically, and reading the - closing auction would be lookahead at our decision time. - 6. AN ASYMMETRIC BAR FLOOR — ``min_peak_bars`` (=5) against ``min_ridge_bars`` (=10). - PEAKS ARE THE SCARCER LEG HERE, the reverse of PR-J's valley/ridge asymmetry: a peak - must erupt AND be ISOLATED, and isolation is a strong condition on a liquid name whose - eruptions cluster. Holding the peak leg to the ridge floor would discard sound days - and bias the surviving sample toward names whose eruptions happen to arrive alone. - The floor is therefore set LOW, deliberately, and both the realized peak-bar - distribution and the resulting day-validity rate are REPORTED - (``with_diagnostics=True``) rather than left implicit — together with the - counterfactual valid-day count at a peak floor of 10. - 7. BOTH BAR COUNTS ARE TAKEN AFTER THE GUARD, so a day cannot qualify on the strength of - bars that traded nothing. - -Factor value: ``peak_ridge_amount_ratio_20`` = ``Σ peak_amt / Σ ridge_amt`` where both sums -run over the symbol's most recent ``lookback_days`` (=20) VALID trading days INCLUDING -``d``. A day is VALID iff it clears four gates: at least ``min_classifiable`` (=100) -classifiable bars (PR-F's gate, unchanged), at least ``min_peak_bars`` (=5) tradable peak -bars, at least ``min_ridge_bars`` (=10) tradable ridge bars, and strictly positive amount in -BOTH legs. Fewer than ``min_valid_days`` (=10) valid days in the trailing window -> NaN -(honest missing; the runner discloses the coverage). Values are emitted only on valid days. - -Pre-registered sign = +1, READ FROM THE REPORT, not from data and not from semantics: §7.2 -states "峰岭成交比因子 RankIC 均值 10.28%，RankICIR 4.07" — an explicitly POSITIVE RankIC — -with a long leg of 16.06%/yr, long-short 27.13%/yr, IR 2.89, max drawdown 7.88% and a 74.3% -monthly win rate (every year positive 2013–2025). The report's §1 taxonomy summary agrees -independently: "对于量峰时点，分钟数、成交额类因子更加有效，反映知情交易参与度，为正向因子" -— amount factors at PEAK minutes are POSITIVE factors — while the same sentence marks the -ridge leg's amount factors as negative-alpha, so peak-over-ridge is positive on both counts. -NOTE the report is a MONTHLY, market-cap + industry neutral full-market series on Wind data -while our eval cell is CSI500 daily with industry + size neutralization, so its numbers are -a LOOSE reference only (disclosed, never mislabeled, never written in as an expected value). - -The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future bar -(invariant #1); it is a DAILY signal traded close-to-close from d+1. This module is -DATA-layer only: it does not fetch, does not touch factors / alpha / portfolio / runtime, -and never sees a token. +Single definition point from D2 on (design v3.2 §6.4): the peak/ridge +traded-amount-ratio factor lives in +:mod:`factors.compute.minute.peak_ridge_amount_ratio`. This shim keeps every +name the pre-D2 module exported importable from its old path; it is deleted in +D6d. Import-only by contract (locked by the shim purity test). """ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import DAILY_INDEX_NAMES, DEFAULT_DECISION_TIME -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars -from data.clean.intraday_volume_prv import ( - VOLUME_PRV_BASELINE_DAYS, - VOLUME_PRV_BASELINE_MIN_OBS, - VOLUME_PRV_MIN_CLASSIFIABLE, - VOLUME_PRV_MIN_VALID_DAYS, - VOLUME_PRV_SIGMA_K, - peak_mask_for_symbol, - prepare_visible_minute_bars, +from factors.compute.minute.peak_ridge_amount_ratio import ( + DIAGNOSTIC_COLUMNS, + PEAK_RIDGE_LOOKBACK_DAYS, + PEAK_RIDGE_MIN_PEAK_BARS, + PEAK_RIDGE_MIN_RIDGE_BARS, + compute_peak_ridge_amount_ratio, + peak_ridge_amount_by_day, ) -# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). -# The classification constants are IMPORTED from PR-F, never redefined. -PEAK_RIDGE_LOOKBACK_DAYS = 20 # trailing VALID trading-day window for BOTH sums, includes d -# PINNED LOWER than the ridge floor (module docstring §6): a peak must erupt AND be -# ISOLATED, which makes peaks the structurally scarcer leg of this pair. -PEAK_RIDGE_MIN_PEAK_BARS = 5 # min TRADABLE peak bars for a valid day -PEAK_RIDGE_MIN_RIDGE_BARS = 10 # min TRADABLE ridge bars for a valid day - -# The extra 1min column this factor needs on top of PR-F's (volume): the traded value. -# Unlike PR-I / PR-J this factor never divides by volume — amount is the whole quantity. -_AMOUNT = "amount" - -# Per-day diagnostic columns (the peak-scarcity disclosure the task card requires). -DIAGNOSTIC_COLUMNS = ("classifiable_bars", "peak_bars", "ridge_bars", "valid") - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -# --------------------------------------------------------------------------- # -# The three load-bearing steps, each a NAMED function. -# -# They are small enough to inline, but each one carries a correctness property that the -# test-suite has to be able to BREAK on purpose: the positive-trade guard, the trailing -# (never forward-looking) window, and the per-symbol split. A test that asserts "perturbing -# X changes nothing" proves nothing unless the defective implementation can be substituted -# and shown to FAIL it, so each property gets its own substitutable seam. -# --------------------------------------------------------------------------- # -def _tradable_amount(amt: np.ndarray) -> np.ndarray: - """Positive-trade guard (module docstring §3): finite, strictly positive amount. - - ``volume`` is deliberately absent — this factor never divides by volume, so a bar with - a real traded value contributes it regardless of how its volume is recorded. - """ - return np.isfinite(amt) & (amt > 0.0) - - -def _trailing_ratio_of_sums( - legs: pd.DataFrame, *, lookback_days: int, min_valid_days: int -) -> pd.Series: - """The report's ratio of 20-day sums over a STRICTLY TRAILING valid-day window. - - Both legs are pooled with the SAME window and the SAME ``min_periods``, so numerator - and denominator always cover exactly the same days. The window is trailing and the - index is sorted ascending first, so a value at ``d`` can never absorb a later day. - """ - ordered = legs.sort_index() - roll = ordered.rolling(lookback_days, min_periods=min_valid_days).sum() - return roll["peak_amt"] / roll["ridge_amt"] - - -def _symbol_frames(visible: pd.DataFrame): - """Split the visible bars into ONE FRAME PER SYMBOL, in sorted symbol order. - - The whole cross-symbol isolation guarantee lives here: every downstream step (the - same-slot baseline, the classification, both amount legs, the trailing window) runs on - a single symbol's rows, so one symbol's bars can never reach another's factor value. - """ - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - yield str(sym), g.reset_index(drop=True) - - -def peak_ridge_amount_by_day( - work: pd.DataFrame, - *, - min_peak_bars: int = PEAK_RIDGE_MIN_PEAK_BARS, - min_ridge_bars: int = PEAK_RIDGE_MIN_RIDGE_BARS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - with_diagnostics: bool = False, -) -> pd.DataFrame | tuple[pd.DataFrame, pd.DataFrame]: - """Daily peak / ridge traded-AMOUNT totals for ONE symbol, on VALID days only. - - ``work`` is one symbol's frame as returned by - :func:`~data.clean.intraday_volume_prv.peak_mask_for_symbol`, which must have been - built from bars prepared with ``extra_columns=("amount",)`` so the traded value is - available alongside the ``peak`` / ``ridge`` / ``classifiable`` masks. - - Returns the two LEGS rather than their daily ratio, because the factor is the report's - RATIO OF 20-DAY SUMS (module docstring), not the mean of daily ratios — the caller - pools each leg over the trailing window first and divides once at the end. - - Args: - work: one symbol's classified minute frame (see above). - min_peak_bars: minimum TRADABLE peak bars for a valid day (PINNED lower than the - ridge floor — see §6; peaks are the scarcer leg of this pair). - min_ridge_bars: minimum TRADABLE ridge bars for a valid day. - min_classifiable: minimum classifiable bars for a valid day (PR-F's gate). - with_diagnostics: also return the per-day bar-count frame, so the caller can - REPORT the peak-scarcity distribution instead of only gating on it. - - Returns: - DataFrame indexed by ``trade_date`` (ascending) with ``peak_amt`` / ``ridge_amt`` - columns for the days that clear all gates — invalid days are ABSENT, not NaN, so - they do not occupy a slot in the caller's trailing window (the same rule - PR-F / PR-H / PR-I / PR-J use). With ``with_diagnostics=True``, a - ``(legs, diagnostics)`` pair where ``diagnostics`` is indexed by EVERY day present - in ``work`` and carries :data:`DIAGNOSTIC_COLUMNS`. - """ - amt = work[_AMOUNT].to_numpy(dtype=float) - # Positive-trade guard: a bar with no finite positive traded value contributes nothing. - # Applied HERE, at the summation step, never before classification — PR-F's same-slot - # baseline must stay bit-identical. - tradable = _tradable_amount(amt) - peak = work["peak"].to_numpy(dtype=bool) & tradable - ridge = work["ridge"].to_numpy(dtype=bool) & tradable - - per_bar = pd.DataFrame( - { - "trade_date": work["trade_date"].to_numpy(), - "peak_amt": np.where(peak, amt, 0.0), - "ridge_amt": np.where(ridge, amt, 0.0), - "peak_bars": peak.astype(np.int64), - "ridge_bars": ridge.astype(np.int64), - "classifiable_bars": work["classifiable"] - .to_numpy(dtype=bool) - .astype(np.int64), - } - ) - agg = per_bar.groupby("trade_date", sort=True).sum() - - # Four validity gates (module docstring): PR-F's classifiable floor (unchanged), - # enough TRADABLE peak bars, enough TRADABLE ridge bars, and strictly positive amount - # on both legs (a day contributing 0 to the denominator pool is never fabricated). - valid = ( - (agg["classifiable_bars"] >= min_classifiable) - & (agg["peak_bars"] >= min_peak_bars) - & (agg["ridge_bars"] >= min_ridge_bars) - & (agg["peak_amt"] > 0.0) - & (agg["ridge_amt"] > 0.0) - ) - legs = agg.loc[valid, ["peak_amt", "ridge_amt"]].astype(float) - - if not with_diagnostics: - return legs - diagnostics = agg[["classifiable_bars", "peak_bars", "ridge_bars"]].copy() - diagnostics["valid"] = valid - return legs, diagnostics - - -def _peak_ridge_amount_ratio_for_symbol( - g: pd.DataFrame, - *, - baseline_days: int, - baseline_min_obs: int, - sigma_k: float, - lookback_days: int, - min_valid_days: int, - min_classifiable: int, - min_peak_bars: int, - min_ridge_bars: int, - collect_diagnostics: bool = False, -) -> tuple[list[pd.Timestamp], list[float], pd.DataFrame | None]: - """Daily peak/ridge amount-ratio values for ONE symbol from its PIT-visible bars. - - Classifies the minutes with the REUSED :func:`peak_mask_for_symbol`, reduces each valid - day to its two amount legs, then divides the trailing-``lookback_days``-valid-day SUM of - the peak leg by that of the ridge leg (the report's ratio-of-sums form). No cross-symbol - leakage (``g`` is one symbol's slice) and no lookahead (the baseline is strictly prior, - both rolling windows are trailing). - """ - work = peak_mask_for_symbol( - g, - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - ) - result = peak_ridge_amount_by_day( - work, - min_peak_bars=min_peak_bars, - min_ridge_bars=min_ridge_bars, - min_classifiable=min_classifiable, - with_diagnostics=collect_diagnostics, - ) - if collect_diagnostics: - legs, diagnostics = result - else: - legs, diagnostics = result, None - if legs.empty: - return [], [], diagnostics - - # RATIO OF SUMS over the trailing lookback_days VALID days (including d); NaN until - # min_valid_days valid days have accumulated. - ratio = _trailing_ratio_of_sums( - legs, lookback_days=lookback_days, min_valid_days=min_valid_days - ) - days = [pd.Timestamp(d).normalize() for d in ratio.index] - return days, list(ratio.to_numpy(dtype=float)), diagnostics - - -def compute_peak_ridge_amount_ratio( - bars: pd.DataFrame, - *, - lookback_days: int = PEAK_RIDGE_LOOKBACK_DAYS, - baseline_days: int = VOLUME_PRV_BASELINE_DAYS, - baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, - sigma_k: float = VOLUME_PRV_SIGMA_K, - min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - min_peak_bars: int = PEAK_RIDGE_MIN_PEAK_BARS, - min_ridge_bars: int = PEAK_RIDGE_MIN_RIDGE_BARS, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "peak_ridge_amount_ratio", - diagnostics_out: list | None = None, -) -> pd.Series: - """PIT-safe daily "peak/ridge traded-amount ratio" factor from 1min ``bars``. - - Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, classifies - every visible minute with the REUSED PR-F taxonomy, totals each valid day's peak and - ridge traded amount, and returns the ratio of the trailing-``lookback_days``-VALID-day - SUMS of the two legs. See the module docstring for the LOCKED definition, the report's - ratio-of-sums wording, and the seven pinned choices. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is - strictly per symbol (no cross-symbol leakage). - lookback_days: trailing VALID trading-day window POOLED by both legs (definition). - baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). - baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). - sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). - min_valid_days: minimum valid days in the trailing window for a finite value. - min_classifiable: a day needs at least this many classifiable bars (PR-F). - min_peak_bars: a day needs at least this many TRADABLE peak bars (PINNED lower than - the ridge floor — see the module docstring §6). - min_ridge_bars: a day needs at least this many TRADABLE ridge bars. - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - name: the returned Series name (the factor-panel column name). - diagnostics_out: optional list the per-symbol day-level bar-count frames are - APPENDED to (each carries a ``symbol`` column), so a caller can report the - peak-scarcity distribution. Purely observational — supplying it does not change - the returned factor. - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily factor - value, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if baseline_days < 2: - # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is - # defined. - raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") - if baseline_min_obs < 2: - raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") - if sigma_k < 0.0: - raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") - if min_valid_days < 1: - raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") - if min_classifiable < 1: - raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") - if min_peak_bars < 1: - raise ValueError(f"min_peak_bars must be >= 1; got {min_peak_bars!r}.") - if min_ridge_bars < 1: - raise ValueError(f"min_ridge_bars must be >= 1; got {min_ridge_bars!r}.") - if len(bars) == 0: - return _empty_series(name) - - # extra_columns=("amount",) is the ONLY difference from the PR-F / PR-H entry points: - # the traded value rides along on the surviving rows, and the truncation / volume guard - # / slot assignment are untouched. - visible = prepare_visible_minute_bars( - bars, decision_time=decision_time, extra_columns=(_AMOUNT,) - ) - if visible.empty: - return _empty_series(name) - - collect = diagnostics_out is not None - index_tuples: list[tuple] = [] - values: list[float] = [] - for sym, g in _symbol_frames(visible): - days, vals, diagnostics = _peak_ridge_amount_ratio_for_symbol( - g, - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - lookback_days=lookback_days, - min_valid_days=min_valid_days, - min_classifiable=min_classifiable, - min_peak_bars=min_peak_bars, - min_ridge_bars=min_ridge_bars, - collect_diagnostics=collect, - ) - if collect and diagnostics is not None and not diagnostics.empty: - frame = diagnostics.copy() - frame[SYMBOL_LEVEL] = sym - diagnostics_out.append(frame) - for day, val in zip(days, vals): - index_tuples.append((day, sym)) - values.append(val) - - if not index_tuples: - return _empty_series(name) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - - __all__ = [ "DIAGNOSTIC_COLUMNS", "PEAK_RIDGE_LOOKBACK_DAYS", diff --git a/data/clean/intraday_amp_anomaly.py b/data/clean/intraday_amp_anomaly.py index b2da0a8..a2e89da 100644 --- a/data/clean/intraday_amp_anomaly.py +++ b/data/clean/intraday_amp_anomaly.py @@ -1,252 +1,20 @@ -"""Amplitude marginal-anomaly relative-volatility factor (PR-E). +"""D2 re-export shim — the PR-E factor math moved to ``factors.compute.minute``. -Reproduces the Changjiang high-frequency-factor series #19 (长江证券《高频因子（十九）》, -2026-06-03, reportId 5462994) "振幅边际异常相对波动因子" as a daily PIT-safe column -derived from 5min bars (themselves DERIVED from the 1min cache). Kept in the -DATA-clean layer (like :func:`data.clean.intraday_amplitude.compute_minute_ideal_amplitude`) -so the ``factors`` layer only SELECTS the pre-aggregated column and never fetches or -sees a forward return. - -The source report is UNDER-SPECIFIED; the five choices below are deliberate, -DISCLOSED interpretations pinned in the task card (task_card_pr_e_*.md §0), not tuned -knobs. They are reproduced verbatim on the factor spec so a reader can see exactly -what was assumed: - - 1. bar frequency = 5min, DERIVED from the 1min cache via - :func:`data.clean.intraday_aggregate.resample_intraday_bars` (a derived bar - inherits ``available_time = max(source_1min.available_time)`` — PIT-faithful). - 2. lookback window N = 20 trading days (the symbol's own days that have bars, - trailing and INCLUDING ``d``). - 3. anomaly threshold = ``1{|Δamp_t| > μ + σ}`` with μ / σ = mean / ddof=1 std of - ALL pooled ``|Δamp|`` in the 20-day pool (k = 1). - 4. weighted volatility = the ddof=1 SAMPLE std of the RETURNS on the selected - (weight-1) bars. - 5. bar return ``r_t = close_t/close_{t-1} - 1`` and ``Δamp_t = amp_t - amp_{t-1}`` - are BOTH WITHIN-DAY lagged — each day's FIRST bar has no return / no Δamp — so - the overnight gap never contaminates a pair. - -Definition (per symbol, per panel date ``d``): - - 1. Take the symbol's most recent ``N`` (=20) trading days INCLUDING ``d`` of 5min - bars. PIT truncation (standing authorization): keep only bars with - ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50), - so every day is truncated to its own [session-open, 14:50]. - 2. Per bar: ``amp = high/low - 1`` (drop a bar unless ``low > 0`` and - ``high >= low``). WITHIN each (symbol, day), sorted by ``bar_end``, form - ``Δamp_t = amp_t - amp_{t-1}`` and ``r_t = close_t/close_{t-1} - 1``; the first - surviving bar of each day has neither (no cross-day lag). - 3. Pool ALL surviving ``(|Δamp|, r)`` pairs of the ``N``-day window into ONE set. - If the pool has fewer than ``min_pool`` (=460 ≈ 46 x 20 / 2) valid pairs, the - value is NaN (honest missing — coverage is disclosed by the runner). - 4. Pool ``μ = mean(|Δamp|)`` and ``σ = std(|Δamp|, ddof=1)``; select the bars with - ``|Δamp_t| > μ + k*σ`` (k = 1). If fewer than ``min_selected`` (=20) bars are - selected, the value is NaN (an anomaly std over too few bars is unreliable). - 5. factor(d, s) = ``std(r_t | selected, ddof=1)`` (column - ``amp_marginal_anomaly_vol_{N}``). - -Pre-registered sign = +1 (the report's IC is positive across its universes: -raw CSI800 +4.47% / full-market +4.92%; market-cap + industry neutral +4.11% / -+5.56%, ICIR 65.85% / 108.71%). NOTE the report's sample is CSI800 / full-market on -a MONTHLY series; our eval cell is CSI500 daily — a LOOSE reference only. The value -at ``(d, s)`` uses only bars at dates <= d, so a factor value never sees a future -bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. - -This module is DATA-layer only: it does not fetch, does not touch factors / alpha / -portfolio / runtime, and never sees a token. +Single definition point from D2 on (design v3.2 §6.4): the amplitude +marginal-anomaly relative-volatility factor lives in +:mod:`factors.compute.minute.amp_marginal_anomaly_vol`. This shim keeps every +name the pre-D2 module exported importable from its old path; it is deleted in +D6d. Import-only by contract (locked by the shim purity test). """ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import ( - DAILY_INDEX_NAMES, - DEFAULT_DECISION_TIME, - resample_intraday_bars, +from factors.compute.minute.amp_marginal_anomaly_vol import ( + AMP_ANOMALY_FREQ, + AMP_ANOMALY_LOOKBACK_DAYS, + AMP_ANOMALY_MIN_POOL, + AMP_ANOMALY_MIN_SELECTED, + AMP_ANOMALY_SIGMA_K, + compute_amp_marginal_anomaly_vol, ) -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars - -# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). -AMP_ANOMALY_LOOKBACK_DAYS = 20 # trailing trading-day window (N), includes date d -AMP_ANOMALY_FREQ = "5min" # bar frequency, DERIVED from the 1min cache -AMP_ANOMALY_SIGMA_K = 1.0 # anomaly threshold multiplier k in |Δamp| > μ + k*σ -AMP_ANOMALY_MIN_POOL = 460 # minimum valid pooled (|Δamp|, r) pairs for a finite value -AMP_ANOMALY_MIN_SELECTED = 20 # minimum selected (anomaly) bars for a finite std - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def _anomaly_vol_cut( - dabs: np.ndarray, - ret: np.ndarray, - min_pool: int, - min_selected: int, - sigma_k: float, -) -> float: - """Selected-bar return std over one pooled window; NaN if a gate fails. - - ``dabs`` / ``ret`` are equal-length arrays of the VALID ``(|Δamp|, r)`` pairs in - ONE pooled window (already NaN-free). Fewer than ``min_pool`` pairs -> NaN; select - the bars whose ``|Δamp|`` exceeds ``mean + sigma_k * std(ddof=1)`` of the pooled - ``|Δamp|``; fewer than ``min_selected`` selected -> NaN (an unreliable std). - """ - n = dabs.size - if n < min_pool: - return float("nan") - mu = float(dabs.mean()) - sigma = float(dabs.std(ddof=1)) - threshold = mu + sigma_k * sigma - mask = dabs > threshold - n_sel = int(np.count_nonzero(mask)) - if n_sel < min_selected: - return float("nan") - return float(ret[mask].std(ddof=1)) - - -def _anomaly_vol_for_symbol( - g: pd.DataFrame, - lookback_days: int, - min_pool: int, - min_selected: int, - sigma_k: float, -) -> tuple[list[pd.Timestamp], list[float]]: - """Daily factor values for ONE symbol from its within-day-lagged bars. - - ``g`` holds columns ``trade_date`` / ``dabs`` (``|Δamp_t|``) / ``ret`` (``r_t``) - for a single symbol; ``dabs`` / ``ret`` are NaN on each day's first surviving bar - (no cross-day lag). Per day the VALID (finite) pairs are collected once, then each - date pools the trailing ``lookback_days`` days (including that date) — no - cross-symbol leakage because ``g`` is a single symbol's slice, and no cross-day - leakage because each day's first-bar pair is already NaN and dropped here. - """ - days: list[pd.Timestamp] = [] - day_dabs: list[np.ndarray] = [] - day_ret: list[np.ndarray] = [] - for day, sub in g.groupby("trade_date", sort=True): - days.append(pd.Timestamp(day).normalize()) - d = sub["dabs"].to_numpy(dtype=float) - r = sub["ret"].to_numpy(dtype=float) - valid = np.isfinite(d) & np.isfinite(r) - day_dabs.append(d[valid]) - day_ret.append(r[valid]) - - values: list[float] = [] - for j in range(len(days)): - lo = max(0, j - lookback_days + 1) - dabs = np.concatenate(day_dabs[lo : j + 1]) - ret = np.concatenate(day_ret[lo : j + 1]) - values.append(_anomaly_vol_cut(dabs, ret, min_pool, min_selected, sigma_k)) - return days, values - - -def compute_amp_marginal_anomaly_vol( - bars: pd.DataFrame, - *, - lookback_days: int = AMP_ANOMALY_LOOKBACK_DAYS, - min_pool: int = AMP_ANOMALY_MIN_POOL, - min_selected: int = AMP_ANOMALY_MIN_SELECTED, - sigma_k: float = AMP_ANOMALY_SIGMA_K, - decision_time: str = DEFAULT_DECISION_TIME, - freq: str = AMP_ANOMALY_FREQ, - name: str = "amp_marginal_anomaly_vol", -) -> pd.Series: - """PIT-safe daily "amplitude marginal-anomaly relative-volatility" factor. - - Takes normalized 1min ``bars``, DERIVES ``freq`` (default 5min) bars from them via - :func:`resample_intraday_bars` (so a derived bar is usable only once EVERY - constituent 1min bar is available), PIT-truncates them at ``decision_time``, forms - the within-day ``|Δamp|`` / ``r`` pairs, and returns the trailing-``lookback_days`` - anomaly-bar return std. See the module docstring for the LOCKED definition. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping - is strictly per symbol (no cross-symbol leakage). - lookback_days: trailing trading-day window length (part of the definition). - min_pool: minimum valid pooled ``(|Δamp|, r)`` pairs for a finite value. - min_selected: minimum selected (anomaly) bars for a finite return std. - sigma_k: anomaly threshold multiplier ``k`` in ``|Δamp| > μ + k*σ``. - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - freq: derived bar frequency (default 5min); part of the definition. - name: the returned Series name (the factor-panel column name). - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily - factor value, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if min_pool < 2: - # Need >= 2 pooled points so a ddof=1 std of |Δamp| is defined. - raise ValueError(f"min_pool must be >= 2; got {min_pool!r}.") - if min_selected < 2: - # Need >= 2 selected bars so a ddof=1 std of their returns is defined. - raise ValueError(f"min_selected must be >= 2; got {min_selected!r}.") - if sigma_k < 0.0: - raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") - if len(bars) == 0: - return _empty_series(name) - - # DERIVE the coarse (5min) bars from 1min FIRST: available_time = max source, so a - # coarse bar enters only once all its 1min constituents are available. - coarse = resample_intraday_bars(bars, freq) - if len(coarse) == 0: - return _empty_series(name) - - work = coarse.reset_index()[ - [SYMBOL_LEVEL, "bar_end", "available_time", "high", "low", "close"] - ].copy() - work["trade_date"] = work["bar_end"].dt.normalize() - # PIT truncation FIRST (per-bar timestamps): each bar's cutoff is its own - # trade_date + decision_time, so every day is truncated to [open, 14:50]. - cutoff = work["trade_date"] + pd.Timedelta(decision_time) - visible = work.loc[work["available_time"] <= cutoff].copy() - if visible.empty: - return _empty_series(name) - - low = visible["low"].to_numpy(dtype=float) - high = visible["high"].to_numpy(dtype=float) - guard = (low > 0.0) & (high >= low) - visible = visible.loc[guard].copy() - if visible.empty: - return _empty_series(name) - - visible["amp"] = visible["high"].to_numpy(dtype=float) / visible["low"].to_numpy( - dtype=float - ) - 1.0 - # Sort so the within-day lag sees bars in chronological order within each - # (symbol, day); mergesort keeps it stable. - visible = visible.sort_values([SYMBOL_LEVEL, "bar_end"], kind="mergesort") - # Δamp and r are WITHIN-DAY lagged: grouping by (symbol, trade_date) makes each - # day's FIRST surviving bar NaN, so no pair ever crosses the overnight gap. - by_session = visible.groupby([SYMBOL_LEVEL, "trade_date"], sort=False) - visible["dabs"] = by_session["amp"].diff().abs() - visible["ret"] = by_session["close"].pct_change() - - index_tuples: list[tuple] = [] - values: list[float] = [] - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - days, vals = _anomaly_vol_for_symbol( - g, lookback_days, min_pool, min_selected, sigma_k - ) - for day, val in zip(days, vals): - index_tuples.append((day, str(sym))) - values.append(val) - - if not index_tuples: - return _empty_series(name) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - __all__ = [ "AMP_ANOMALY_FREQ", diff --git a/data/clean/intraday_amp_cut.py b/data/clean/intraday_amp_cut.py index e6affd2..4a4badd 100644 --- a/data/clean/intraday_amp_cut.py +++ b/data/clean/intraday_amp_cut.py @@ -1,375 +1,24 @@ -"""Intraday amplitude-cut factor (PR-G). +"""D2 re-export shim — the PR-G factor math moved to ``factors.compute.minute``. -Reproduces the SECOND factor of the Kaiyuan market-microstructure series #30 (开源证券 -《高频振幅因子的内部切割——市场微观结构系列（30）》, 2025-08-09, reportId 4988549) — the -"日内振幅切割因子" (§3, Table 3, thought (2)) — as a daily PIT-safe column derived from -the 1min cache. Kept in the DATA-clean layer (like -:func:`data.clean.intraday_amplitude.compute_minute_ideal_amplitude`) so the ``factors`` -layer only SELECTS the pre-aggregated column and never fetches or sees a forward return. - -DISTINCTION FROM PR-D (must be stated on the factor spec): PR-D (``minute_ideal_amp``, -the report's FLAGSHIP, thought (1)) POOLS the trailing 10 days' 1min bars into ONE set -and cuts by minute CLOSE PRICE. THIS factor (thought (2)) cuts EACH DAY independently by -the 1-MINUTE RETURN, produces a daily ``V_day`` series, then takes its trailing-10-valid- -day mean / std and combines them cross-sectionally. The report finds the two are only -~30% correlated — they are mathematically distinct constructions, not variants. - -Definition (report Table 3 four steps + §3 terminal parameters). For each symbol and each -panel date ``d`` (a DAILY signal, close-to-close, ``is_intraday=False``): - - 1. PIT truncation (standing authorization): each day keeps only the 1min bars with - ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50), so - every day (history AND signal day) is truncated to its own [session-open, 14:50]. - 2. PER-DAY CUT (each trading day in the window, independent): - - per-bar amplitude ``amp = high/low - 1`` (drop a bar unless ``low > 0`` and - ``high >= low``); - - the sentiment indicator is the 1-MINUTE RETURN ``r_t = close_t/close_{t-1} - 1`` - (the report's terminal pick), WITHIN-DAY lagged: each day's FIRST surviving bar - has no ``r`` and is NOT in that day's cut (no overnight gap; PR-E precedent); - - a bar is VALID iff both ``amp`` and ``r`` are finite; ``n_day`` = the valid-bar - count; ``n_day < min_day_minutes`` (=100) -> the day is INVALID (no ``V_day``); - - ``k = floor(lam * n_day)`` (lam=0.20, report terminal), ``k >= 1``; a stable - total order ``(r, bar_end)`` makes the selection deterministic; - - ``V_high`` = mean ``amp`` of the ``k`` HIGHEST-``r`` bars, ``V_low`` = mean - ``amp`` of the ``k`` LOWEST-``r`` bars; the day's cut value is - ``V_day = V_high - V_low``. - 3. TIME-SERIES aggregation: take the most recent ``lookback_days`` (=10) VALID trading - days INCLUDING ``d``; fewer than ``min_valid_days`` (=6) valid days -> NaN (honest - missing). ``V_mean = mean(V_day)`` and ``V_std = std(V_day, ddof=1)``. - 4. CROSS-SECTIONAL standardization (report step 4): for each panel date, z-score - ``V_mean`` and ``V_std`` SEPARATELY over the covered-universe cross-section (ddof=1 - std denominator; a date whose finite-pair cross-section is smaller than - ``min_cross_section`` (=10) -> all NaN that date), then the factor value is - ``(z(V_mean) + z(V_std)) / 2`` (column ``intraday_amp_cut_{N}``). - -Pre-registered sign = -1 (report: rankIC mean -0.067, rankICIR -3.82, quintile -long-short 16.7%/yr at N=10, lambda=20%, 1-minute-return indicator; the V_mean and V_std -sub-factors are each negative too). Low-volatility family reading: when high-return -minutes have a MUCH larger amplitude than low-return minutes, future returns are lower. - -PINNED interpretation (disclosed, not a tuned knob): the report cross-standardizes on the -FULL market; our eval cell cross-standardizes on the CSI500 covered set. The factor value -at ``(d, s)`` uses only bars at dates <= d, so a value never sees a future bar -(invariant #1). This module is DATA-layer only: it does not fetch, does not touch -factors / alpha / portfolio / runtime, and never sees a token. - -The heavy per-symbol work (steps 1-3, producing the ``(V_mean, V_std)`` panel) is split -from the cross-sectional combine (step 4) on purpose: the runner streams one symbol at a -time through :func:`compute_amp_cut_stats` (memory-bounded), assembles the full-universe -two-column panel, then calls :func:`combine_amp_cut_cross_section` ONCE — because step 4 -needs every symbol's ``(V_mean, V_std)`` present before it can z-score a date's -cross-section. :func:`compute_intraday_amp_cut` chains the two for a single-call path. +Single definition point from D2 on (design v3.2 §6.4): the intraday +amplitude-cut factor lives in :mod:`factors.compute.minute.intraday_amp_cut`. +This shim keeps every name the pre-D2 module exported importable from its old +path; it is deleted in D6d. Import-only by contract (locked by the shim purity +test). """ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import ( - DAILY_INDEX_NAMES, - DATE_LEVEL, - DEFAULT_DECISION_TIME, +from factors.compute.minute.intraday_amp_cut import ( + AMP_CUT_LAMBDA, + AMP_CUT_LOOKBACK_DAYS, + AMP_CUT_MIN_CROSS_SECTION, + AMP_CUT_MIN_DAY_MINUTES, + AMP_CUT_MIN_VALID_DAYS, + V_MEAN_COL, + V_STD_COL, + combine_amp_cut_cross_section, + compute_amp_cut_stats, + compute_intraday_amp_cut, ) -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars - -# Factor DEFINITION constants (report terminal parameters; NOT tuned knobs). -AMP_CUT_LOOKBACK_DAYS = 10 # trailing VALID trading-day window (N), includes date d -AMP_CUT_LAMBDA = 0.20 # top/bottom fraction by 1-min return that forms V_high/V_low -AMP_CUT_MIN_DAY_MINUTES = 100 # a day is valid iff it has >= this many valid (amp & r) bars -AMP_CUT_MIN_VALID_DAYS = 6 # min valid days in the trailing window for a finite V_mean/V_std -AMP_CUT_MIN_CROSS_SECTION = 10 # min finite-pair cross-section for a finite z-score on a date - -# Internal two-column stats panel labels (V_mean / V_std of the daily V_day series). -V_MEAN_COL = "v_mean" -V_STD_COL = "v_std" - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def _empty_stats() -> pd.DataFrame: - """Schema-shaped empty ``MultiIndex(date, symbol)`` ``(v_mean, v_std)`` panel.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.DataFrame( - {V_MEAN_COL: pd.Series([], dtype=float), V_STD_COL: pd.Series([], dtype=float)}, - index=index, - ) - - -def _day_cut( - amps: np.ndarray, rets: np.ndarray, bar_ends: np.ndarray, lam: float, min_day_minutes: int -) -> float: - """``V_high - V_low`` for ONE day's valid bars; NaN if the day is invalid or k < 1. - - ``amps`` / ``rets`` / ``bar_ends`` are equal-length arrays of the VALID (amp & r - finite) bars of ONE trading day. The bars are ranked by the 1-minute return ``r`` - with ``(r, bar_end)`` as a stable total order (lexsort with ``r`` primary), so the - top-k / bottom-k selection is deterministic even when two bars share a return. - """ - n = amps.size - if n < min_day_minutes: - return float("nan") - k = int(np.floor(lam * n)) - if k < 1: - return float("nan") - order = np.lexsort((bar_ends, rets)) # r primary, bar_end tie-break - a = amps[order] - return float(a[-k:].mean() - a[:k].mean()) # V_high (top-r) - V_low (bottom-r) - - -def _amp_cut_stats_for_symbol( - g: pd.DataFrame, - lookback_days: int, - lam: float, - min_day_minutes: int, - min_valid_days: int, -) -> tuple[list[pd.Timestamp], list[float], list[float]]: - """Trailing V_mean / V_std for ONE symbol from its within-day-lagged bars. - - ``g`` holds columns ``trade_date`` / ``amp`` / ``ret`` (the within-day 1-minute - return, NaN on each day's first surviving bar) / ``bar_end_ns`` for a single symbol. - Per day the VALID bars are cut into ``V_day``; only VALID days (``V_day`` finite) - enter the trailing series, so the rolling window spans the most recent - ``lookback_days`` VALID days (report "取最近 10 个有效交易日"). Values are emitted only - on valid days; a window with fewer than ``min_valid_days`` valid days -> NaN (both - stats). No cross-symbol leakage — ``g`` is one symbol's slice — and no cross-day - leakage — each day's first-bar return is already NaN and excluded from that day's cut. - """ - days: list[pd.Timestamp] = [] - vdays: list[float] = [] - for day, sub in g.groupby("trade_date", sort=True): - amp = sub["amp"].to_numpy(dtype=float) - ret = sub["ret"].to_numpy(dtype=float) - be = sub["bar_end_ns"].to_numpy(dtype="int64") - valid = np.isfinite(amp) & np.isfinite(ret) - vdays.append(_day_cut(amp[valid], ret[valid], be[valid], lam, min_day_minutes)) - days.append(pd.Timestamp(day).normalize()) - - series = pd.Series(vdays, index=pd.DatetimeIndex(days)).sort_index() - valid_series = series.dropna() # only VALID days carry a V_day - if valid_series.empty: - return [], [], [] - roll = valid_series.rolling(lookback_days, min_periods=min_valid_days) - vmean = roll.mean() - vstd = roll.std() # ddof=1 (pandas rolling default) - out_days = [pd.Timestamp(d).normalize() for d in vmean.index] - return out_days, list(vmean.to_numpy(dtype=float)), list(vstd.to_numpy(dtype=float)) - - -def compute_amp_cut_stats( - bars: pd.DataFrame, - *, - lookback_days: int = AMP_CUT_LOOKBACK_DAYS, - lam: float = AMP_CUT_LAMBDA, - min_day_minutes: int = AMP_CUT_MIN_DAY_MINUTES, - min_valid_days: int = AMP_CUT_MIN_VALID_DAYS, - decision_time: str = DEFAULT_DECISION_TIME, -) -> pd.DataFrame: - """Per-symbol trailing ``(V_mean, V_std)`` panel (steps 1-3; NO cross-section yet). - - Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, forms - the within-day 1-minute returns, cuts each valid day by return into ``V_day``, and - returns the trailing-``lookback_days``-VALID-day mean / std of ``V_day``. The - cross-sectional standardization (step 4) is deliberately NOT applied here — it needs - the full-universe panel and is done by :func:`combine_amp_cut_cross_section`. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the per-day cut - and trailing aggregation are strictly per symbol (no cross-symbol leakage). - lookback_days: trailing VALID trading-day window (definition). - lam: top/bottom 1-minute-return fraction defining V_high/V_low (0 < lam <= 0.5). - min_day_minutes: a day is valid iff it has >= this many valid (amp & r) bars. - min_valid_days: minimum valid days in the trailing window for a finite value. - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - - Returns: - ``MultiIndex(date, symbol)`` DataFrame with columns ``v_mean`` / ``v_std`` - (midnight-normalized dates), sorted. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if not (0.0 < lam <= 0.5): - raise ValueError(f"lam must be in (0, 0.5]; got {lam!r}.") - if min_day_minutes < 2: - # Need at least 2 valid minutes so a non-empty top/bottom cut can exist. - raise ValueError(f"min_day_minutes must be >= 2; got {min_day_minutes!r}.") - if min_valid_days < 2: - # Need >= 2 valid days so a ddof=1 std of the V_day series is defined. - raise ValueError(f"min_valid_days must be >= 2; got {min_valid_days!r}.") - if len(bars) == 0: - return _empty_stats() - - work = bars.reset_index()[ - [SYMBOL_LEVEL, "bar_end", "available_time", "high", "low", "close"] - ].copy() - work["trade_date"] = work["bar_end"].dt.normalize() - # PIT truncation FIRST (per-bar timestamps): each bar's cutoff is its own - # trade_date + decision_time, so every day is truncated to [open, 14:50]. - cutoff = work["trade_date"] + pd.Timedelta(decision_time) - visible = work.loc[work["available_time"] <= cutoff].copy() - if visible.empty: - return _empty_stats() - - low = visible["low"].to_numpy(dtype=float) - high = visible["high"].to_numpy(dtype=float) - guard = (low > 0.0) & (high >= low) - visible = visible.loc[guard].copy() - if visible.empty: - return _empty_stats() - - visible["amp"] = visible["high"].to_numpy(dtype=float) / visible["low"].to_numpy( - dtype=float - ) - 1.0 - # int64 nanoseconds for a deterministic lexsort tie-break (view avoids the - # datetime->int astype deprecation). - visible["bar_end_ns"] = visible["bar_end"].to_numpy(dtype="datetime64[ns]").astype( - "int64" - ) - # Sort so the within-day lag sees bars in chronological order within each - # (symbol, day); mergesort keeps it stable. - visible = visible.sort_values([SYMBOL_LEVEL, "bar_end"], kind="mergesort") - # r is WITHIN-DAY lagged: grouping by (symbol, trade_date) makes each day's FIRST - # surviving bar NaN, so no return ever crosses the overnight gap. - visible["ret"] = visible.groupby([SYMBOL_LEVEL, "trade_date"], sort=False)[ - "close" - ].pct_change() - - index_tuples: list[tuple] = [] - vmean_vals: list[float] = [] - vstd_vals: list[float] = [] - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - days, vmean, vstd = _amp_cut_stats_for_symbol( - g, lookback_days, lam, min_day_minutes, min_valid_days - ) - for day, m, s in zip(days, vmean, vstd): - index_tuples.append((day, str(sym))) - vmean_vals.append(m) - vstd_vals.append(s) - - if not index_tuples: - return _empty_stats() - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.DataFrame( - {V_MEAN_COL: vmean_vals, V_STD_COL: vstd_vals}, index=index - ).sort_index() - - -def combine_amp_cut_cross_section( - stats: pd.DataFrame, - *, - min_cross_section: int = AMP_CUT_MIN_CROSS_SECTION, - name: str = "intraday_amp_cut", -) -> pd.Series: - """Cross-sectional z-score combine of the ``(V_mean, V_std)`` panel (step 4). - - For each panel date, the cross-section is the symbols with BOTH ``V_mean`` and - ``V_std`` finite. If that count is below ``min_cross_section`` (=10), every symbol on - that date is NaN (honest missing). Otherwise ``V_mean`` and ``V_std`` are each - z-scored (ddof=1 denominator) across the cross-section and the factor value is - ``(z(V_mean) + z(V_std)) / 2``. A degenerate date (a column's cross-sectional std is - zero or non-finite) is likewise NaN. The pre-registered sign (-1) lives on the factor - spec, not here — this returns the RAW combined score. - - Args: - stats: ``MultiIndex(date, symbol)`` panel with ``v_mean`` / ``v_std`` columns, - typically the full covered universe assembled by the runner. - min_cross_section: minimum finite-pair cross-section for a finite z-score. - name: the returned Series name (the factor-panel column name). - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the combined - factor value, sorted, named ``name``. Pure: never mutates ``stats``. - """ - if min_cross_section < 2: - # Need >= 2 cross-section members so a ddof=1 std across the section is defined. - raise ValueError(f"min_cross_section must be >= 2; got {min_cross_section!r}.") - if stats is None or stats.empty: - return _empty_series(name) - if V_MEAN_COL not in stats.columns or V_STD_COL not in stats.columns: - raise ValueError( - f"stats must carry '{V_MEAN_COL}' and '{V_STD_COL}' columns; got " - f"{list(stats.columns)}." - ) - - vm_all = stats[V_MEAN_COL].to_numpy(dtype=float) - vs_all = stats[V_STD_COL].to_numpy(dtype=float) - finite_pair = np.isfinite(vm_all) & np.isfinite(vs_all) - valid = stats.loc[finite_pair, [V_MEAN_COL, V_STD_COL]] - if valid.empty: - return _empty_series(name) - - index_tuples: list[tuple] = [] - values: list[float] = [] - for date, grp in valid.groupby(level=DATE_LEVEL, sort=True): - syms = grp.index.get_level_values(SYMBOL_LEVEL) - n = len(grp) - if n < min_cross_section: - vals = np.full(n, np.nan) - else: - vm = grp[V_MEAN_COL].to_numpy(dtype=float) - vs = grp[V_STD_COL].to_numpy(dtype=float) - sm = float(vm.std(ddof=1)) - ss = float(vs.std(ddof=1)) - if not (np.isfinite(sm) and np.isfinite(ss)) or sm == 0.0 or ss == 0.0: - vals = np.full(n, np.nan) - else: - zm = (vm - vm.mean()) / sm - zs = (vs - vs.mean()) / ss - vals = (zm + zs) / 2.0 - day = pd.Timestamp(date).normalize() - for sym, v in zip(syms, vals): - index_tuples.append((day, str(sym))) - values.append(float(v)) - - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - - -def compute_intraday_amp_cut( - bars: pd.DataFrame, - *, - lookback_days: int = AMP_CUT_LOOKBACK_DAYS, - lam: float = AMP_CUT_LAMBDA, - min_day_minutes: int = AMP_CUT_MIN_DAY_MINUTES, - min_valid_days: int = AMP_CUT_MIN_VALID_DAYS, - min_cross_section: int = AMP_CUT_MIN_CROSS_SECTION, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "intraday_amp_cut", -) -> pd.Series: - """Single-call intraday amplitude-cut factor: stats (steps 1-3) then combine (step 4). - - Convenience that chains :func:`compute_amp_cut_stats` and - :func:`combine_amp_cut_cross_section`. The runner does NOT use this (it needs the - per-symbol stats loop for memory-boundedness) but tests and any all-in-memory caller - can. The cross-section is whatever ``bars`` carry — see the module docstring's PINNED - interpretation (the runner scopes it to the CSI500 covered set). - - See the module docstring for the LOCKED definition. Pure: never mutates ``bars``. - """ - stats = compute_amp_cut_stats( - bars, - lookback_days=lookback_days, - lam=lam, - min_day_minutes=min_day_minutes, - min_valid_days=min_valid_days, - decision_time=decision_time, - ) - return combine_amp_cut_cross_section( - stats, min_cross_section=min_cross_section, name=name - ) - __all__ = [ "AMP_CUT_LAMBDA", diff --git a/data/clean/intraday_amplitude.py b/data/clean/intraday_amplitude.py index 23aa4f8..a2c15a7 100644 --- a/data/clean/intraday_amplitude.py +++ b/data/clean/intraday_amplitude.py @@ -1,196 +1,18 @@ -"""Minute "ideal amplitude" factor (PR-D): trailing-window price-ranked amplitude. +"""D2 re-export shim — the PR-D factor math moved to ``factors.compute.minute``. -Reproduces the Kaiyuan report §30 (市场微观结构系列 30) 分钟理想振幅因子 as a daily -PIT-safe column derived from 1min bars. Kept in the DATA-clean layer (like -:func:`data.clean.intraday_aggregate.compute_jump_amount_corr`) so the ``factors`` -layer only SELECTS the pre-aggregated column and never fetches or sees a forward -return. - -Definition (LOCKED — reproduced from the report; N/lambda/min-minutes are part of -the factor DEFINITION, not tuned knobs). For each symbol and each panel date ``d``: - - 1. Take the symbol's most recent ``N`` (=10) trading days INCLUDING ``d`` (the - symbol's own minute-trading days — mirrors the jump factor's trailing window). - 2. PIT truncation (standing authorization): keep only bars with - ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50). - This reuses the I3 per-bar cutoff path, so EVERY day in the window is truncated - to its own [session-open, 14:50] and post-14:50 / close data is never touched. - 3. Per-bar minute amplitude ``amp = high/low - 1``; a bar is dropped unless - ``low > 0`` and ``high >= low``. - 4. Pool ALL surviving bars of the window into ONE set ("merged cut", not a - per-day cut). If the pool has fewer than ``min_minutes`` (=1150 ≈ half of - 10 x ~230) valid minutes, the value is NaN (honest missing — no fabricated - warm-up; coverage is disclosed by the runner). - 5. Rank the pooled minutes by RAW minute close (unadjusted — amplitude is a ratio - so it needs no adjustment, and the report ranks on the raw minute price), with - ``(close, bar_end)`` as a stable total order. With ``k = floor(lambda * n)`` - (lambda=0.25): - V_high = mean amp of the ``k`` HIGHEST-close minutes - V_low = mean amp of the ``k`` LOWEST-close minutes - 6. factor(d, s) = ``V_high - V_low`` (column ``minute_ideal_amp_{N}``). - -Pre-registered sign = -1 (report full-market IC -0.059 / ICIR -3.1 / rankIC -0.076). -The value at ``(d, s)`` uses only bars at dates <= d, so a factor value never sees a -future bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. - -This module is DATA-layer only: it does not fetch, does not touch factors / alpha / -portfolio / runtime, and never sees a token. +Single definition point from D2 on (design v3.2 §6.4): the minute +ideal-amplitude factor lives in +:mod:`factors.compute.minute.minute_ideal_amplitude`. This shim keeps every +name the pre-D2 module exported importable from its old path; it is deleted in +D6d. Import-only by contract (locked by the shim purity test). """ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import DAILY_INDEX_NAMES, DEFAULT_DECISION_TIME -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars - -# Factor DEFINITION constants (report terminal parameters; NOT tuned knobs). -IDEAL_AMP_LOOKBACK_DAYS = 10 # trailing trading-day window (N), includes date d -IDEAL_AMP_LAMBDA = 0.25 # top/bottom fraction by close (lambda) that forms V_high/V_low -IDEAL_AMP_MIN_MINUTES = 1150 # minimum valid pooled minutes for a finite value - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def _rank_cut( - closes: np.ndarray, amps: np.ndarray, bar_ends: np.ndarray, lam: float, min_minutes: int -) -> float: - """V_high - V_low over one pooled window; NaN if too few minutes or k < 1. - - ``closes``/``amps``/``bar_ends`` are equal-length arrays for ONE pooled window. - Ranking is a stable total order ``(close, bar_end)`` (lexsort with close as the - primary key), so the top-k / bottom-k selection is fully deterministic even when - two minutes share a close. - """ - n = closes.size - if n < min_minutes: - return float("nan") - k = int(np.floor(lam * n)) - if k < 1: - return float("nan") - order = np.lexsort((bar_ends, closes)) # close primary, bar_end tie-break - a = amps[order] - return float(a[-k:].mean() - a[:k].mean()) - - -def _amplitude_for_symbol( - g: pd.DataFrame, lookback_days: int, lam: float, min_minutes: int -) -> tuple[list[pd.Timestamp], list[float]]: - """Daily factor values for ONE symbol from its PIT-filtered, guarded bars. - - ``g`` holds columns ``trade_date`` / ``close`` / ``amp`` / ``bar_end_ns`` for a - single symbol. Per-day arrays are built once, then each date pools the trailing - ``lookback_days`` days (including that date) — no cross-symbol leakage because - ``g`` is a single symbol's slice. - """ - days: list[pd.Timestamp] = [] - day_close: list[np.ndarray] = [] - day_amp: list[np.ndarray] = [] - day_be: list[np.ndarray] = [] - for day, sub in g.groupby("trade_date", sort=True): - days.append(pd.Timestamp(day).normalize()) - day_close.append(sub["close"].to_numpy(dtype=float)) - day_amp.append(sub["amp"].to_numpy(dtype=float)) - day_be.append(sub["bar_end_ns"].to_numpy(dtype="int64")) - - values: list[float] = [] - for j in range(len(days)): - lo = max(0, j - lookback_days + 1) - closes = np.concatenate(day_close[lo : j + 1]) - amps = np.concatenate(day_amp[lo : j + 1]) - bes = np.concatenate(day_be[lo : j + 1]) - values.append(_rank_cut(closes, amps, bes, lam, min_minutes)) - return days, values - - -def compute_minute_ideal_amplitude( - bars: pd.DataFrame, - *, - lookback_days: int = IDEAL_AMP_LOOKBACK_DAYS, - lam: float = IDEAL_AMP_LAMBDA, - min_minutes: int = IDEAL_AMP_MIN_MINUTES, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "minute_ideal_amp", -) -> pd.Series: - """PIT-safe daily "minute ideal amplitude" factor from 1min ``bars``. - - See the module docstring for the LOCKED definition. The heavy per-symbol loop is - memory-bounded (the runner feeds one symbol at a time), but this function also - accepts a multi-symbol frame and keeps symbols strictly isolated. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. - lookback_days: trailing trading-day window length (part of the definition). - lam: top/bottom close fraction defining V_high/V_low (0 < lam <= 0.5). - min_minutes: minimum valid pooled minutes for a finite value. - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - name: the returned Series name (the factor-panel column name). - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily - factor value, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if not (0.0 < lam <= 0.5): - raise ValueError(f"lam must be in (0, 0.5]; got {lam!r}.") - if min_minutes < 2: - # Need at least 2 minutes so a non-empty top/bottom cut can exist. - raise ValueError(f"min_minutes must be >= 2; got {min_minutes!r}.") - if len(bars) == 0: - return _empty_series(name) - - work = bars.reset_index()[ - [SYMBOL_LEVEL, "bar_end", "available_time", "high", "low", "close"] - ].copy() - work["trade_date"] = work["bar_end"].dt.normalize() - # PIT truncation FIRST (per-bar timestamps): each bar's cutoff is its own - # trade_date + decision_time, so every day is truncated to [open, 14:50]. - cutoff = work["trade_date"] + pd.Timedelta(decision_time) - visible = work.loc[work["available_time"] <= cutoff].copy() - if visible.empty: - return _empty_series(name) - - low = visible["low"].to_numpy(dtype=float) - high = visible["high"].to_numpy(dtype=float) - guard = (low > 0.0) & (high >= low) - visible = visible.loc[guard].copy() - if visible.empty: - return _empty_series(name) - - visible["amp"] = visible["high"].to_numpy(dtype=float) / visible["low"].to_numpy( - dtype=float - ) - 1.0 - # int64 nanoseconds for a deterministic lexsort tie-break (view avoids the - # datetime->int astype deprecation). - visible["bar_end_ns"] = visible["bar_end"].to_numpy(dtype="datetime64[ns]").astype( - "int64" - ) - visible = visible.sort_values([SYMBOL_LEVEL, "bar_end"], kind="mergesort") - - index_tuples: list[tuple] = [] - values: list[float] = [] - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - days, vals = _amplitude_for_symbol(g, lookback_days, lam, min_minutes) - for day, val in zip(days, vals): - index_tuples.append((day, str(sym))) - values.append(val) - - if not index_tuples: - return _empty_series(name) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - +from factors.compute.minute.minute_ideal_amplitude import ( + IDEAL_AMP_LAMBDA, + IDEAL_AMP_LOOKBACK_DAYS, + IDEAL_AMP_MIN_MINUTES, + compute_minute_ideal_amplitude, +) __all__ = [ "IDEAL_AMP_LAMBDA", diff --git a/data/clean/intraday_peak_interval.py b/data/clean/intraday_peak_interval.py index 3030ba3..3859361 100644 --- a/data/clean/intraday_peak_interval.py +++ b/data/clean/intraday_peak_interval.py @@ -1,307 +1,20 @@ -"""Volume-peak INTERVAL-KURTOSIS factor (PR-H). +"""D2 re-export shim — the PR-H factor math moved to ``factors.compute.minute``. -Reproduces the SECOND factor of the Kaiyuan market-microstructure series #27 (开源证券 -《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, -§6): "针对两个量峰之间的时间间隔计算统计指标，对过去 20 日同日前后两个量峰之间的时间间隔 -分布，计算其峰度". - -Same MACHINE as PR-F, different STATISTIC. The volume-peak identification is REUSED -verbatim from :mod:`data.clean.intraday_volume_prv` (``prepare_visible_minute_bars`` + -``peak_mask_for_symbol``) — same-slot strictly-prior μ+kσ eruptive/mild classification, -1-minute same-session mild-neighbour peak rule, session-boundary and missing-minute -bars are never peaks, a day is VALID iff it has enough classifiable bars. Nothing about -the taxonomy is re-implemented here, so the two factors can never drift apart; this -module only measures the GAPS between consecutive peaks and reduces them to a kurtosis. - -The report is silent on two operative details; both are deliberate, DISCLOSED choices -PINNED in the task card (task_card_pr_h_*.md §1), not tuned knobs, and both are -reproduced on the factor spec: - - 1. INTERVAL UNIT = TRADING MINUTES, i.e. the difference in TRADABLE SLOT POSITION - within the day's PIT-visible bar sequence — NOT wall-clock minutes. The lunch break - therefore costs nothing: a peak at 11:29 and a peak at 13:02 are 3 trading minutes - apart (11:29 -> 11:30 -> 13:01 -> 13:02), not 93. Measuring wall clock would inject - a fixed ~90-minute spike into every interval distribution that happens to straddle - lunch and would dominate the kurtosis. A consequence, disclosed rather than - corrected: a minute MISSING from the cache is not a tradable slot in our sequence, - so an interval spanning it is measured one shorter (the same stance PR-F takes on - gaps — it consults no exchange calendar). - 2. KURTOSIS = FISHER EXCESS, BIAS-CORRECTED — the ``pandas.Series.kurt()`` / - ``scipy.stats.kurtosis(fisher=True, bias=False)`` convention (a normal sample sits - near 0, not 3). ``excess_kurtosis`` implements it in numpy for speed and is locked - against ``pandas.kurt()`` by test. - -Factor value: ``peak_interval_kurtosis_20`` = the kurtosis of the POOLED interval -multiset over the symbol's most recent ``lookback_days`` (=20) VALID trading days -INCLUDING ``d``. A day with fewer than 2 peaks contributes ZERO intervals but is still a -valid day (it is not skipped and does not poison the window). Two NaN gates, both honest -missing rather than a fabricated number: fewer than ``min_intervals`` (=20) pooled -intervals (kurtosis is wildly unstable on small samples), or a zero-variance pool -(kurtosis is undefined). The PR-F valid-day floor is kept as well (``min_valid_days``). -Note the reused taxonomy makes two peaks 1 minute apart impossible — adjacent eruptive -minutes are RIDGES — so the smallest attainable interval is 2. - -Pre-registered sign = +1 (the report's full-market RankIC is +7.19% / RankICIR 4.63, -long-short 23.3%/yr, IR 3.39, 13/13 positive years — the most stable factor in the -report). Semantics: a peaky, fat-tailed interval distribution means informed trading -arrives in BURSTS rather than spread evenly through the session -> higher future return. -NOTE the report is a MONTHLY, market-cap + industry neutral full-market series on Wind -data; our eval cell is CSI500 daily with industry + size neutralization, so its numbers -are a LOOSE reference only (the report gives no CSI500 sub-domain figure for THIS -factor, and none is invented). Raw minute volume (cached as-is) has split-day magnitude -jumps that pollute the 20-day σ; the report (Wind) does not adjust for this either, so -it is disclosed and NOT corrected. - -The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future -bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. This module is -DATA-layer only: it does not fetch, does not touch factors / alpha / portfolio / -runtime, and never sees a token. +Single definition point from D2 on (design v3.2 §6.4): the volume-peak +interval-kurtosis factor lives in +:mod:`factors.compute.minute.peak_interval_kurtosis`. This shim keeps every +name the pre-D2 module exported importable from its old path; it is deleted in +D6d. Import-only by contract (locked by the shim purity test). """ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import DAILY_INDEX_NAMES, DEFAULT_DECISION_TIME -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars -from data.clean.intraday_volume_prv import ( - VOLUME_PRV_BASELINE_DAYS, - VOLUME_PRV_BASELINE_MIN_OBS, - VOLUME_PRV_MIN_CLASSIFIABLE, - VOLUME_PRV_MIN_VALID_DAYS, - VOLUME_PRV_SIGMA_K, - peak_mask_for_symbol, - prepare_visible_minute_bars, +from factors.compute.minute.peak_interval_kurtosis import ( + PEAK_INTERVAL_LOOKBACK_DAYS, + PEAK_INTERVAL_MIN_INTERVALS, + compute_peak_interval_kurtosis, + excess_kurtosis, + peak_intervals_by_day, ) -# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). -# The peak-identification constants are IMPORTED from PR-F, never redefined. -PEAK_INTERVAL_LOOKBACK_DAYS = 20 # trailing VALID trading-day pool window, includes d -PEAK_INTERVAL_MIN_INTERVALS = 20 # min pooled intervals for a finite kurtosis - -# Kurtosis needs at least 4 observations for the bias-corrected estimator to exist. -_KURTOSIS_MIN_N = 4 - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def excess_kurtosis(x: np.ndarray) -> float: - """Fisher excess, bias-corrected kurtosis of ``x`` — the ``pandas.kurt()`` convention. - - Equivalent to ``pandas.Series(x).kurt()`` and to - ``scipy.stats.kurtosis(x, fisher=True, bias=False)`` (locked by test), computed in - numpy because the caller evaluates it once per symbol-day:: - - G2 = n(n+1)(n-1)·M4 / ((n-2)(n-3)·M2²) - 3(n-1)² / ((n-2)(n-3)) - - with ``M2 = Σ(x-x̄)²`` and ``M4 = Σ(x-x̄)⁴`` (central sums, NOT means). Central sums - are accumulated after subtracting the mean, so no catastrophic power-sum - cancellation. - - Returns: - The excess kurtosis, or NaN when it is undefined: fewer than 4 observations, or - a zero-variance sample. Never raises, never returns ±inf. - """ - n = x.size - if n < _KURTOSIS_MIN_N: - return float("nan") - d = x - x.mean() - m2 = float(np.dot(d, d)) - if not m2 > 0.0: # zero variance (or a non-finite that made it NaN) -> undefined - return float("nan") - m4 = float(np.dot(d * d, d * d)) - left = n * (n + 1.0) * (n - 1.0) * m4 / ((n - 2.0) * (n - 3.0) * m2 * m2) - adj = 3.0 * (n - 1.0) ** 2 / ((n - 2.0) * (n - 3.0)) - return left - adj - - -def peak_intervals_by_day(work: pd.DataFrame) -> pd.Series: - """Trading-minute gaps between consecutive same-day peaks, per trade date. - - ``work`` is one symbol's frame as returned by - :func:`~data.clean.intraday_volume_prv.peak_mask_for_symbol` (sorted by - ``(trade_date, bar_end)`` with a boolean ``peak`` column). - - The interval unit is the TRADABLE SLOT POSITION difference inside the day's visible - bar sequence (PINNED §1 of the module docstring): position 0, 1, 2, ... is assigned - to the day's PIT-visible bars in time order, so consecutive tradable minutes are 1 - apart REGARDLESS of the lunch break sitting between them. Intervals are never taken - ACROSS days (the positions restart every trade date). - - Returns: - Series indexed by ``trade_date`` (every day present in ``work``, in order) whose - values are integer numpy arrays of that day's peak-to-peak intervals — an EMPTY - array for a day with fewer than 2 peaks (zero intervals, still a real day). - """ - # position within the day's visible bar sequence == "trading minute" coordinate - pos = work.groupby("trade_date", sort=False).cumcount().to_numpy() - peak = work["peak"].to_numpy(dtype=bool) - dates = work["trade_date"].to_numpy() - - out_dates: list[pd.Timestamp] = [] - out_intervals: list[np.ndarray] = [] - for day, idx in pd.Series(np.arange(len(work))).groupby(dates, sort=True): - sel = idx.to_numpy() - day_pos = pos[sel][peak[sel]] - out_dates.append(pd.Timestamp(day)) - out_intervals.append(np.diff(day_pos).astype(np.int64)) - return pd.Series(out_intervals, index=pd.Index(out_dates, name="trade_date")) - - -def _kurtosis_for_symbol( - g: pd.DataFrame, - *, - baseline_days: int, - baseline_min_obs: int, - sigma_k: float, - lookback_days: int, - min_valid_days: int, - min_classifiable: int, - min_intervals: int, -) -> tuple[list[pd.Timestamp], list[float]]: - """Daily peak-interval-kurtosis values for ONE symbol from its PIT-visible bars. - - Identifies the peaks with the REUSED :func:`peak_mask_for_symbol`, measures each - valid day's peak-to-peak trading-minute intervals, then pools the trailing - ``lookback_days`` valid days and reduces the pool to its excess kurtosis. No - cross-symbol leakage (``g`` is one symbol's slice) and no lookahead (the baseline is - strictly prior, the pool is trailing). - """ - work = peak_mask_for_symbol( - g, - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - ) - - classifiable_count = work.groupby("trade_date")["classifiable"].sum() - valid_days = classifiable_count.index[classifiable_count >= min_classifiable] - if len(valid_days) == 0: - return [], [] - - intervals = peak_intervals_by_day(work) - # Keep ONLY valid days, in order: a day below the classifiable floor contributes - # nothing and does not occupy a slot in the trailing window (same rule as PR-F). - valid = [pd.Timestamp(d) for d in sorted(valid_days)] - per_day = [intervals.loc[d] for d in valid] - - days: list[pd.Timestamp] = [] - values: list[float] = [] - for i, day in enumerate(valid): - window = per_day[max(0, i - lookback_days + 1) : i + 1] - days.append(day.normalize()) - if len(window) < min_valid_days: - values.append(float("nan")) - continue - pool = np.concatenate(window) if window else np.empty(0, dtype=np.int64) - # Gate 1: too few pooled intervals -> honest NaN (kurtosis is wild on small - # samples). Gate 2 (zero variance / n < 4) lives inside excess_kurtosis. - if pool.size < min_intervals: - values.append(float("nan")) - continue - values.append(excess_kurtosis(pool.astype(float))) - return days, values - - -def compute_peak_interval_kurtosis( - bars: pd.DataFrame, - *, - lookback_days: int = PEAK_INTERVAL_LOOKBACK_DAYS, - baseline_days: int = VOLUME_PRV_BASELINE_DAYS, - baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, - sigma_k: float = VOLUME_PRV_SIGMA_K, - min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - min_intervals: int = PEAK_INTERVAL_MIN_INTERVALS, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "peak_interval_kurtosis", -) -> pd.Series: - """PIT-safe daily "volume-peak interval kurtosis" factor from 1min ``bars``. - - Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, - identifies the volume peaks with the REUSED PR-F taxonomy, measures the - trading-minute gaps between consecutive same-day peaks, pools the trailing - ``lookback_days`` VALID days and returns the pool's Fisher excess (bias-corrected) - kurtosis. See the module docstring for the LOCKED definition and the two pinned - interpretations (interval unit, kurtosis convention). - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping - is strictly per symbol (no cross-symbol leakage). - lookback_days: trailing VALID trading-day window pooled for the kurtosis. - baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). - baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). - sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). - min_valid_days: minimum valid days in the trailing window for a finite value. - min_classifiable: a day is VALID iff it has at least this many classifiable bars. - min_intervals: minimum POOLED intervals for a finite kurtosis (>= 4, the point - below which the bias-corrected estimator does not exist). - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - name: the returned Series name (the factor-panel column name). - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily - factor value, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if baseline_days < 2: - # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is - # defined. - raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") - if baseline_min_obs < 2: - raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") - if sigma_k < 0.0: - raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") - if min_valid_days < 1: - raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") - if min_classifiable < 1: - raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") - if min_intervals < _KURTOSIS_MIN_N: - raise ValueError( - f"min_intervals must be >= {_KURTOSIS_MIN_N} (the bias-corrected kurtosis " - f"does not exist below that); got {min_intervals!r}." - ) - if len(bars) == 0: - return _empty_series(name) - - visible = prepare_visible_minute_bars(bars, decision_time=decision_time) - if visible.empty: - return _empty_series(name) - - index_tuples: list[tuple] = [] - values: list[float] = [] - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - days, vals = _kurtosis_for_symbol( - g.reset_index(drop=True), - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - lookback_days=lookback_days, - min_valid_days=min_valid_days, - min_classifiable=min_classifiable, - min_intervals=min_intervals, - ) - for day, val in zip(days, vals): - index_tuples.append((day, str(sym))) - values.append(val) - - if not index_tuples: - return _empty_series(name) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - - __all__ = [ "PEAK_INTERVAL_LOOKBACK_DAYS", "PEAK_INTERVAL_MIN_INTERVALS", diff --git a/data/clean/intraday_ridge_return.py b/data/clean/intraday_ridge_return.py index 1597dea..9594eb6 100644 --- a/data/clean/intraday_ridge_return.py +++ b/data/clean/intraday_ridge_return.py @@ -1,372 +1,19 @@ -"""RIDGE MINUTE-RETURN factor (PR-K). +"""D2 re-export shim — the PR-K factor math moved to ``factors.compute.minute``. -Reproduces the FIFTH factor of the Kaiyuan market-microstructure series #27 (开源证券 -《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, -§3): "计算过去 20 日量岭时点的累计收益作为量岭分钟收益因子，衡量个人投资者交易的收益贡献, -个人投资者交易的过度反应，导致量岭分钟收益因子为显著负向因子". - -Same MACHINE as PR-F / PR-H / PR-I / PR-J — what changes is the STATISTIC. The four prior -reproductions from this report covered a COUNT (weak), a TIMING moment (null) and two -PRICE LEVELS (both passed). This is the RETURN family, a fourth statistic type, and the -report's ONLY NEGATIVE peak/ridge/valley factor — so it also tests whether the SIGN -transfers along with the family. - -The classification is REUSED verbatim from :mod:`data.clean.intraday_volume_prv` -(``prepare_visible_minute_bars`` + ``peak_mask_for_symbol``, whose ``ridge`` column PR-J -already exposed) — same same-slot strictly-prior μ+kσ eruptive test, same classifiable -rule, same valid-day floor. Nothing about the taxonomy is re-implemented or modified here, -so the five factors can never drift apart. - -PINNED choices (deliberate and DISCLOSED, not tuned knobs; the report is silent on every -one of them, and each is reproduced on the factor spec so a reader sees what was assumed): - - 1. THE RIDGE MASK IS ``eruptive & ~peak`` (PR-J's, unchanged). A RIDGE (量岭) is an - eruptive minute that is NOT an isolated peak — wider than the literal "eruptive next - to an eruptive", because it also covers session-boundary eruptions and eruptions - whose neighbour is unclassifiable. It is the exact COMPLEMENT of PR-F's conservative - peak test, so ``valley | peak | ridge == classifiable`` stays an exact partition and - an isolated PEAK's return is counted on NEITHER side. - 2. THE MINUTE RETURN IS ``close_t / close_{t-1} - 1`` WITH A WITHIN-DAY LAG. The - predecessor is the previous VISIBLE bar of the SAME trade date, so each day's FIRST - visible bar has no return and never enters the sum. The lag deliberately does NOT - require exact 60s adjacency: a bar that opens a new session block (in practice 13:01, - after the lunch break) returns against the last bar before the gap, which is a - genuine price change of the stock over that interval. Dropping such bars instead - would silently discard the post-lunch minute whenever it is a ridge. The obvious - selection worry is defused by the taxonomy itself: the 13:01 slot is compared against - its OWN same-slot history, so a systematically busy post-lunch minute is not - systematically eruptive. - 3. RAW (UNADJUSTED) CLOSES. The cached minute bars are unadjusted, and that is CORRECT - here: a split/dividend adjustment factor is constant WITHIN a day, so it cancels - exactly in ``close_t / close_{t-1}``; and because the within-day lag already excludes - each day's first bar, no return ever straddles an ex-date boundary. (Same reasoning - PR-I / PR-J's ratios, PR-D's amplitude and I5b's price-limit checks rely on.) - 4. POSITIVE-CLOSE GUARD. A return is formed only when BOTH closes are finite and - strictly positive; otherwise the bar contributes nothing and is not counted towards - the ridge floor. The guard runs at the return step only — never before classification - — because PR-F's same-slot μ/σ baseline must stay bit-identical. - 5. THE DAILY AGGREGATE IS A SIMPLE SUM ``s_day = Σ r_t`` over the selected ridge bars, - NOT ``Π(1+r_t) - 1``. The report says only "累计收益" (cumulative return). Ridge - minutes are NON-CONTIGUOUS within the day — they are scattered eruptive moments, not - a held position — so a holding-period/compounding reading does not apply to them; and - at minute scale the two conventions differ negligibly anyway. The choice is therefore - the simple sum, disclosed rather than silently equated to the report's wording. - 6. THE TRAILING AGGREGATE IS ALSO A SUM. "过去 20 日累计" is read as the sum of the daily - sums over the trailing window, i.e. the factor accumulates across days rather than - averaging (which is what PR-J's ratio did). - 7. A RIDGE-BAR FLOOR OF 10, counted AFTER the return guard. Ridge bars are STRUCTURALLY - scarce: a minute must erupt (a minority event by construction, since the threshold is - its own strictly-prior same-slot μ+σ) AND fail the isolation test. This is the SAME - floor PR-J pinned for its ridge leg, so the two runs' coverage is directly - comparable, and the realized ridge-bar distribution plus the day-validity rate are - REPORTED by the runner rather than left implicit. - 8. BOTH THE SUM AND THE COUNT COVER THE PIT-VISIBLE WINDOW ONLY (09:31–14:50), not the - full session. A NECESSARY DEVIATION from the report, which uses the whole day: the - standing 14:50 decision cutoff truncates history days and the signal day identically, - and reading the closing auction would be lookahead at our decision time. - -Factor value: ``ridge_minute_return_20`` = the SUM of ``s_day`` over the symbol's most -recent ``lookback_days`` (=20) VALID trading days INCLUDING ``d``. A day is VALID iff it -has at least ``min_classifiable`` (=100) classifiable bars (PR-F's gate, unchanged) and at -least ``min_ridge_bars`` (=10) ridge bars carrying a valid return. Fewer than -``min_valid_days`` (=10) valid days in the trailing window -> NaN (honest missing; the -runner discloses the coverage). Values are emitted only on valid days. - -Pre-registered sign = -1. The report's full-market RankIC is -6.29% / RankICIR -3.55 (long -leg 7.47%/yr, long-short 14.98%/yr, IR 1.73, max drawdown 13.84%, monthly win rate 70.3%); -it gives NO CSI500 sub-domain figure for this factor, so none is quoted. Semantics per the -report: ridge minutes are retail follow-the-crowd trading, and their return contribution -measures that crowd's OVER-REACTION — the higher the accumulated ridge-minute return, the -more over-extended the stock and the worse it performs going forward. NOTE the report is a -MONTHLY, market-cap + industry neutral full-market series on Wind data while our eval cell -is CSI500 daily with industry + size neutralization, so its numbers are a LOOSE reference -only (disclosed, never mislabeled, never written in as an expected value). - -The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future bar -(invariant #1); it is a DAILY signal traded close-to-close from d+1. This module is -DATA-layer only: it does not fetch, does not touch factors / alpha / portfolio / runtime, -and never sees a token. +Single definition point from D2 on (design v3.2 §6.4): the ridge minute-return +factor lives in :mod:`factors.compute.minute.ridge_minute_return`. This shim +keeps every name the pre-D2 module exported importable from its old path; it is +deleted in D6d. Import-only by contract (locked by the shim purity test). """ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import DAILY_INDEX_NAMES, DEFAULT_DECISION_TIME -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars -from data.clean.intraday_volume_prv import ( - VOLUME_PRV_BASELINE_DAYS, - VOLUME_PRV_BASELINE_MIN_OBS, - VOLUME_PRV_MIN_CLASSIFIABLE, - VOLUME_PRV_MIN_VALID_DAYS, - VOLUME_PRV_SIGMA_K, - peak_mask_for_symbol, - prepare_visible_minute_bars, +from factors.compute.minute.ridge_minute_return import ( + DIAGNOSTIC_COLUMNS, + RIDGE_RETURN_LOOKBACK_DAYS, + RIDGE_RETURN_MIN_RIDGE_BARS, + compute_ridge_minute_return, + ridge_minute_return_by_day, ) -# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). -# The classification constants are IMPORTED from PR-F, never redefined. -RIDGE_RETURN_LOOKBACK_DAYS = 20 # trailing VALID trading-day SUM window, includes d -# The SAME scarcity floor PR-J pinned for its ridge leg (module docstring §7), so the two -# runs' ridge coverage is directly comparable. -RIDGE_RETURN_MIN_RIDGE_BARS = 10 # min RETURN-CARRYING ridge bars for a valid day - -# The extra 1min column this family needs on top of PR-F's (volume): the close, which -# turns consecutive bars into a minute return. -_CLOSE = "close" - -# Per-day diagnostic columns (the ridge-scarcity disclosure the task card requires). -# ``ridge_bars`` is every ridge minute; ``ridge_return_bars`` is the subset that carries a -# valid return — the one the floor actually gates on — so the attrition is visible. -DIAGNOSTIC_COLUMNS = ("classifiable_bars", "ridge_bars", "ridge_return_bars", "valid") - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def ridge_minute_return_by_day( - work: pd.DataFrame, - *, - min_ridge_bars: int = RIDGE_RETURN_MIN_RIDGE_BARS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - with_diagnostics: bool = False, -) -> pd.Series | tuple[pd.Series, pd.DataFrame]: - """Daily summed ridge-minute return for ONE symbol, on VALID days only. - - ``work`` is one symbol's frame as returned by - :func:`~data.clean.intraday_volume_prv.peak_mask_for_symbol`, which must have been - built from bars prepared with ``extra_columns=("close",)`` so the close is available - alongside the ``ridge`` / ``classifiable`` masks. ``peak_mask_for_symbol`` returns the - rows sorted by ``(trade_date, bar_end)``, which is exactly the order the WITHIN-DAY lag - needs. - - The minute return is ``close_t / close_{t-1} - 1`` against the previous VISIBLE bar of - the SAME day (PINNED §2), guarded so both closes are finite and strictly positive - (§4), and the day's value is the SIMPLE SUM of those returns over the ridge bars (§5) - — an isolated PEAK's return is never included (§1). - - Args: - work: one symbol's classified minute frame (see above). - min_ridge_bars: minimum ridge bars CARRYING A VALID RETURN for a valid day. - min_classifiable: minimum classifiable bars for a valid day (PR-F's gate). - with_diagnostics: also return the per-day bar-count frame, so the caller can - REPORT the ridge-scarcity distribution instead of only gating on it. - - Returns: - Series indexed by ``trade_date`` (ascending) holding the summed ridge-minute - return for the days that clear both gates — invalid days are ABSENT, not NaN, so - they do not occupy a slot in the caller's trailing window (the same rule PR-F / - PR-H / PR-I / PR-J use). With ``with_diagnostics=True``, a ``(daily, diagnostics)`` - pair where ``diagnostics`` is indexed by EVERY day present in ``work`` and carries - :data:`DIAGNOSTIC_COLUMNS`. - """ - close = work[_CLOSE].to_numpy(dtype=float) - # WITHIN-DAY lag: the previous visible bar of the SAME trade date. grouping by - # trade_date is what stops a return from ever crossing a day boundary, so each day's - # first visible bar gets a NaN predecessor and drops out below. - prev_close = ( - work.groupby("trade_date", sort=False)[_CLOSE].shift(1).to_numpy(dtype=float) - ) - # Positive-close guard: a non-finite or non-positive close on EITHER side carries no - # usable return. Applied HERE, at the return step, never before classification — - # PR-F's same-slot baseline must stay bit-identical. - has_return = ( - np.isfinite(close) & (close > 0.0) - & np.isfinite(prev_close) & (prev_close > 0.0) - ) - # Denominator neutralized where the guard failed, so no divide-by-zero / inf is ever - # formed; those entries are discarded by the np.where immediately after. - safe_prev = np.where(has_return, prev_close, 1.0) - ret = np.where(has_return, close / safe_prev - 1.0, 0.0) - - ridge = work["ridge"].to_numpy(dtype=bool) - ridge_return = ridge & has_return - - per_bar = pd.DataFrame( - { - "trade_date": work["trade_date"].to_numpy(), - # SIMPLE SUM of the selected returns (PINNED §5) — not Π(1+r)-1. - "ridge_return_sum": np.where(ridge_return, ret, 0.0), - "ridge_bars": ridge.astype(np.int64), - "ridge_return_bars": ridge_return.astype(np.int64), - "classifiable_bars": work["classifiable"] - .to_numpy(dtype=bool) - .astype(np.int64), - } - ) - agg = per_bar.groupby("trade_date", sort=True).sum() - - # Two validity gates (§ module docstring / task card §1.5): PR-F's classifiable floor - # (unchanged) and enough RETURN-CARRYING ridge bars. - valid = (agg["classifiable_bars"] >= min_classifiable) & ( - agg["ridge_return_bars"] >= min_ridge_bars - ) - ok = agg.loc[valid] - if ok.empty: - daily = pd.Series([], index=pd.DatetimeIndex([], name="trade_date"), dtype=float) - else: - daily = ok["ridge_return_sum"].astype(float) - - if not with_diagnostics: - return daily - diagnostics = agg[["classifiable_bars", "ridge_bars", "ridge_return_bars"]].copy() - diagnostics["valid"] = valid - return daily, diagnostics - - -def _ridge_return_sum_for_symbol( - g: pd.DataFrame, - *, - baseline_days: int, - baseline_min_obs: int, - sigma_k: float, - lookback_days: int, - min_valid_days: int, - min_classifiable: int, - min_ridge_bars: int, - collect_diagnostics: bool = False, -) -> tuple[list[pd.Timestamp], list[float], pd.DataFrame | None]: - """Daily ridge-minute-return values for ONE symbol from its PIT-visible bars. - - Classifies the minutes with the REUSED :func:`peak_mask_for_symbol`, reduces each valid - day to its summed ridge-minute return, then takes the trailing-``lookback_days``- - valid-day SUM. No cross-symbol leakage (``g`` is one symbol's slice) and no lookahead - (the baseline is strictly prior, the window is trailing). - """ - work = peak_mask_for_symbol( - g, - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - ) - result = ridge_minute_return_by_day( - work, - min_ridge_bars=min_ridge_bars, - min_classifiable=min_classifiable, - with_diagnostics=collect_diagnostics, - ) - if collect_diagnostics: - daily, diagnostics = result - else: - daily, diagnostics = result, None - if daily.empty: - return [], [], diagnostics - - # SUM over the trailing lookback_days VALID days (including d); NaN until - # min_valid_days valid days have accumulated. Emitted only on valid days. - rolled = daily.sort_index().rolling(lookback_days, min_periods=min_valid_days).sum() - days = [pd.Timestamp(d).normalize() for d in rolled.index] - return days, list(rolled.to_numpy(dtype=float)), diagnostics - - -def compute_ridge_minute_return( - bars: pd.DataFrame, - *, - lookback_days: int = RIDGE_RETURN_LOOKBACK_DAYS, - baseline_days: int = VOLUME_PRV_BASELINE_DAYS, - baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, - sigma_k: float = VOLUME_PRV_SIGMA_K, - min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - min_ridge_bars: int = RIDGE_RETURN_MIN_RIDGE_BARS, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "ridge_minute_return", - diagnostics_out: list | None = None, -) -> pd.Series: - """PIT-safe daily "ridge minute-return" factor from 1min ``bars``. - - Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, classifies - every visible minute with the REUSED PR-F taxonomy, sums each valid day's ridge-minute - returns, and returns the trailing-``lookback_days``-VALID-day SUM of those daily sums. - See the module docstring for the LOCKED definition and the eight pinned choices. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is - strictly per symbol (no cross-symbol leakage). - lookback_days: trailing VALID trading-day window summed (definition). - baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). - baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). - sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). - min_valid_days: minimum valid days in the trailing window for a finite value. - min_classifiable: a day needs at least this many classifiable bars (PR-F). - min_ridge_bars: a day needs at least this many ridge bars CARRYING A VALID RETURN. - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - name: the returned Series name (the factor-panel column name). - diagnostics_out: optional list the per-symbol day-level bar-count frames are - APPENDED to (each carries a ``symbol`` column), so a caller can report the - ridge-scarcity distribution. Purely observational — supplying it does not - change the returned factor. - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily factor - value, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if baseline_days < 2: - # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is - # defined. - raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") - if baseline_min_obs < 2: - raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") - if sigma_k < 0.0: - raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") - if min_valid_days < 1: - raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") - if min_classifiable < 1: - raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") - if min_ridge_bars < 1: - raise ValueError(f"min_ridge_bars must be >= 1; got {min_ridge_bars!r}.") - if len(bars) == 0: - return _empty_series(name) - - # extra_columns=("close",) is the ONLY difference from the PR-F / PR-H entry points: - # the close rides along on the surviving rows, and the truncation / volume guard / - # slot assignment are untouched. - visible = prepare_visible_minute_bars( - bars, decision_time=decision_time, extra_columns=(_CLOSE,) - ) - if visible.empty: - return _empty_series(name) - - collect = diagnostics_out is not None - index_tuples: list[tuple] = [] - values: list[float] = [] - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - days, vals, diagnostics = _ridge_return_sum_for_symbol( - g.reset_index(drop=True), - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - lookback_days=lookback_days, - min_valid_days=min_valid_days, - min_classifiable=min_classifiable, - min_ridge_bars=min_ridge_bars, - collect_diagnostics=collect, - ) - if collect and diagnostics is not None and not diagnostics.empty: - frame = diagnostics.copy() - frame[SYMBOL_LEVEL] = str(sym) - diagnostics_out.append(frame) - for day, val in zip(days, vals): - index_tuples.append((day, str(sym))) - values.append(val) - - if not index_tuples: - return _empty_series(name) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - - __all__ = [ "DIAGNOSTIC_COLUMNS", "RIDGE_RETURN_LOOKBACK_DAYS", diff --git a/data/clean/intraday_schema.py b/data/clean/intraday_schema.py index d716941..e5628c2 100644 --- a/data/clean/intraday_schema.py +++ b/data/clean/intraday_schema.py @@ -64,6 +64,21 @@ SYMBOL_LEVEL = "symbol" INTRADAY_INDEX_NAMES: list[str] = [TIME_LEVEL, SYMBOL_LEVEL] +# Daily-aggregation view of minute bars: the (date, symbol) grid every +# minute-DERIVED daily factor emits. Homed HERE since D2 (moved from +# data.clean.intraday_aggregate, which re-exports them) so that both the +# data-layer generic core (intraday_aggregate) and the factor math in +# factors.compute.minute can import one definition without an import cycle — +# the aggregate module re-exports the migrated MMP/jump math FROM factors, so +# the factor modules must never import the aggregate module back. +DATE_LEVEL = "date" +DAILY_INDEX_NAMES: list[str] = [DATE_LEVEL, SYMBOL_LEVEL] + +# Session time-of-day defaults shared by the PIT cutoff machinery (I3) and the +# minute factors (D2). Same single-home rationale as DAILY_INDEX_NAMES above. +DEFAULT_DECISION_TIME = "14:50:00" +DEFAULT_SESSION_OPEN = "09:30:00" + RAW_INTRADAY_FREQ = "1min" # The schema can represent derived coarser bars, but raw feeds/cache should only diff --git a/data/clean/intraday_valley_quantile.py b/data/clean/intraday_valley_quantile.py index c12434e..ff6e260 100644 --- a/data/clean/intraday_valley_quantile.py +++ b/data/clean/intraday_valley_quantile.py @@ -1,557 +1,23 @@ -"""VALLEY WEIGHTED-PRICE-QUANTILE factor (PR-L). +"""D2 re-export shim — the PR-L factor math moved to ``factors.compute.minute``. -Reproduces the SIXTH factor of the Kaiyuan market-microstructure series #27 (开源证券 -《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, -§5): "将『日内最高价、日内最低价、昨日收盘价』三者的最高值与最低值作为区间的价格高低位，并计算 -每日量谷成交量加权价格的相对分位点，将 20 日的分位点均值作为量谷加权价格分位点因子…考虑到因子 -计算中正向暴露 20 日反转因子，我们进一步对该因子做反转中性化处理". - -Same MACHINE as PR-F / PR-H / PR-I / PR-J / PR-K, a new STATISTIC FAMILY. The minute -classification is REUSED verbatim from :mod:`data.clean.intraday_volume_prv` -(``prepare_visible_minute_bars`` + ``peak_mask_for_symbol``) — same same-slot -strictly-prior μ+kσ eruptive test, same classifiable rule. A VALLEY (量谷) is exactly -that module's ``valley`` column. Where PR-F counted peaks, PR-H measured their timing, -PR-I / PR-J took price RATIOS and PR-K a RETURN, this module measures a price POSITION: -WHERE in the day's price range the valley VWAP sits. That is the scientific point of the -run — PR-I and PR-J both passed, and a position statistic tests whether that success came -from the ratio FORM or from price-level INFORMATION generally. - -Definition, per symbol and per panel date ``d`` (a DAILY signal traded close-to-close, -``is_intraday=False`` like all eight precedents): - - 1. PIT truncation (standing authorization): each day keeps only the 1min bars with - ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50). - 2. PRICE RANGE from three prices — the visible day's high, the visible day's low, and - the PREVIOUS trading day's close: ``hi = max(visible high, prev_close)`` and - ``lo = min(visible low, prev_close)``. - 3. VALLEY VWAP = ``Σamount(valley bars) / Σvolume(valley bars)`` (PR-I's identity). - 4. DAILY QUANTILE ``q_day = (valley VWAP - lo) / (hi - lo)``, deliberately NOT clipped. - 5. TRAILING MEAN ``qbar`` over the most recent ``lookback_days`` (=20) VALID days - including ``d``; fewer than ``min_valid_days`` (=10) valid days -> NaN. - 6. REVERSAL NEUTRALIZATION: per panel date, the cross-sectional OLS residual of - ``qbar`` on a 20-day reversal factor. - -PINNED choices (deliberate and DISCLOSED, reproduced on the factor spec): - - 1. PREV_CLOSE IS THE LAST VISIBLE (<=14:50) RAW CLOSE OF THE PREVIOUS TRADING DAY, not - that day's true 15:00 daily close. DEVIATION FROM THE REPORT, which uses the real - previous close. The previous day's 15:00 close is NOT itself a lookahead at a 14:50 - decision on day d — it is already past — so this is not a leakage fix; it is a - SINGLE-VISIBILITY-DEFINITION choice, so that every price entering the factor (the - range, the VWAP, the previous close) comes from one source under one cutoff rather - than mixing a minute-cache window with a daily-close feed. Disclosed, never silently - equated to the report's construction. - 2. NO CLIPPING of ``q_day``. If the valley VWAP falls outside ``[lo, hi]`` the range - construction is wrong, and clipping to [0,1] would hide exactly the defect worth - seeing. Under a correct range a traded price cannot escape it, so an out-of-range - value is a signal to investigate, not a value to sanitize. - 3. THE REVERSAL IS TAKEN AT T-1: ``rev20 = -(close_{d-1}/close_{d-21} - 1)`` on - FRONT-ADJUSTED (qfq) daily closes. The report's naive 20-day reversal uses - ``close_d``, which is 15:00 information and WOULD be a lookahead at our 14:50 - decision time. This is the direct application of the standing authorization that - implicitly-leaking fields be taken from pre-14:50 data. The closes come from the - panel the runner already loads — NO new data source. Front-adjusted closes are - REQUIRED here (unlike the within-day quantities below) because the ratio spans 20 - trading days and would otherwise break across any split or dividend. - 4. RAW (UNADJUSTED) MINUTE PRICES for the range and the VWAP, with ONE DISCLOSED - IMPERFECTION. PR-I/PR-J/PR-K could argue the adjustment factor cancels because both - of their legs sat inside ONE trading day. That argument does NOT fully hold here: - ``prev_close`` crosses the overnight boundary, so on an EX-DIVIDEND / SPLIT date the - previous raw close sits on the OLD (higher) price scale while the day's own bars sit - on the new one. The DIRECTION is therefore one-sided, not symmetric: ``prev_close`` - pulls ``hi`` UP (measured on real cache: 42 top-widening vs 4 bottom-widening among - true ex-dates), inflating the denominator ``hi - lo`` while the valley VWAP and - ``lo`` stay on the new scale — which compresses that day's ``q_day`` toward the LOW - END of an artificially wide range, NOT toward the middle. - MEASURED on the 30-name prototype panel: 15.6% of valid days show ANY range - widening from ``prev_close``, but that is overwhelmingly ordinary overnight gapping, - which is the INTENDED behaviour (the report's range includes the previous close - precisely so a gap counts); only 0.73% of valid days are true ex-dates, and only - ~0.21% are BOTH a true ex-date AND actually range-distorted — before the 20-day mean - dilutes it further. DISCLOSED here rather than silently corrected, because - correcting it would mix an adjusted price into an otherwise raw, single-visibility - construction. - 5. THE RANGE USES A PRICE GUARD, THE VWAP USES A TRADE GUARD. A bar enters the - high/low range if its prices are finite with ``high >= low > 0`` (a zero-volume - minute still quotes a real price); it enters the VWAP only if BOTH ``volume`` and - ``amount`` are finite and strictly positive (PR-I's positive-trade guard). Both - guards run at the summation step only, never before classification, so PR-F's - same-slot baseline — and therefore all five merged factors — stay bit-identical. - 6. DAY VALIDITY needs all of: >= ``min_classifiable`` (=100) classifiable bars (PR-F's - gate, unchanged), >= ``min_valley_bars`` (=20) TRADABLE valley bars (PR-I's floor), - strictly positive valley volume, an available ``prev_close``, and ``hi > lo``. A - symbol's FIRST visible day therefore never produces a value (no previous day). - 7. THE RESIDUALIZATION IS PER DATE ON THE COVERED CROSS-SECTION, mirroring PR-G's - per-date cross-sectional post-processing. A date with fewer than - ``min_cross_section`` (=10) symbols carrying BOTH ``qbar`` and ``rev20`` is all-NaN; - a symbol missing ``rev20`` is dropped from the regression and its residual is NaN - (never a silent zero fill); a degenerate cross-section (zero-variance ``rev20``, - which cannot identify a slope) is likewise NaN rather than an unresidualized - ``qbar`` passed off as neutralized. - 8. EVERYTHING SPANS THE PIT-VISIBLE WINDOW 09:31-14:50 ONLY, while the report uses the - full session. Reading the closing auction would be lookahead at our decision time. - Disclosed, not silently equated. - -Pre-registered sign = +1. The report's full-market RankIC is +6.34% / RankICIR 4.32 -(long leg 13.1%/yr, long-short 20.22%/yr, IR 3.29, max drawdown 10.18%, monthly win rate -80.4%); its CSI500 sub-domain long-short is 11.71% / IR 1.76, the closest comparable to -our eval cell. Semantics per the report: a HIGH relative position of the calm-minute -price within the day's range means the informed, unhurried part of the day traded near -the top of the range -> higher future return. NOTE the report is a MONTHLY, market-cap + -industry neutral full-market series on Wind data while our eval cell is CSI500 daily with -industry + size neutralization, so its numbers are a LOOSE reference only (disclosed, -never mislabeled, never written in as an expected value). - -The value at ``d`` uses only bars at dates <= d and closes at dates <= d-1, so a factor -value never sees a future bar (invariant #1). This module is DATA-layer only: it does not -fetch, does not touch factors / alpha / portfolio / runtime, and never sees a token. - -The heavy per-symbol work (steps 1-5) is split from the cross-sectional residualization -(step 6) on purpose — the same split PR-G uses: the runner streams one symbol at a time -through :func:`compute_valley_price_quantile_stats` (memory-bounded), assembles the -full-universe ``qbar`` panel, computes ``rev20`` once from the daily panel, then calls -:func:`residualize_on_reversal` ONCE. :func:`compute_valley_price_quantile` chains the -three for a single-call path. +Single definition point from D2 on (design v3.2 §6.4): the valley +weighted-price-quantile factor lives in +:mod:`factors.compute.minute.valley_price_quantile`. This shim keeps every name +the pre-D2 module exported importable from its old path; it is deleted in D6d. +Import-only by contract (locked by the shim purity test). """ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import ( - DAILY_INDEX_NAMES, - DATE_LEVEL, - DEFAULT_DECISION_TIME, +from factors.compute.minute.valley_price_quantile import ( + VALLEY_QUANTILE_LOOKBACK_DAYS, + VALLEY_QUANTILE_MIN_CROSS_SECTION, + VALLEY_QUANTILE_MIN_VALLEY_BARS, + VALLEY_QUANTILE_REVERSAL_DAYS, + compute_valley_price_quantile, + compute_valley_price_quantile_stats, + residualize_on_reversal, + reversal_20, + valley_price_quantile_by_day, ) -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars -from data.clean.intraday_volume_prv import ( - VOLUME_PRV_BASELINE_DAYS, - VOLUME_PRV_BASELINE_MIN_OBS, - VOLUME_PRV_MIN_CLASSIFIABLE, - VOLUME_PRV_MIN_VALID_DAYS, - VOLUME_PRV_SIGMA_K, - peak_mask_for_symbol, - prepare_visible_minute_bars, -) - -# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). -# The classification constants are IMPORTED from PR-F, never redefined. -VALLEY_QUANTILE_LOOKBACK_DAYS = 20 # trailing VALID trading-day mean window, includes d -VALLEY_QUANTILE_MIN_VALLEY_BARS = 20 # min TRADABLE valley bars for a valid day (PR-I's) -VALLEY_QUANTILE_REVERSAL_DAYS = 20 # span of the reversal factor neutralized against -VALLEY_QUANTILE_MIN_CROSS_SECTION = 10 # min paired cross-section for a finite residual - -# The extra 1min columns this family needs on top of PR-F's (volume): the traded value -# for the VWAP, and the prices that form the day's range / the previous close. -_EXTRA_COLUMNS = ("amount", "high", "low", "close") - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def valley_price_quantile_by_day( - work: pd.DataFrame, - *, - min_valley_bars: int = VALLEY_QUANTILE_MIN_VALLEY_BARS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, -) -> pd.Series: - """Daily valley-VWAP price quantile for ONE symbol, on VALID days only. - - ``work`` is one symbol's frame as returned by - :func:`~data.clean.intraday_volume_prv.peak_mask_for_symbol`, built from bars - prepared with ``extra_columns=("amount", "high", "low", "close")`` so the traded - value and the prices ride alongside the ``valley`` / ``classifiable`` masks. - - Implements pinned choices 1-6 of the module docstring: the range is - ``[min(visible low, prev_close), max(visible high, prev_close)]`` where ``prev_close`` - is the previous trading day's LAST VISIBLE usable close; the valley VWAP is - ``Σamount/Σvolume`` over guarded valley bars; the quantile is NOT clipped. - - Returns: - Series indexed by ``trade_date`` (ascending) holding ``q_day`` for the days that - clear every validity gate — invalid days are ABSENT, not NaN, so they do not - occupy a slot in the caller's trailing window (the rule PR-F..PR-K use). - """ - # Chronological within the symbol so "the day's last visible close" is well defined. - ordered = work.sort_values("bar_end", kind="mergesort") - - vol = ordered["volume"].to_numpy(dtype=float) - amt = ordered["amount"].to_numpy(dtype=float) - high = ordered["high"].to_numpy(dtype=float) - low = ordered["low"].to_numpy(dtype=float) - close = ordered["close"].to_numpy(dtype=float) - - # PINNED §5: a TRADE guard for the VWAP, a PRICE guard for the range. A zero-volume - # minute still quotes a real price, so it may set the day's high/low while - # contributing nothing to the volume-weighted price. - tradable = np.isfinite(vol) & (vol > 0.0) & np.isfinite(amt) & (amt > 0.0) - priced = np.isfinite(high) & np.isfinite(low) & (low > 0.0) & (high >= low) - valley = ordered["valley"].to_numpy(dtype=bool) & tradable - - per_bar = pd.DataFrame( - { - "trade_date": ordered["trade_date"].to_numpy(), - "valley_amt": np.where(valley, amt, 0.0), - "valley_vol": np.where(valley, vol, 0.0), - "valley_bars": valley.astype(np.int64), - "classifiable_bars": ordered["classifiable"] - .to_numpy(dtype=bool) - .astype(np.int64), - # +/-inf so a day with no priced bar reduces to a non-finite range and fails - # the hi > lo gate rather than fabricating one. - "day_high": np.where(priced, high, -np.inf), - "day_low": np.where(priced, low, np.inf), - } - ) - grouped = per_bar.groupby("trade_date", sort=True) - agg = grouped[ - ["valley_amt", "valley_vol", "valley_bars", "classifiable_bars"] - ].sum() - agg["day_high"] = grouped["day_high"].max() - agg["day_low"] = grouped["day_low"].min() - - # PINNED §1: prev_close = the previous trading day's LAST VISIBLE usable close. Taken - # from the SAME visible (<=14:50) bars as everything else — a post-cutoff bar on the - # previous day can never become it. - usable_close = np.isfinite(close) & (close > 0.0) - closes = pd.DataFrame( - { - "trade_date": ordered["trade_date"].to_numpy()[usable_close], - "close": close[usable_close], - } - ) - last_close = closes.groupby("trade_date", sort=True)["close"].last() - # shift(1) over the symbol's own trading days: day d takes day d-1's last visible - # close, and the FIRST day has none (NaN) -> that day is invalid. - prev_close = last_close.reindex(agg.index).shift(1) - - pc = prev_close.to_numpy(dtype=float) - hi = np.maximum(agg["day_high"].to_numpy(dtype=float), pc) - lo = np.minimum(agg["day_low"].to_numpy(dtype=float), pc) - - valid = ( - (agg["classifiable_bars"].to_numpy() >= min_classifiable) - & (agg["valley_bars"].to_numpy() >= min_valley_bars) - & (agg["valley_vol"].to_numpy(dtype=float) > 0.0) - & np.isfinite(pc) - & np.isfinite(hi) - & np.isfinite(lo) - & (hi > lo) - ) - if not valid.any(): - return pd.Series( - [], index=pd.DatetimeIndex([], name="trade_date"), dtype=float - ) - - valley_vwap = ( - agg["valley_amt"].to_numpy(dtype=float)[valid] - / agg["valley_vol"].to_numpy(dtype=float)[valid] - ) - # PINNED §2: NOT clipped — an out-of-range value must be visible, not sanitized. - q = (valley_vwap - lo[valid]) / (hi[valid] - lo[valid]) - return pd.Series(q, index=agg.index[valid], dtype=float) - - -def _quantile_mean_for_symbol( - g: pd.DataFrame, - *, - baseline_days: int, - baseline_min_obs: int, - sigma_k: float, - lookback_days: int, - min_valid_days: int, - min_classifiable: int, - min_valley_bars: int, -) -> tuple[list[pd.Timestamp], list[float]]: - """Trailing-mean daily quantile for ONE symbol from its PIT-visible bars. - - Classifies the minutes with the REUSED :func:`peak_mask_for_symbol`, reduces each - valid day to its price quantile, then takes the trailing-``lookback_days``-valid-day - mean. No cross-symbol leakage (``g`` is one symbol's slice) and no lookahead (the - baseline is strictly prior, the mean window is trailing, the previous close is the - previous day's). - """ - work = peak_mask_for_symbol( - g, - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - ) - q = valley_price_quantile_by_day( - work, min_valley_bars=min_valley_bars, min_classifiable=min_classifiable - ) - if q.empty: - return [], [] - rolled = q.sort_index().rolling(lookback_days, min_periods=min_valid_days).mean() - days = [pd.Timestamp(d).normalize() for d in rolled.index] - return days, list(rolled.to_numpy(dtype=float)) - - -def compute_valley_price_quantile_stats( - bars: pd.DataFrame, - *, - lookback_days: int = VALLEY_QUANTILE_LOOKBACK_DAYS, - baseline_days: int = VOLUME_PRV_BASELINE_DAYS, - baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, - sigma_k: float = VOLUME_PRV_SIGMA_K, - min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - min_valley_bars: int = VALLEY_QUANTILE_MIN_VALLEY_BARS, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "valley_price_quantile_raw", -) -> pd.Series: - """RAW trailing-mean price quantile panel (steps 1-5; NO neutralization yet). - - Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, - classifies every visible minute with the REUSED PR-F taxonomy, computes each valid - day's quantile of the valley VWAP within the prev-close-extended price range, and - returns the trailing-``lookback_days``-VALID-day mean of that quantile. - - The REVERSAL NEUTRALIZATION (step 6) is deliberately NOT applied here — it needs the - full-universe panel plus daily closes and is done by :func:`residualize_on_reversal`. - The returned values are therefore the RAW ``qbar``, which is exactly what the - prototype's before/after neutralization comparison needs. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is - strictly per symbol (no cross-symbol leakage). - lookback_days: trailing VALID trading-day window averaged (definition). - baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). - baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). - sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). - min_valid_days: minimum valid days in the trailing window for a finite value. - min_classifiable: a day needs at least this many classifiable bars (PR-F). - min_valley_bars: a day needs at least this many TRADABLE valley bars. - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - name: the returned Series name. - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the raw - trailing-mean quantile, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if baseline_days < 2: - # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is - # defined. - raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") - if baseline_min_obs < 2: - raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") - if sigma_k < 0.0: - raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") - if min_valid_days < 1: - raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") - if min_classifiable < 1: - raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") - if min_valley_bars < 1: - raise ValueError(f"min_valley_bars must be >= 1; got {min_valley_bars!r}.") - if len(bars) == 0: - return _empty_series(name) - - visible = prepare_visible_minute_bars( - bars, decision_time=decision_time, extra_columns=_EXTRA_COLUMNS - ) - if visible.empty: - return _empty_series(name) - - index_tuples: list[tuple] = [] - values: list[float] = [] - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - days, vals = _quantile_mean_for_symbol( - g.reset_index(drop=True), - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - lookback_days=lookback_days, - min_valid_days=min_valid_days, - min_classifiable=min_classifiable, - min_valley_bars=min_valley_bars, - ) - for day, val in zip(days, vals): - index_tuples.append((day, str(sym))) - values.append(val) - - if not index_tuples: - return _empty_series(name) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - - -def reversal_20( - closes: pd.DataFrame | pd.Series, - *, - days: int = VALLEY_QUANTILE_REVERSAL_DAYS, - close_col: str = "close", - name: str = "rev20", -) -> pd.Series: - """T-1-based ``days``-day reversal factor from FRONT-ADJUSTED daily closes. - - ``rev20(d) = -(close_{d-1} / close_{d-(days+1)} - 1)`` — PINNED §3 of the module - docstring. The report's naive form uses ``close_d``, which is 15:00 information and - would be a LOOKAHEAD at our 14:50 decision time; taking the whole ratio one day - earlier makes every input strictly visible when the factor is formed. - - The closes MUST be front-adjusted: the ratio spans ``days`` trading days and a raw - series would break across any split or dividend inside the window. - - Args: - closes: ``MultiIndex(date, symbol)`` daily panel (DataFrame with ``close_col``, - or a Series of closes). Typically ``panel[["close"]]`` from the runner — - NO new data source. - days: reversal span in trading days (definition, not a tuned knob). - close_col: the close column name when ``closes`` is a DataFrame. - name: the returned Series name. - - Returns: - ``MultiIndex(date, symbol)`` Series over the SAME rows as the input, NaN where - the T-1 window is incomplete or a close is non-positive / non-finite (honest - missing — never fabricated). Pure: never mutates ``closes``. - """ - if days < 1: - raise ValueError(f"days must be >= 1; got {days!r}.") - if isinstance(closes, pd.DataFrame): - if close_col not in closes.columns: - raise ValueError( - f"reversal_20 needs a '{close_col}' column; got {list(closes.columns)}." - ) - series = closes[close_col] - else: - series = closes - if series.empty: - return _empty_series(name) - - s = series.astype(float).sort_index() - # Non-positive / non-finite closes cannot form a ratio; blank them BEFORE shifting so - # a bad print poisons only the windows that actually touch it. - clean = s.where(np.isfinite(s.to_numpy(dtype=float)) & (s.to_numpy(dtype=float) > 0.0)) - by_symbol = clean.groupby(level=SYMBOL_LEVEL, sort=False) - prior = by_symbol.shift(1) # close_{d-1} - base = by_symbol.shift(days + 1) # close_{d-(days+1)} - rev = -(prior / base - 1.0) - return rev.rename(name) - - -def residualize_on_reversal( - qbar: pd.Series, - rev: pd.Series, - *, - min_cross_section: int = VALLEY_QUANTILE_MIN_CROSS_SECTION, - name: str = "valley_price_quantile", -) -> pd.Series: - """Per-date cross-sectional OLS residual of ``qbar`` on ``rev`` (step 6). - - The report neutralizes this factor against the 20-day reversal because the raw - quantile is positively exposed to it. For each panel date the cross-section is the - symbols carrying BOTH a finite ``qbar`` and a finite ``rev``; the residual is - ``qbar - (a + b*rev)`` from an intercept OLS fit on that cross-section. Mirrors PR-G's - per-date cross-sectional post-processing, and follows the project's neutralization - convention that an under-determined cross-section yields NaN rather than a fabricated - value. - - PINNED §7: a date with fewer than ``min_cross_section`` paired symbols is all-NaN; a - symbol missing ``rev`` is dropped from the fit and its residual is NaN (never a silent - zero fill); a degenerate cross-section (zero-variance ``rev``, which cannot identify a - slope) is likewise NaN rather than an unresidualized ``qbar`` passed off as - neutralized. - - Args: - qbar: ``MultiIndex(date, symbol)`` raw trailing-mean quantile panel. - rev: ``MultiIndex(date, symbol)`` reversal panel (from :func:`reversal_20`). - min_cross_section: minimum paired cross-section for a finite residual. - name: the returned Series name (the factor-panel column name). - - Returns: - A Series over EXACTLY ``qbar``'s rows, in ``qbar``'s order, holding the residual - (or NaN). Pure: never mutates its inputs. - """ - if min_cross_section < 2: - # Need >= 2 cross-section members before an intercept + slope fit is determined. - raise ValueError( - f"min_cross_section must be >= 2; got {min_cross_section!r}." - ) - if qbar is None or qbar.empty: - return _empty_series(name) - - y_all = qbar.to_numpy(dtype=float) - # Align the reversal onto qbar's rows WITHOUT reordering them (the output must line - # up with the caller's panel row-for-row). - x_all = rev.reindex(qbar.index).to_numpy(dtype=float) - out = np.full(y_all.shape, np.nan, dtype=float) - - dates = qbar.index.get_level_values(DATE_LEVEL).to_numpy() - paired = np.isfinite(y_all) & np.isfinite(x_all) - for date in pd.unique(dates): - on_date = dates == date - fit_rows = on_date & paired - n = int(fit_rows.sum()) - if n < min_cross_section: - continue # honest missing: the whole date stays NaN - x = x_all[fit_rows] - y = y_all[fit_rows] - x_mean = x.mean() - sxx = float(((x - x_mean) ** 2).sum()) - if not np.isfinite(sxx) or sxx <= 0.0: - continue # degenerate: no slope is identified -> NaN, not a raw qbar - y_mean = y.mean() - slope = float(((x - x_mean) * (y - y_mean)).sum()) / sxx - out[fit_rows] = y - (y_mean + slope * (x - x_mean)) - - return pd.Series(out, index=qbar.index, name=name) - - -def compute_valley_price_quantile( - bars: pd.DataFrame, - closes: pd.DataFrame | pd.Series, - *, - lookback_days: int = VALLEY_QUANTILE_LOOKBACK_DAYS, - baseline_days: int = VOLUME_PRV_BASELINE_DAYS, - baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, - sigma_k: float = VOLUME_PRV_SIGMA_K, - min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - min_valley_bars: int = VALLEY_QUANTILE_MIN_VALLEY_BARS, - min_cross_section: int = VALLEY_QUANTILE_MIN_CROSS_SECTION, - reversal_days: int = VALLEY_QUANTILE_REVERSAL_DAYS, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "valley_price_quantile", -) -> pd.Series: - """Single-call factor: raw quantile stats (1-5) then reversal neutralization (6). - - Convenience that chains :func:`compute_valley_price_quantile_stats`, - :func:`reversal_20` and :func:`residualize_on_reversal`. The runner does NOT use this - (it needs the per-symbol stats loop for memory-boundedness, and it reports the - before/after-neutralization diagnostics) but tests and any all-in-memory caller can. - - ``closes`` must be the FRONT-ADJUSTED daily close panel — see PINNED §3. Pure: never - mutates its inputs. - """ - stats = compute_valley_price_quantile_stats( - bars, - lookback_days=lookback_days, - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - min_valid_days=min_valid_days, - min_classifiable=min_classifiable, - min_valley_bars=min_valley_bars, - decision_time=decision_time, - ) - if stats.empty: - return _empty_series(name) - rev = reversal_20(closes, days=reversal_days) - return residualize_on_reversal( - stats, rev, min_cross_section=min_cross_section, name=name - ) - __all__ = [ "VALLEY_QUANTILE_LOOKBACK_DAYS", diff --git a/data/clean/intraday_valley_ridge_vwap.py b/data/clean/intraday_valley_ridge_vwap.py index 4234957..ea9961c 100644 --- a/data/clean/intraday_valley_ridge_vwap.py +++ b/data/clean/intraday_valley_ridge_vwap.py @@ -1,375 +1,21 @@ -"""VALLEY/RIDGE VWAP-RATIO factor (PR-J). +"""D2 re-export shim — the PR-J factor math moved to ``factors.compute.minute``. -Reproduces the FOURTH factor of the Kaiyuan market-microstructure series #27 (开源证券 -《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, -§7.1): "计算每日量谷成交量加权价格，与量岭成交量加权价格做比，并计算 20 日价格比均值作为 -谷岭加权价格比因子". - -Same MACHINE as PR-F / PR-H / PR-I and the same VWAP identity as PR-I — what changes is -the DENOMINATOR. PR-I divided the valley VWAP by the WHOLE VISIBLE DAY's VWAP; this -factor divides it by the RIDGE VWAP, so the two behavioural groups of the report's -peak/ridge/valley taxonomy are contrasted head-on rather than one being compared against -an aggregate that contains it. That makes PR-J the natural robustness test of PR-I: if -the price-level signal survives the swap, it is a property of the FAMILY; if it does not, -PR-I depended on its specific denominator. Either outcome is a legitimate result. - -The classification is REUSED verbatim from :mod:`data.clean.intraday_volume_prv` -(``prepare_visible_minute_bars`` + ``peak_mask_for_symbol``) — same same-slot -strictly-prior μ+kσ eruptive test, same classifiable rule, same valid-day floor. Nothing -about the taxonomy is re-implemented here, so the four factors can never drift apart. - -PINNED choices (deliberate and DISCLOSED, not tuned knobs; reproduced on the factor spec -so a reader sees exactly what was assumed): - - 1. THE RIDGE MASK IS ``eruptive & ~peak``. A RIDGE (量岭) is an eruptive minute that is - NOT an isolated peak. This is WIDER than the literal "eruptive next to an eruptive": - it also covers the session-boundary eruptions (a neighbour missing across the lunch - break / the cutoff / a data gap) and the eruptions whose neighbour is unclassifiable. - The rule is the exact COMPLEMENT of PR-F's deliberately conservative peak test, so - the taxonomy stays a clean partition — ``valley | peak | ridge == classifiable`` — - and no eruptive bar is silently dropped on either side. Disclosed rather than - silently equated to the report's wording. - 2. VWAP VIA THE AGGREGATION IDENTITY (same as PR-I). A bar's volume-weighted price is - ``p = amount / volume``, so a set's volume-weighted price ``Σ(p_i·v_i) / Σv_i`` - collapses EXACTLY to ``Σamount / Σvolume``. Both legs use that identity: valley VWAP - = Σamount(valley bars)/Σvolume(valley bars), ridge VWAP = Σamount(ridge - bars)/Σvolume(ridge bars). - 3. POSITIVE-TRADE GUARD. A bar with non-finite or non-positive ``volume`` OR ``amount`` - carries no price information, so it is dropped from BOTH sums. The guard runs at the - summation step only — never before classification — because the same-slot μ/σ - baseline is PR-F's and must stay bit-identical. A zero-volume minute is therefore - still classified (and still a valley) but contributes nothing to either VWAP. - 4. RAW (UNADJUSTED) PRICES. The cached minute bars are unadjusted. Both legs are sums - over the SAME trading day, and a split/dividend adjustment factor is constant within - a day, so it cancels exactly in the ratio — no adjustment is needed (the same - reasoning PR-I, PR-D's amplitude and I5b's price-limit checks rely on). - 5. BOTH LEGS COVER THE PIT-VISIBLE WINDOW ONLY (09:31–14:50), not the full session. - A NECESSARY DEVIATION from the report, which uses the whole day: the standing 14:50 - decision cutoff truncates history days and the signal day identically, and reading - the closing auction would be lookahead at our decision time. - 6. AN ASYMMETRIC BAR FLOOR — ``min_valley_bars`` (=20) but ``min_ridge_bars`` (=10). - Ridge bars are STRUCTURALLY far scarcer than valley bars: a minute must erupt (a - minority event by construction, since the threshold is its own strictly-prior - same-slot μ+σ) AND fail the isolation test. Holding the ridge leg to the valley - leg's floor would throw away a large share of otherwise sound days and bias the - surviving sample towards unusually turbulent sessions. The floor is therefore - LOWERED, deliberately, and both the realized ridge-bar distribution and the - resulting day-validity rate are REPORTED (``with_diagnostics=True``) rather than - left implicit — if coverage comes out materially worse than PR-I's, that is a - finding to surface, not to hide. - 7. BOTH BAR COUNTS ARE TAKEN AFTER THE GUARD, so a day cannot qualify on the strength - of bars that traded nothing. - -Factor value: ``valley_ridge_vwap_ratio_20`` = the MEAN of the daily ratio -``valley VWAP / ridge VWAP`` over the symbol's most recent ``lookback_days`` (=20) VALID -trading days INCLUDING ``d``. A day is VALID iff it clears four gates: at least -``min_classifiable`` (=100) classifiable bars (PR-F's gate, unchanged), at least -``min_valley_bars`` (=20) tradable valley bars, at least ``min_ridge_bars`` (=10) -tradable ridge bars, and strictly positive volume in BOTH denominators. Fewer than -``min_valid_days`` (=10) valid days in the trailing window -> NaN (honest missing; the -runner discloses the coverage). Values are emitted only on valid days. - -Pre-registered sign = +1. The report's full-market RankIC is +6.98% / RankICIR 3.56 -(long-short 15.83%/yr, IR 1.83, monthly win rate 72.3%, only 2023 negative in 13 years); -its CSI500 sub-domain long-short is 10.49% / IR 1.34, the closest comparable to our eval -cell. Semantics per the report: a HIGH valley/ridge price ratio means retail -over-reaction pushed the eruptive minutes' price DOWN relative to the calm ones, so the -stock is depressed and performs better going forward. NOTE the report is a MONTHLY, -market-cap + industry neutral full-market series on Wind data while our eval cell is -CSI500 daily with industry + size neutralization, so its numbers are a LOOSE reference -only (disclosed, never mislabeled, never written in as an expected value). - -The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future -bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. This module is -DATA-layer only: it does not fetch, does not touch factors / alpha / portfolio / -runtime, and never sees a token. +Single definition point from D2 on (design v3.2 §6.4): the valley/ridge +VWAP-ratio factor lives in +:mod:`factors.compute.minute.valley_ridge_vwap_ratio`. This shim keeps every +name the pre-D2 module exported importable from its old path; it is deleted in +D6d. Import-only by contract (locked by the shim purity test). """ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import DAILY_INDEX_NAMES, DEFAULT_DECISION_TIME -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars -from data.clean.intraday_volume_prv import ( - VOLUME_PRV_BASELINE_DAYS, - VOLUME_PRV_BASELINE_MIN_OBS, - VOLUME_PRV_MIN_CLASSIFIABLE, - VOLUME_PRV_MIN_VALID_DAYS, - VOLUME_PRV_SIGMA_K, - peak_mask_for_symbol, - prepare_visible_minute_bars, +from factors.compute.minute.valley_ridge_vwap_ratio import ( + DIAGNOSTIC_COLUMNS, + VALLEY_RIDGE_LOOKBACK_DAYS, + VALLEY_RIDGE_MIN_RIDGE_BARS, + VALLEY_RIDGE_MIN_VALLEY_BARS, + compute_valley_ridge_vwap_ratio, + valley_ridge_vwap_ratio_by_day, ) -# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). -# The classification constants are IMPORTED from PR-F, never redefined. -VALLEY_RIDGE_LOOKBACK_DAYS = 20 # trailing VALID trading-day mean window, includes d -VALLEY_RIDGE_MIN_VALLEY_BARS = 20 # min TRADABLE valley bars for a valid day -# PINNED LOWER than the valley floor (module docstring §6): ridge bars are structurally -# far scarcer, so holding both legs to 20 would bias the surviving sample. -VALLEY_RIDGE_MIN_RIDGE_BARS = 10 # min TRADABLE ridge bars for a valid day - -# The extra 1min column this family needs on top of PR-F's (volume): the traded value, -# which turns the bar set into a volume-weighted price via Σamount/Σvolume. -_AMOUNT = "amount" - -# Per-day diagnostic columns (the ridge-scarcity disclosure the task card requires). -DIAGNOSTIC_COLUMNS = ("classifiable_bars", "valley_bars", "ridge_bars", "valid") - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def valley_ridge_vwap_ratio_by_day( - work: pd.DataFrame, - *, - min_valley_bars: int = VALLEY_RIDGE_MIN_VALLEY_BARS, - min_ridge_bars: int = VALLEY_RIDGE_MIN_RIDGE_BARS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - with_diagnostics: bool = False, -) -> pd.Series | tuple[pd.Series, pd.DataFrame]: - """Daily ``valley VWAP / ridge VWAP`` ratio for ONE symbol, on VALID days only. - - ``work`` is one symbol's frame as returned by - :func:`~data.clean.intraday_volume_prv.peak_mask_for_symbol`, which must have been - built from bars prepared with ``extra_columns=("amount",)`` so the traded value is - available alongside the ``valley`` / ``ridge`` / ``classifiable`` masks. - - Both VWAPs use the ``Σamount / Σvolume`` aggregation identity (PINNED §2 of the - module docstring). The POSITIVE-TRADE GUARD (§3) drops non-finite / non-positive - volume or amount bars from both sums. Unlike PR-I, the denominator is NOT the whole - visible day but the RIDGE bars alone (§1: ``eruptive & ~peak``), so an isolated PEAK - contributes to NEITHER leg. - - Args: - work: one symbol's classified minute frame (see above). - min_valley_bars: minimum TRADABLE valley bars for a valid day. - min_ridge_bars: minimum TRADABLE ridge bars for a valid day (PINNED lower than - the valley floor — see §6). - min_classifiable: minimum classifiable bars for a valid day (PR-F's gate). - with_diagnostics: also return the per-day bar-count frame, so the caller can - REPORT the ridge-scarcity distribution instead of only gating on it. - - Returns: - Series indexed by ``trade_date`` (ascending) holding the ratio for the days that - clear all gates — invalid days are ABSENT, not NaN, so they do not occupy a slot - in the caller's trailing window (the same rule PR-F / PR-H / PR-I use). With - ``with_diagnostics=True``, a ``(ratio, diagnostics)`` pair where ``diagnostics`` - is indexed by EVERY day present in ``work`` and carries - :data:`DIAGNOSTIC_COLUMNS`. - """ - vol = work["volume"].to_numpy(dtype=float) - amt = work[_AMOUNT].to_numpy(dtype=float) - # Positive-trade guard: no price information in a bar that traded nothing (or whose - # fields are non-finite / negative). Applied HERE, at the summation step, never - # before classification — PR-F's same-slot baseline must stay bit-identical. - tradable = np.isfinite(vol) & (vol > 0.0) & np.isfinite(amt) & (amt > 0.0) - valley = work["valley"].to_numpy(dtype=bool) & tradable - ridge = work["ridge"].to_numpy(dtype=bool) & tradable - - per_bar = pd.DataFrame( - { - "trade_date": work["trade_date"].to_numpy(), - "valley_amt": np.where(valley, amt, 0.0), - "valley_vol": np.where(valley, vol, 0.0), - "ridge_amt": np.where(ridge, amt, 0.0), - "ridge_vol": np.where(ridge, vol, 0.0), - "valley_bars": valley.astype(np.int64), - "ridge_bars": ridge.astype(np.int64), - "classifiable_bars": work["classifiable"] - .to_numpy(dtype=bool) - .astype(np.int64), - } - ) - agg = per_bar.groupby("trade_date", sort=True).sum() - - # Four validity gates (§ module docstring / task card §1.4): PR-F's classifiable - # floor (unchanged), enough TRADABLE valley bars, enough TRADABLE ridge bars, and a - # positive denominator on both legs (a 0/0 ratio is never fabricated). - valid = ( - (agg["classifiable_bars"] >= min_classifiable) - & (agg["valley_bars"] >= min_valley_bars) - & (agg["ridge_bars"] >= min_ridge_bars) - & (agg["valley_vol"] > 0.0) - & (agg["ridge_vol"] > 0.0) - ) - ok = agg.loc[valid] - if ok.empty: - ratio = pd.Series([], index=pd.DatetimeIndex([], name="trade_date"), dtype=float) - else: - valley_vwap = ok["valley_amt"] / ok["valley_vol"] - ridge_vwap = ok["ridge_amt"] / ok["ridge_vol"] - ratio = (valley_vwap / ridge_vwap).astype(float) - - if not with_diagnostics: - return ratio - diagnostics = agg[["classifiable_bars", "valley_bars", "ridge_bars"]].copy() - diagnostics["valid"] = valid - return ratio, diagnostics - - -def _valley_ridge_ratio_mean_for_symbol( - g: pd.DataFrame, - *, - baseline_days: int, - baseline_min_obs: int, - sigma_k: float, - lookback_days: int, - min_valid_days: int, - min_classifiable: int, - min_valley_bars: int, - min_ridge_bars: int, - collect_diagnostics: bool = False, -) -> tuple[list[pd.Timestamp], list[float], pd.DataFrame | None]: - """Daily valley/ridge VWAP-ratio values for ONE symbol from its PIT-visible bars. - - Classifies the minutes with the REUSED :func:`peak_mask_for_symbol`, reduces each - valid day to its VWAP ratio, then takes the trailing-``lookback_days``-valid-day - mean. No cross-symbol leakage (``g`` is one symbol's slice) and no lookahead (the - baseline is strictly prior, the mean window is trailing). - """ - work = peak_mask_for_symbol( - g, - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - ) - result = valley_ridge_vwap_ratio_by_day( - work, - min_valley_bars=min_valley_bars, - min_ridge_bars=min_ridge_bars, - min_classifiable=min_classifiable, - with_diagnostics=collect_diagnostics, - ) - if collect_diagnostics: - ratio, diagnostics = result - else: - ratio, diagnostics = result, None - if ratio.empty: - return [], [], diagnostics - - # Mean over the trailing lookback_days VALID days (including d); NaN until - # min_valid_days valid days have accumulated. Emitted only on valid days. - rolled = ratio.sort_index().rolling(lookback_days, min_periods=min_valid_days).mean() - days = [pd.Timestamp(d).normalize() for d in rolled.index] - return days, list(rolled.to_numpy(dtype=float)), diagnostics - - -def compute_valley_ridge_vwap_ratio( - bars: pd.DataFrame, - *, - lookback_days: int = VALLEY_RIDGE_LOOKBACK_DAYS, - baseline_days: int = VOLUME_PRV_BASELINE_DAYS, - baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, - sigma_k: float = VOLUME_PRV_SIGMA_K, - min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - min_valley_bars: int = VALLEY_RIDGE_MIN_VALLEY_BARS, - min_ridge_bars: int = VALLEY_RIDGE_MIN_RIDGE_BARS, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "valley_ridge_vwap_ratio", - diagnostics_out: list | None = None, -) -> pd.Series: - """PIT-safe daily "valley/ridge VWAP ratio" factor from 1min ``bars``. - - Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, - classifies every visible minute with the REUSED PR-F taxonomy, computes each valid - day's ``valley VWAP / ridge VWAP`` ratio via the ``Σamount/Σvolume`` identity, and - returns the trailing-``lookback_days``-VALID-day mean of that ratio. See the module - docstring for the LOCKED definition and the seven pinned choices. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is - strictly per symbol (no cross-symbol leakage). - lookback_days: trailing VALID trading-day window averaged (definition). - baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). - baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). - sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). - min_valid_days: minimum valid days in the trailing window for a finite value. - min_classifiable: a day needs at least this many classifiable bars (PR-F). - min_valley_bars: a day needs at least this many TRADABLE valley bars. - min_ridge_bars: a day needs at least this many TRADABLE ridge bars (PINNED lower - than the valley floor — see the module docstring §6). - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - name: the returned Series name (the factor-panel column name). - diagnostics_out: optional list the per-symbol day-level bar-count frames are - APPENDED to (each carries a ``symbol`` column), so a caller can report the - ridge-scarcity distribution. Purely observational — supplying it does not - change the returned factor. - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily - factor value, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if baseline_days < 2: - # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is - # defined. - raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") - if baseline_min_obs < 2: - raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") - if sigma_k < 0.0: - raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") - if min_valid_days < 1: - raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") - if min_classifiable < 1: - raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") - if min_valley_bars < 1: - raise ValueError(f"min_valley_bars must be >= 1; got {min_valley_bars!r}.") - if min_ridge_bars < 1: - raise ValueError(f"min_ridge_bars must be >= 1; got {min_ridge_bars!r}.") - if len(bars) == 0: - return _empty_series(name) - - # extra_columns=("amount",) is the ONLY difference from the PR-F / PR-H entry - # points: the traded value rides along on the surviving rows, and the truncation / - # volume guard / slot assignment are untouched. - visible = prepare_visible_minute_bars( - bars, decision_time=decision_time, extra_columns=(_AMOUNT,) - ) - if visible.empty: - return _empty_series(name) - - collect = diagnostics_out is not None - index_tuples: list[tuple] = [] - values: list[float] = [] - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - days, vals, diagnostics = _valley_ridge_ratio_mean_for_symbol( - g.reset_index(drop=True), - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - lookback_days=lookback_days, - min_valid_days=min_valid_days, - min_classifiable=min_classifiable, - min_valley_bars=min_valley_bars, - min_ridge_bars=min_ridge_bars, - collect_diagnostics=collect, - ) - if collect and diagnostics is not None and not diagnostics.empty: - frame = diagnostics.copy() - frame[SYMBOL_LEVEL] = str(sym) - diagnostics_out.append(frame) - for day, val in zip(days, vals): - index_tuples.append((day, str(sym))) - values.append(val) - - if not index_tuples: - return _empty_series(name) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - - __all__ = [ "DIAGNOSTIC_COLUMNS", "VALLEY_RIDGE_LOOKBACK_DAYS", diff --git a/data/clean/intraday_valley_vwap.py b/data/clean/intraday_valley_vwap.py index 35c704e..de18889 100644 --- a/data/clean/intraday_valley_vwap.py +++ b/data/clean/intraday_valley_vwap.py @@ -1,304 +1,18 @@ -"""VALLEY-RELATIVE VWAP factor (PR-I). +"""D2 re-export shim — the PR-I factor math moved to ``factors.compute.minute``. -Reproduces the THIRD factor of the Kaiyuan market-microstructure series #27 (开源证券 -《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, -§4): "参考聪明钱因子的构建方式，计算每日量谷的成交量加权价格，与当日总成交量加权价格做比， -并计算 20 日均值作为量谷相对加权价格因子". - -Same MACHINE as PR-F / PR-H, different FAMILY. The volume classification is REUSED -verbatim from :mod:`data.clean.intraday_volume_prv` (``prepare_visible_minute_bars`` + -``peak_mask_for_symbol``) — same-slot strictly-prior μ+kσ eruptive/mild test, same -classifiable rule, same valid-day floor. A VALLEY (量谷) is exactly that module's -``valley`` column: a classifiable, NON-eruptive minute. Nothing about the taxonomy is -re-implemented here, so the three factors can never drift apart. Where PR-F counts peaks -and PR-H measures the TIMING of peaks, this module measures the PRICE LEVEL at the -valleys — a different statistic on a different part of the taxonomy, which is why it is -worth running after two null results on the peak-counting family. - -PINNED choices (deliberate and DISCLOSED, not tuned knobs; reproduced on the factor spec -so a reader sees exactly what was assumed): - - 1. VWAP VIA THE AGGREGATION IDENTITY. A bar's volume-weighted price is - ``p = amount / volume``, so a set's volume-weighted price - ``Σ(p_i·v_i) / Σv_i`` collapses EXACTLY to ``Σamount / Σvolume``. Both legs use - that identity: valley VWAP = Σamount(valley bars)/Σvolume(valley bars), day VWAP - = Σamount(all visible bars)/Σvolume(all visible bars). This is the day's REAL - VWAP, strictly better than approximating each bar by its close. - 2. POSITIVE-TRADE GUARD. A bar with non-finite or non-positive ``volume`` OR - ``amount`` carries no price information (``amount/volume`` is meaningless or - degenerate), so it is dropped from BOTH sums. The guard runs at the summation - step only — never before classification — because the same-slot μ/σ baseline is - PR-F's and must stay bit-identical. A zero-volume minute is therefore still - classified (and still a valley) but contributes nothing to either VWAP. - 3. RAW (UNADJUSTED) PRICES. The cached minute bars are unadjusted. Both legs are - sums over the SAME trading day, and a split/dividend adjustment factor is constant - within a day, so it cancels exactly in the ratio — no adjustment is needed (the - same reasoning PR-D's amplitude and I5b's price-limit checks rely on). - 4. THE DAY VWAP COVERS THE PIT-VISIBLE WINDOW ONLY (09:31–14:50), not the full - session. This is a NECESSARY DEVIATION from the report, which uses the whole day: - the standing 14:50 decision cutoff truncates history days and the signal day - identically, and reading the closing auction would be lookahead at our decision - time. Disclosed, not silently equated to the report's construction. - 5. VALLEY-BAR COUNT IS TAKEN AFTER THE GUARD. The ``min_valley_bars`` floor counts - the valley minutes that actually CONTRIBUTE to the numerator, so a day cannot - qualify on the strength of valley bars that traded nothing. - -Factor value: ``valley_relative_vwap_20`` = the MEAN of the daily ratio -``valley VWAP / day VWAP`` over the symbol's most recent ``lookback_days`` (=20) VALID -trading days INCLUDING ``d``. A day is VALID iff it clears three gates: at least -``min_classifiable`` (=100) classifiable bars (PR-F's gate, unchanged), at least -``min_valley_bars`` (=20) tradable valley bars, and strictly positive volume in BOTH -denominators. Fewer than ``min_valid_days`` (=10) valid days in the trailing window -> -NaN (honest missing; the runner discloses the coverage). Values are emitted only on -valid days. - -Pre-registered sign = +1. The report's full-market RankIC is +8.69% / RankICIR 4.44 -(long-short 25.35%/yr, IR 3.04, monthly win rate 79.7%) — the strongest factor in the -report; its CSI500 sub-domain long-short is 9.94% / IR 1.26, the closest comparable to -our eval cell. Semantics per the report: valley minutes are moments of subdued trading -sentiment where prices are unlikely to have over-reacted, so a HIGH relative valley -price means the calm, informed part of the day was bid up -> higher future return. NOTE -the report is a MONTHLY, market-cap + industry neutral full-market series on Wind data -while our eval cell is CSI500 daily with industry + size neutralization, so its numbers -are a LOOSE reference only (disclosed, never mislabeled, never written in as an expected -value). - -The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future -bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. This module is -DATA-layer only: it does not fetch, does not touch factors / alpha / portfolio / -runtime, and never sees a token. +Single definition point from D2 on (design v3.2 §6.4): the valley-relative +VWAP factor lives in :mod:`factors.compute.minute.valley_relative_vwap`. This +shim keeps every name the pre-D2 module exported importable from its old path; +it is deleted in D6d. Import-only by contract (locked by the shim purity test). """ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import DAILY_INDEX_NAMES, DEFAULT_DECISION_TIME -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars -from data.clean.intraday_volume_prv import ( - VOLUME_PRV_BASELINE_DAYS, - VOLUME_PRV_BASELINE_MIN_OBS, - VOLUME_PRV_MIN_CLASSIFIABLE, - VOLUME_PRV_MIN_VALID_DAYS, - VOLUME_PRV_SIGMA_K, - peak_mask_for_symbol, - prepare_visible_minute_bars, +from factors.compute.minute.valley_relative_vwap import ( + VALLEY_VWAP_LOOKBACK_DAYS, + VALLEY_VWAP_MIN_VALLEY_BARS, + compute_valley_relative_vwap, + valley_vwap_ratio_by_day, ) -# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). -# The classification constants are IMPORTED from PR-F, never redefined. -VALLEY_VWAP_LOOKBACK_DAYS = 20 # trailing VALID trading-day mean window, includes d -VALLEY_VWAP_MIN_VALLEY_BARS = 20 # min TRADABLE valley bars for a valid day - -# The extra 1min column this family needs on top of PR-F's (volume): the traded value, -# which turns the bar set into a volume-weighted price via Σamount/Σvolume. -_AMOUNT = "amount" - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def valley_vwap_ratio_by_day( - work: pd.DataFrame, - *, - min_valley_bars: int = VALLEY_VWAP_MIN_VALLEY_BARS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, -) -> pd.Series: - """Daily ``valley VWAP / day VWAP`` ratio for ONE symbol, on VALID days only. - - ``work`` is one symbol's frame as returned by - :func:`~data.clean.intraday_volume_prv.peak_mask_for_symbol`, which must have been - built from bars prepared with ``extra_columns=("amount",)`` so the traded value is - available alongside the ``valley`` / ``classifiable`` masks. - - Both VWAPs use the ``Σamount / Σvolume`` aggregation identity (PINNED §1 of the - module docstring). The POSITIVE-TRADE GUARD (§2) drops non-finite / non-positive - volume or amount bars from both sums; the day leg spans ALL visible bars that clear - the guard (eruptive minutes included — it is the WHOLE visible day's VWAP), while - the valley leg spans only the guarded valley bars. - - Returns: - Series indexed by ``trade_date`` (ascending) holding the ratio for the days that - clear all three validity gates — invalid days are ABSENT, not NaN, so they do - not occupy a slot in the caller's trailing window (the same rule PR-F/PR-H use). - """ - vol = work["volume"].to_numpy(dtype=float) - amt = work[_AMOUNT].to_numpy(dtype=float) - # Positive-trade guard: no price information in a bar that traded nothing (or whose - # fields are non-finite / negative). Applied HERE, at the summation step, never - # before classification — PR-F's same-slot baseline must stay bit-identical. - tradable = np.isfinite(vol) & (vol > 0.0) & np.isfinite(amt) & (amt > 0.0) - valley = work["valley"].to_numpy(dtype=bool) & tradable - - per_bar = pd.DataFrame( - { - "trade_date": work["trade_date"].to_numpy(), - "day_amt": np.where(tradable, amt, 0.0), - "day_vol": np.where(tradable, vol, 0.0), - "valley_amt": np.where(valley, amt, 0.0), - "valley_vol": np.where(valley, vol, 0.0), - "valley_bars": valley.astype(np.int64), - "classifiable_bars": work["classifiable"].to_numpy(dtype=bool).astype( - np.int64 - ), - } - ) - agg = per_bar.groupby("trade_date", sort=True).sum() - - # Three validity gates (§ module docstring / task card §1.4): PR-F's classifiable - # floor (unchanged), enough TRADABLE valley bars, and a positive denominator on both - # legs (a 0/0 ratio is never fabricated). - valid = ( - (agg["classifiable_bars"] >= min_classifiable) - & (agg["valley_bars"] >= min_valley_bars) - & (agg["day_vol"] > 0.0) - & (agg["valley_vol"] > 0.0) - ) - ok = agg.loc[valid] - if ok.empty: - return pd.Series( - [], index=pd.DatetimeIndex([], name="trade_date"), dtype=float - ) - valley_vwap = ok["valley_amt"] / ok["valley_vol"] - day_vwap = ok["day_amt"] / ok["day_vol"] - return (valley_vwap / day_vwap).astype(float) - - -def _valley_ratio_mean_for_symbol( - g: pd.DataFrame, - *, - baseline_days: int, - baseline_min_obs: int, - sigma_k: float, - lookback_days: int, - min_valid_days: int, - min_classifiable: int, - min_valley_bars: int, -) -> tuple[list[pd.Timestamp], list[float]]: - """Daily valley-relative-VWAP values for ONE symbol from its PIT-visible bars. - - Classifies the minutes with the REUSED :func:`peak_mask_for_symbol`, reduces each - valid day to its VWAP ratio, then takes the trailing-``lookback_days``-valid-day - mean. No cross-symbol leakage (``g`` is one symbol's slice) and no lookahead (the - baseline is strictly prior, the mean window is trailing). - """ - work = peak_mask_for_symbol( - g, - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - ) - ratio = valley_vwap_ratio_by_day( - work, min_valley_bars=min_valley_bars, min_classifiable=min_classifiable - ) - if ratio.empty: - return [], [] - - # Mean over the trailing lookback_days VALID days (including d); NaN until - # min_valid_days valid days have accumulated. Emitted only on valid days. - rolled = ratio.sort_index().rolling(lookback_days, min_periods=min_valid_days).mean() - days = [pd.Timestamp(d).normalize() for d in rolled.index] - return days, list(rolled.to_numpy(dtype=float)) - - -def compute_valley_relative_vwap( - bars: pd.DataFrame, - *, - lookback_days: int = VALLEY_VWAP_LOOKBACK_DAYS, - baseline_days: int = VOLUME_PRV_BASELINE_DAYS, - baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, - sigma_k: float = VOLUME_PRV_SIGMA_K, - min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - min_valley_bars: int = VALLEY_VWAP_MIN_VALLEY_BARS, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "valley_relative_vwap", -) -> pd.Series: - """PIT-safe daily "valley-relative VWAP" factor from 1min ``bars``. - - Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, - classifies every visible minute with the REUSED PR-F taxonomy, computes each valid - day's ``valley VWAP / whole-visible-day VWAP`` ratio via the ``Σamount/Σvolume`` - identity, and returns the trailing-``lookback_days``-VALID-day mean of that ratio. - See the module docstring for the LOCKED definition and the five pinned choices. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is - strictly per symbol (no cross-symbol leakage). - lookback_days: trailing VALID trading-day window averaged (definition). - baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). - baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). - sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). - min_valid_days: minimum valid days in the trailing window for a finite value. - min_classifiable: a day needs at least this many classifiable bars (PR-F). - min_valley_bars: a day needs at least this many TRADABLE valley bars. - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - name: the returned Series name (the factor-panel column name). - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily - factor value, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if baseline_days < 2: - # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is - # defined. - raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") - if baseline_min_obs < 2: - raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") - if sigma_k < 0.0: - raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") - if min_valid_days < 1: - raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") - if min_classifiable < 1: - raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") - if min_valley_bars < 1: - raise ValueError(f"min_valley_bars must be >= 1; got {min_valley_bars!r}.") - if len(bars) == 0: - return _empty_series(name) - - # extra_columns=("amount",) is the ONLY difference from the PR-F / PR-H entry - # points: the traded value rides along on the surviving rows, and the truncation / - # volume guard / slot assignment are untouched. - visible = prepare_visible_minute_bars( - bars, decision_time=decision_time, extra_columns=(_AMOUNT,) - ) - if visible.empty: - return _empty_series(name) - - index_tuples: list[tuple] = [] - values: list[float] = [] - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - days, vals = _valley_ratio_mean_for_symbol( - g.reset_index(drop=True), - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - lookback_days=lookback_days, - min_valid_days=min_valid_days, - min_classifiable=min_classifiable, - min_valley_bars=min_valley_bars, - ) - for day, val in zip(days, vals): - index_tuples.append((day, str(sym))) - values.append(val) - - if not index_tuples: - return _empty_series(name) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - - __all__ = [ "VALLEY_VWAP_LOOKBACK_DAYS", "VALLEY_VWAP_MIN_VALLEY_BARS", diff --git a/data/clean/intraday_volume_prv.py b/data/clean/intraday_volume_prv.py index 3f0756b..645fa32 100644 --- a/data/clean/intraday_volume_prv.py +++ b/data/clean/intraday_volume_prv.py @@ -1,401 +1,25 @@ -"""Volume-peak-count factor (PR-F). +"""D2 re-export shim — the PR-F factor math moved to ``factors.compute.minute``. -Reproduces the Kaiyuan market-microstructure series #27 (开源证券《高频成交量的峰、岭、 -谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417) flagship -"volume-peak-minute-count" factor (§2) as a daily PIT-safe column derived DIRECTLY -from the 1min cache (no coarser resampling — the peak/ridge/valley taxonomy lives at -the 1-minute grain). Kept in the DATA-clean layer (like -:func:`data.clean.intraday_amp_anomaly.compute_amp_marginal_anomaly_vol`) so the -``factors`` layer only SELECTS the pre-aggregated column and never fetches or sees a -forward return. - -The report is under-specified about the PIT boundary; the interpretations below are -deliberate, DISCLOSED choices PINNED in the task card (task_card_pr_f_*.md §1), not -tuned knobs. They are reproduced on the factor spec so a reader can see exactly what -was assumed: - - 1. PIT truncation (standing authorization): each day keeps only the 1min bars with - ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50) — - history days AND the signal day are truncated identically, so the same-slot - cross-day baseline is measured on a consistent window (≈ the morning session plus - the afternoon up to the cutoff). - 2. Same-slot baseline (PINNED as STRICTLY PRIOR — the report does not say whether - the current day is included, and strictly-prior is the more PIT-stable reading): - for minute slot ``s`` and day ``t`` the baseline ``μ_s`` / ``σ_s`` (ddof=1) is - the symbol's SAME-SLOT volume over the trailing ``baseline_days`` (=20) trading - days STRICTLY BEFORE ``t``; fewer than ``baseline_min_obs`` (=10) same-slot - observations in that window -> the ``(t, s)`` bar is NOT classifiable. - 3. classify: ``vol > μ_s + k*σ_s`` (k = 1) -> ERUPTIVE, else MILD (a "valley"). - 4. a slot is a PEAK iff it is eruptive AND both its 1-minute neighbours in the SAME - continuous session exist and are MILD. An eruptive bar whose neighbour is also - eruptive is a RIDGE (not a peak); a session-boundary bar (each session's first / - last visible bar, so a neighbour is missing across the lunch break or the cutoff) - is likewise NOT a peak (its isolation is unprovable — PINNED disclosure). A - neighbour that is not classifiable (baseline too thin) also blocks the peak. - "Same continuous session" and "the 1-minute neighbour" are enforced by requiring - the adjacent bar to be EXACTLY 60s away (the lunch break 11:30->13:01 and any - missing minute fail this, so they are correctly not neighbours). - -Factor value: ``volume_peak_count_20`` = the total count of peak minutes over the -symbol's most recent ``lookback_days`` (=20) VALID trading days INCLUDING ``d`` (a -simple count — after truncation the per-day slot count is uniform, so counts are -cross-sectionally comparable without normalization). A day is VALID iff it has at -least ``min_classifiable`` (=100) classifiable bars; fewer than ``min_valid_days`` -(=10) valid days in the trailing window -> NaN (honest missing; the runner discloses -the coverage). Values are emitted only on valid days. - -Pre-registered sign = +1 (more volume peaks = more informed-trading participation = -higher future returns; the report's full-market RankIC is +10.62% / RankICIR 4.36 and -its CSI500 sub-domain long-short is +14.96%/yr — the CSI500 line is the direct anchor -for our eval cell). NOTE the report is a monthly, market-cap + industry neutral series -on Wind data; our eval cell is CSI500 daily with industry + size neutral, so the -report numbers are a LOOSE reference only (disclosed, never mislabeled). Raw minute -volume (cached as-is) has magnitude jumps across split days that pollute the 20-day σ; -the report (Wind) does not adjust for this either, so we disclose it and do NOT correct -it. The value at ``(d, s)`` uses only bars at dates <= d, so a factor value never sees -a future bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. - -This module is DATA-layer only: it does not fetch, does not touch factors / alpha / -portfolio / runtime, and never sees a token. +Single definition point from D2 on (design v3.2 §6.4): the volume-peak-count +factor lives in :mod:`factors.compute.minute.volume_peak_count` and the shared +peak/ridge/valley taxonomy in :mod:`factors.compute.minute.primitives`. This +shim keeps every name the pre-D2 module exported importable from its old path; +it is deleted in D6d. Import-only by contract (locked by the shim purity test). """ -from __future__ import annotations - -from collections.abc import Sequence - -import numpy as np -import pandas as pd - -from data.clean.intraday_aggregate import DAILY_INDEX_NAMES, DEFAULT_DECISION_TIME -from data.clean.intraday_schema import SYMBOL_LEVEL, validate_intraday_bars - -# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). -VOLUME_PRV_LOOKBACK_DAYS = 20 # trailing VALID trading-day count window (N), includes d -VOLUME_PRV_BASELINE_DAYS = 20 # strictly-prior same-slot baseline window (trading days) -VOLUME_PRV_BASELINE_MIN_OBS = 10 # min same-slot obs for a classifiable bar -VOLUME_PRV_SIGMA_K = 1.0 # eruptive threshold multiplier k in vol > μ + k*σ -VOLUME_PRV_MIN_VALID_DAYS = 10 # min valid days in the trailing window for a finite value -VOLUME_PRV_MIN_CLASSIFIABLE = 100 # a day is valid iff it has >= this many classifiable bars - -# The "strictly-next minute" test in seconds: two bars are same-session 1-minute -# neighbours iff their bar_end gap is EXACTLY 60s. The lunch break (11:30 -> 13:01), -# the session close and any missing minute all differ, so they are not neighbours. -_ONE_MINUTE_SECONDS = 60.0 - -# The columns every peak-family factor needs. ``prepare_visible_minute_bars`` always -# emits EXACTLY these (plus the derived ``trade_date`` / ``slot``); anything else a -# caller needs is opt-in via ``extra_columns``, so the default output stays byte-identical -# for the callers that predate the option. -_BASE_VISIBLE_COLUMNS = [SYMBOL_LEVEL, "bar_end", "available_time", "volume"] - - -def _empty_series(name: str) -> pd.Series: - """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" - index = pd.MultiIndex.from_arrays( - [pd.DatetimeIndex([]), pd.Index([], dtype=object)], - names=DAILY_INDEX_NAMES, - ) - return pd.Series([], index=index, dtype=float, name=name) - - -def prepare_visible_minute_bars( - bars: pd.DataFrame, - *, - decision_time: str = DEFAULT_DECISION_TIME, - extra_columns: Sequence[str] = (), -) -> pd.DataFrame: - """PIT-truncate ``bars`` at ``decision_time`` and add ``trade_date`` / ``slot``. - - The shared front half of every peak-taxonomy factor: keep only the bars whose - ``available_time`` is at or before their own ``trade_date + decision_time`` (so - history days and the signal day are truncated identically), drop non-finite / - negative volumes (invalid data that would poison the same-slot μ/σ), and add the - minute-of-day ``slot`` that aligns the SAME time-of-day across days. - - ``bars`` must ALREADY be schema-validated by the caller - (:func:`~data.clean.intraday_schema.validate_intraday_bars`) — this helper is the - post-validation preparation step, kept separate so the entry points keep their own - validate-then-check-params ordering. - - Args: - bars: normalized 1min bars, ``MultiIndex(time, symbol)``; one or many symbols. - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - extra_columns: additional ``bars`` columns to carry through, appended AFTER the - base columns. The truncation and the volume guard are unaffected — the extra - values merely ride along on the surviving rows — so the default ``()`` keeps - the output byte-identical for callers that do not need them (PR-F / PR-H). - PR-I passes ``("amount",)`` to compute a volume-weighted price. - - Returns: - A fresh ``RangeIndex`` frame with columns ``symbol`` / ``bar_end`` / - ``available_time`` / ``volume`` / ``trade_date`` / ``slot`` plus any - ``extra_columns`` (possibly empty). Pure: never mutates ``bars``. - """ - extra = list(extra_columns) - unknown = [c for c in extra if c not in bars.columns] - if unknown: - raise ValueError( - f"prepare_visible_minute_bars extra_columns not present on bars: {unknown}; " - f"bars has {list(bars.columns)}." - ) - clashing = [c for c in extra if c in _BASE_VISIBLE_COLUMNS] - if clashing: - raise ValueError( - f"prepare_visible_minute_bars extra_columns are already emitted: {clashing}." - ) - work = bars.reset_index()[_BASE_VISIBLE_COLUMNS + extra].copy() - work["trade_date"] = work["bar_end"].dt.normalize() - # PIT truncation FIRST (per-bar timestamps): each bar's cutoff is its own - # trade_date + decision_time, so every day is truncated to [open, cutoff]. - cutoff = work["trade_date"] + pd.Timedelta(decision_time) - visible = work.loc[work["available_time"] <= cutoff].copy() - if visible.empty: - return visible - - # Guard bad volume BEFORE anything else: a non-finite / negative volume is invalid - # data that would poison the same-slot μ/σ. (Split-day magnitude jumps are NOT - # corrected — disclosed, matching the report's Wind treatment.) - vol = visible["volume"].to_numpy(dtype=float) - visible = visible.loc[np.isfinite(vol) & (vol >= 0.0)].copy() - if visible.empty: - return visible - - # slot = minute-of-day (minutes since midnight); bars are minute-aligned so this is - # exact and aligns the SAME time-of-day across days for the same-slot baseline. - visible["slot"] = ( - (visible["bar_end"] - visible["trade_date"]) // pd.Timedelta(minutes=1) - ).astype(int) - return visible - - -def peak_mask_for_symbol( - g: pd.DataFrame, - *, - baseline_days: int = VOLUME_PRV_BASELINE_DAYS, - baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, - sigma_k: float = VOLUME_PRV_SIGMA_K, -) -> pd.DataFrame: - """Per-minute PEAK / classifiable mask for ONE symbol's PIT-visible bars. - - This is THE volume-peak identification of the report (§2), shared by every factor in - the peak family so the taxonomy is defined in exactly one place: same-slot - strictly-prior μ/σ baseline -> eruptive vs mild -> a peak is an eruptive minute whose - both 1-minute same-session neighbours are mild. - - ``g`` holds columns ``trade_date`` / ``slot`` / ``bar_end`` / ``volume`` for a - SINGLE symbol (a fresh ``RangeIndex``, as produced by - :func:`prepare_visible_minute_bars`). The same-slot STRICTLY-PRIOR baseline is - computed on a day x slot pivot (rolling over the day axis then ``shift(1)``), so no - per-``(day, slot)`` python loop is needed and the current day never enters its own - baseline. Peak detection is done back in long form with a within-day 60s-gap - neighbour test (which enforces "same continuous session" and "the 1-minute - neighbour" at once). No cross-symbol leakage — ``g`` is one symbol's slice — and no - cross-day leakage — the baseline is strictly prior and the neighbour test never - crosses a day boundary (grouped by ``trade_date``). - - Returns: - ``g`` sorted by ``(trade_date, bar_end)`` on a fresh ``RangeIndex`` with the - added boolean columns ``classifiable`` (a strictly-prior baseline existed), - ``valley`` (classifiable and NOT eruptive — the report's 量谷, what PR-I prices), - ``peak`` and ``ridge`` (eruptive but NOT an isolated peak — the report's 量岭, - what PR-J prices). The three masks PARTITION the classifiable bars exactly: - ``valley`` holds the mild ones and ``peak`` / ``ridge`` split the eruptive ones, - so ``valley | peak | ridge == classifiable`` and no two overlap. Pure: never - mutates ``g``. - """ - # day x slot volume matrix: rows are the symbol's trading days, columns are the - # minute-of-day slots; a missing (day, slot) cell is NaN (no bar that minute). - v = g.pivot(index="trade_date", columns="slot", values="volume") - v = v.sort_index().sort_index(axis=1) - - # Strictly-prior same-slot baseline: rolling over the DAY axis (per slot column), - # requiring >= baseline_min_obs actual same-slot observations, THEN shift(1) so - # day t uses days t-baseline_days .. t-1 only (never day t itself). - roll = v.rolling(baseline_days, min_periods=baseline_min_obs) - mu = roll.mean().shift(1) - sigma = roll.std().shift(1) # ddof=1 (pandas rolling default) - thr = mu + sigma_k * sigma - - thr_long = thr.reset_index().melt( - id_vars="trade_date", var_name="slot", value_name="thr" - ) - # melt can hand back the slot column labels as object dtype; keep it int so the - # merge below matches g's int ``slot`` (an object/int mismatch would silently miss). - thr_long["slot"] = thr_long["slot"].astype(int) - work = g.merge(thr_long, on=["trade_date", "slot"], how="left") - work = work.sort_values(["trade_date", "bar_end"], kind="mergesort").reset_index( - drop=True - ) - - vol = work["volume"].to_numpy(dtype=float) - thr_arr = work["thr"].to_numpy(dtype=float) - # Classifiable iff the strictly-prior baseline is finite (>= baseline_min_obs obs); - # eruptive iff strictly above μ + k*σ; mild is any other classifiable bar. - classifiable = np.isfinite(thr_arr) - eruptive = classifiable & (vol > thr_arr) - mild = classifiable & ~eruptive - work["classifiable"] = classifiable - # VALLEY (量谷 in the report's peak/ridge/valley taxonomy) == MILD: a classifiable, - # non-eruptive minute. Exposed as a first-class boolean so the valley-PRICE family - # (PR-I) consumes THE SAME classification instead of re-deriving it and drifting. - # Purely additive — ``classifiable`` and ``peak`` below are computed exactly as - # before and no consumer of this frame reads columns positionally. - work["valley"] = mild - # Carry mild as 0/1 so the within-day neighbour shift stays numeric (a shifted bool - # column would go through an object-dtype fillna and warn). - work["mild_i"] = mild.astype(np.int8) - - by_day = work.groupby("trade_date", sort=False) - prev_end = by_day["bar_end"].shift(1) - next_end = by_day["bar_end"].shift(-1) - gap_prev = (work["bar_end"] - prev_end).dt.total_seconds().to_numpy() - gap_next = (next_end - work["bar_end"]).dt.total_seconds().to_numpy() - prev_mild = by_day["mild_i"].shift(1).fillna(0).to_numpy() > 0 - next_mild = by_day["mild_i"].shift(-1).fillna(0).to_numpy() > 0 - # A peak is an eruptive bar whose BOTH 1-minute neighbours (exactly 60s away, so - # same session) exist and are mild. Missing / lunch-break / cutoff neighbours fail - # the 60s gap; eruptive neighbours (ridge) and unclassifiable ones fail the mild - # test — all correctly block the peak. - peak = ( - eruptive - & (gap_prev == _ONE_MINUTE_SECONDS) - & prev_mild - & (gap_next == _ONE_MINUTE_SECONDS) - & next_mild - ) - work["peak"] = peak - # RIDGE (量岭) == eruptive AND NOT an isolated peak. Exposed as a first-class boolean - # so the ridge-PRICE family (PR-J) consumes THE SAME classification instead of - # re-deriving it and drifting. Purely additive: ``classifiable`` / ``valley`` / - # ``peak`` above are computed exactly as before and no consumer of this frame reads - # columns positionally. - # - # PINNED, and wider than the literal "eruptive next to an eruptive": an eruptive bar - # is a ridge whenever its isolation is NOT PROVABLE, which also covers the - # session-boundary bars (a neighbour missing across the lunch break / the cutoff / - # a gap) and the bars whose neighbour is unclassifiable. That is the exact complement - # of PR-F's deliberately conservative peak rule, so the taxonomy stays a clean - # partition (valley | peak | ridge == classifiable) with no bar silently dropped. - work["ridge"] = eruptive & ~peak - return work - - -def _peak_count_for_symbol( - g: pd.DataFrame, - *, - baseline_days: int, - baseline_min_obs: int, - sigma_k: float, - lookback_days: int, - min_valid_days: int, - min_classifiable: int, -) -> tuple[list[pd.Timestamp], list[float]]: - """Daily volume-peak-count values for ONE symbol from its PIT-visible bars. - - Identifies the peaks with the shared :func:`peak_mask_for_symbol` and reduces them to - the trailing-``lookback_days``-VALID-day count. - """ - work = peak_mask_for_symbol( - g, - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - ) - - peak_count = work.groupby("trade_date")["peak"].sum() - classifiable_count = work.groupby("trade_date")["classifiable"].sum() - valid_days = classifiable_count.index[classifiable_count >= min_classifiable] - if len(valid_days) == 0: - return [], [] - - # Count over the trailing lookback_days VALID days (including d); NaN until - # min_valid_days valid days have accumulated. Values are emitted only on valid days. - pc_valid = peak_count.loc[valid_days].astype(float).sort_index() - factor_valid = pc_valid.rolling(lookback_days, min_periods=min_valid_days).sum() - days = [pd.Timestamp(d).normalize() for d in factor_valid.index] - return days, list(factor_valid.to_numpy(dtype=float)) - - -def compute_volume_peak_count( - bars: pd.DataFrame, - *, - lookback_days: int = VOLUME_PRV_LOOKBACK_DAYS, - baseline_days: int = VOLUME_PRV_BASELINE_DAYS, - baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, - sigma_k: float = VOLUME_PRV_SIGMA_K, - min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, - min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, - decision_time: str = DEFAULT_DECISION_TIME, - name: str = "volume_peak_count", -) -> pd.Series: - """PIT-safe daily "volume-peak-minute-count" factor from 1min ``bars``. - - Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, - classifies every visible minute against its SAME-SLOT strictly-prior baseline - (eruptive vs mild), marks the eruptive minutes whose both 1-minute same-session - neighbours are mild as PEAKS, and returns the trailing-``lookback_days``-VALID-day - peak count. See the module docstring for the LOCKED definition. - - Args: - bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), - ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping - is strictly per symbol (no cross-symbol leakage). - lookback_days: trailing VALID trading-day window for the count (definition). - baseline_days: strictly-prior same-slot baseline window in trading days. - baseline_min_obs: minimum same-slot observations for a classifiable bar. - sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ``. - min_valid_days: minimum valid days in the trailing window for a finite value. - min_classifiable: a day is VALID iff it has at least this many classifiable bars. - decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). - name: the returned Series name (the factor-panel column name). - - Returns: - ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily - factor value, sorted, named ``name``. Pure: never mutates ``bars``. - """ - validate_intraday_bars(bars) - if lookback_days < 1: - raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") - if baseline_days < 2: - # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is - # defined. - raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") - if baseline_min_obs < 2: - raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") - if sigma_k < 0.0: - raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") - if min_valid_days < 1: - raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") - if min_classifiable < 1: - raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") - if len(bars) == 0: - return _empty_series(name) - - visible = prepare_visible_minute_bars(bars, decision_time=decision_time) - if visible.empty: - return _empty_series(name) - - index_tuples: list[tuple] = [] - values: list[float] = [] - for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): - days, vals = _peak_count_for_symbol( - g.reset_index(drop=True), - baseline_days=baseline_days, - baseline_min_obs=baseline_min_obs, - sigma_k=sigma_k, - lookback_days=lookback_days, - min_valid_days=min_valid_days, - min_classifiable=min_classifiable, - ) - for day, val in zip(days, vals): - index_tuples.append((day, str(sym))) - values.append(val) - - if not index_tuples: - return _empty_series(name) - index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) - return pd.Series(values, index=index, name=name).sort_index() - +from factors.compute.minute.primitives import ( + VOLUME_PRV_BASELINE_DAYS, + VOLUME_PRV_BASELINE_MIN_OBS, + VOLUME_PRV_MIN_CLASSIFIABLE, + VOLUME_PRV_MIN_VALID_DAYS, + VOLUME_PRV_SIGMA_K, + peak_mask_for_symbol, + prepare_visible_minute_bars, +) +from factors.compute.minute.volume_peak_count import ( + VOLUME_PRV_LOOKBACK_DAYS, + compute_volume_peak_count, +) __all__ = [ "VOLUME_PRV_BASELINE_DAYS", diff --git a/factors/compute/intraday_derived.py b/factors/compute/intraday_derived.py index 052cbbe..8e32ca9 100644 --- a/factors/compute/intraday_derived.py +++ b/factors/compute/intraday_derived.py @@ -1,1477 +1,23 @@ -"""Daily factors DERIVED from intraday (minute) bars but executed close-to-close. +"""D2 re-export shim — the 11 minute-factor surface classes moved house. -The members today are :class:`JumpAmountCorrFactor` (PR-C, the Kaiyuan report §6 -"price-jump turnover correlation" factor), :class:`MinuteIdealAmplitudeFactor` -(PR-D, the Kaiyuan report §30 "minute ideal amplitude" factor), -:class:`AmpMarginalAnomalyVolFactor` (PR-E, the Changjiang high-frequency-factor -series #19 "amplitude marginal-anomaly relative-volatility" factor) and -:class:`VolumePeakCountFactor` (PR-F, the Kaiyuan microstructure series #27 -"volume-peak-minute-count" factor). Like the value / MMP factors, the heavy -computation runs UPSTREAM (``data.clean.intraday_aggregate`` / -``data.clean.intraday_amplitude`` / ``data.clean.intraday_amp_anomaly`` / -``data.clean.intraday_volume_prv`` aggregate 1min bars into a daily -``MultiIndex(date, symbol)`` column) and the Factor here simply SELECTS its column off -the panel the runner already enriched. Keeping the minute aggregation in the data-clean -layer preserves the layering: ``factors`` never fetches and never sees a forward return. - -WHY ``is_intraday=False`` FOR A MINUTE-DERIVED FACTOR (deliberate, documented): - ``FactorSpec.is_intraday`` flags an intraday-EXECUTION contract — the minute - tail model that DECIDES at 14:50 and FILLS at 14:51, whose holding period runs - exec(T) -> exec(T_next) (I5a). This factor has minute INPUT but its signal is a - DAILY value traded at the daily close and held close-to-close (t -> t+1), just - like the report's monthly rebalance on the daily close and the project's daily - default. It carries no 14:50 decision cutoff, no execution window, no exec-to- - exec holding period — so ``is_intraday`` is False, ``return_basis`` is - ``"close_to_close"``, and the five minute-block spec fields are all None. (The - base ``FactorSpec`` deliberately does NOT force is_intraday from a minute - provenance; a daily signal computed from minute data is a legitimate case.) +Single definition point from D2 on (design v3.2 §6.4): each surface class now +lives WITH its factor math in its own ``factors.compute.minute.{factor}`` +module (one factor per file). This shim keeps the pre-D2 import path working +for the eval runners and tests; it is deleted in D6d. Import-only by contract +(locked by the shim purity test). """ -from __future__ import annotations - -import pandas as pd - -from data.availability_policy import MARKET_DAILY, STK_MINS_1MIN -from data.clean.intraday_aggregate import ( - JUMP_LOOKBACK_DAYS, - JUMP_MIN_PAIRS, -) -from data.clean.intraday_amount_ratio import ( - PEAK_RIDGE_LOOKBACK_DAYS, - PEAK_RIDGE_MIN_PEAK_BARS, - PEAK_RIDGE_MIN_RIDGE_BARS, -) -from data.clean.intraday_amp_anomaly import ( - AMP_ANOMALY_FREQ, - AMP_ANOMALY_LOOKBACK_DAYS, - AMP_ANOMALY_MIN_POOL, - AMP_ANOMALY_MIN_SELECTED, - AMP_ANOMALY_SIGMA_K, -) -from data.clean.intraday_amp_cut import ( - AMP_CUT_LAMBDA, - AMP_CUT_LOOKBACK_DAYS, - AMP_CUT_MIN_CROSS_SECTION, - AMP_CUT_MIN_DAY_MINUTES, - AMP_CUT_MIN_VALID_DAYS, -) -from data.clean.intraday_amplitude import ( - IDEAL_AMP_LAMBDA, - IDEAL_AMP_LOOKBACK_DAYS, - IDEAL_AMP_MIN_MINUTES, -) -from data.clean.intraday_peak_interval import ( - PEAK_INTERVAL_LOOKBACK_DAYS, - PEAK_INTERVAL_MIN_INTERVALS, -) -from data.clean.intraday_ridge_return import ( - RIDGE_RETURN_LOOKBACK_DAYS, - RIDGE_RETURN_MIN_RIDGE_BARS, -) -from data.clean.intraday_valley_quantile import ( - VALLEY_QUANTILE_LOOKBACK_DAYS, - VALLEY_QUANTILE_MIN_CROSS_SECTION, - VALLEY_QUANTILE_MIN_VALLEY_BARS, - VALLEY_QUANTILE_REVERSAL_DAYS, -) -from data.clean.intraday_valley_ridge_vwap import ( - VALLEY_RIDGE_LOOKBACK_DAYS, - VALLEY_RIDGE_MIN_RIDGE_BARS, - VALLEY_RIDGE_MIN_VALLEY_BARS, -) -from data.clean.intraday_valley_vwap import ( - VALLEY_VWAP_LOOKBACK_DAYS, - VALLEY_VWAP_MIN_VALLEY_BARS, -) -from data.clean.intraday_volume_prv import ( - VOLUME_PRV_BASELINE_DAYS, - VOLUME_PRV_BASELINE_MIN_OBS, - VOLUME_PRV_LOOKBACK_DAYS, - VOLUME_PRV_MIN_CLASSIFIABLE, - VOLUME_PRV_MIN_VALID_DAYS, - VOLUME_PRV_SIGMA_K, -) -from factors.base import Factor -from factors.spec import FactorSpec, PanelField - - -def _minute_requires(*fields: str) -> tuple[PanelField, ...]: - """The stk_mins_1min requires tuple of a minute-derived factor (D1). - - Truthful as-of-today provenance (design §9 D1 row: DECLARE, do not move - data flows — the materializer migration is D4): every factor in this - module is aggregated upstream from the cached 1min bars, so its endpoint - requirement is the 1min field set its aggregation reads. The daily panel - column each ``compute`` selects is DERIVED from exactly these fields. - """ - return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) - - -class JumpAmountCorrFactor(Factor): - """Price-jump turnover-correlation factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the - panel (produced by ``compute_jump_amount_corr``); it does NO minute work of its - own, mirroring the value / financial factors that surface an enriched column. - - Args: - lookback_days: trailing trading-day window; part of the factor DEFINITION - (reproduced from the report), not a tuned knob. It only names the - column so a non-default window cannot silently mislabel it. - """ - - name: str = f"jump_amount_corr_{JUMP_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = JUMP_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"jump-amount-corr lookback_days must be a positive integer; got " - f"{lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"jump_amount_corr_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=-1: the report's RankIC mean is -10.23% (full A, market-cap - + industry neutral) — high jump-amount-correlation predicts LOWER forward - returns. The sign is fixed BEFORE the run; a validated prototype reproduced - it (mean RankIC -0.074 on 2022-2024 sampled names). is_intraday=False by the - module docstring's reasoning (daily signal traded close-to-close). - min_history_bars=0: the warm-up is DATA-dependent (a value appears once - >= ``JUMP_MIN_PAIRS`` jump-pairs accumulate in the trailing window), not a - fixed leading count — the honest NaN rate is reported by data_coverage - rather than hidden behind a fabricated warm-up window. - - D1 declarations (D0 pre-assignment table row 1): adjustment= - returns_invariant — the jump is a within-(symbol, day) amplitude - z-score of the SAME-DAY ratio (high-low)/open - (data/clean/intraday_aggregate.py:420) and the correlated quantity is - ``amount`` (anchor-free), so the adjustment anchor cancels. - overnight_boundary=none — jump/amount pairs are strictly same-session - adjacent minutes (intraday_aggregate.py:421-422); no raw-price - comparison crosses the overnight boundary. - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Price-jump turnover correlation (Kaiyuan report §6): trailing " - f"{self._lookback_days}-trading-day lagged Pearson corr between the " - f"traded amount at price-JUMP minutes (within-day amplitude z-score " - f">1) and the amount at the strictly-next minute. Derived from 1min " - f"bars but a DAILY signal traded close-to-close; >= {JUMP_MIN_PAIRS} " - f"jump-pairs required else NaN." - ), - expected_ic_sign=-1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar fields the upstream aggregation is derived from. These - # are declared for honest provenance disclosure (data_coverage lists - # them); the daily panel surfaces the pre-aggregated column itself. - input_fields=("high", "low", "open", "amount"), - requires=_minute_requires("high", "low", "open", "amount"), - adjustment="returns_invariant", - overnight_boundary="none", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily jump-amount-corr column off ``panel``. - - The runner runs ``compute_jump_amount_corr`` on the minute cache upstream - and joins the result as ``self.name``; here we only surface it, so this - factor does no temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"JumpAmountCorrFactor needs the pre-aggregated '{self.name}' column " - f"on the panel (produced upstream by compute_jump_amount_corr and " - f"joined by the runner); panel has {list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class MinuteIdealAmplitudeFactor(Factor): - """Minute ideal-amplitude factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by ``data.clean.intraday_amplitude.compute_minute_ideal_amplitude``); - it does NO minute work of its own, mirroring :class:`JumpAmountCorrFactor` and - the value / financial factors that surface an enriched column. - - Args: - lookback_days: trailing trading-day window; part of the factor DEFINITION - (reproduced from the report), not a tuned knob. It only names the column - so a non-default window cannot silently mislabel it. - """ - - name: str = f"minute_ideal_amp_{IDEAL_AMP_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = IDEAL_AMP_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"minute-ideal-amplitude lookback_days must be a positive integer; " - f"got {lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"minute_ideal_amp_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=-1: the report's RankIC mean is -7.6% (full market, N=10, - lambda=25%) — a HIGH minute ideal amplitude predicts LOWER forward returns. - The sign is fixed BEFORE the run (a validated prototype must reproduce it). - is_intraday=False by the module docstring's reasoning: minute INPUT but a - DAILY signal traded close-to-close. min_history_bars=0: the warm-up is - DATA-dependent (a value appears once >= ``IDEAL_AMP_MIN_MINUTES`` valid - pooled minutes accumulate in the trailing window), not a fixed leading - count — the honest NaN rate is reported by data_coverage. - - D1 declarations (D0 pre-assignment table row 2 + note 1, the table's - flagged judgment call): adjustment=returns_invariant — the amplitude - itself is the same-day ratio high/low - 1 (anchor cancels; - data/clean/intraday_amplitude.py:24-27). overnight_boundary= - CROSSED_DISCLOSED (candidate) — the pooled ranking key is the RAW - minute close across the trailing multi-day window, so when the window - contains an ex-date the pooled ordering interleaves bars on two - different price bases; the definition is pinned to the report (Wind - ranks on the raw price) and deliberately kept. OPEN OBLIGATION, - tracked here per D0 note 1: of the three CROSSED_DISCLOSED - requirements (crossing is real / values kept by definition / deviation - MEASURED and disclosed) the third — a PR-L-style ex-date deviation - measurement (share of pooling windows containing a true ex-date + - realized rank-perturbation magnitude) — is still MISSING and is owed - in D2 alongside that stage's overnight-boundary property tests. - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Minute ideal amplitude (Kaiyuan report §30): pool the 1min bars of " - f"the trailing {self._lookback_days} trading days (PIT-truncated at " - f"14:50 per bar), rank the pooled minutes by RAW close, and return " - f"V_high - V_low where V_high/V_low are the mean per-minute amplitude " - f"(high/low - 1) of the top / bottom floor({IDEAL_AMP_LAMBDA:g}*n) " - f"minutes by close. Derived from 1min bars but a DAILY signal traded " - f"close-to-close; >= {IDEAL_AMP_MIN_MINUTES} valid pooled minutes " - f"required else NaN." - ), - expected_ic_sign=-1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar fields the upstream aggregation is derived from. Declared - # for honest provenance disclosure (data_coverage lists them); the daily - # panel surfaces the pre-aggregated column itself. - input_fields=("high", "low", "close"), - requires=_minute_requires("high", "low", "close"), - adjustment="returns_invariant", - overnight_boundary="crossed_disclosed", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily minute-ideal-amplitude column off ``panel``. - - The runner runs ``compute_minute_ideal_amplitude`` on the minute cache - upstream and joins the result as ``self.name``; here we only surface it, so - this factor does no temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"MinuteIdealAmplitudeFactor needs the pre-aggregated '{self.name}' " - f"column on the panel (produced upstream by " - f"compute_minute_ideal_amplitude and joined by the runner); panel has " - f"{list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class AmpMarginalAnomalyVolFactor(Factor): - """Amplitude marginal-anomaly relative-volatility factor (daily, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by ``data.clean.intraday_amp_anomaly.compute_amp_marginal_anomaly_vol``); - it does NO minute work of its own, mirroring :class:`JumpAmountCorrFactor` / - :class:`MinuteIdealAmplitudeFactor` and the value / financial factors that surface - an enriched column. - - Args: - lookback_days: trailing trading-day window; part of the factor DEFINITION - (a pinned interpretation of the report), not a tuned knob. It only names - the column so a non-default window cannot silently mislabel it. - """ - - name: str = f"amp_marginal_anomaly_vol_{AMP_ANOMALY_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = AMP_ANOMALY_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"amp-marginal-anomaly-vol lookback_days must be a positive integer; " - f"got {lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"amp_marginal_anomaly_vol_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=+1: the report's IC is POSITIVE across its universes (raw - CSI800 +4.47% / full-market +4.92%; market-cap + industry neutral +4.11% / - +5.56%) — a HIGH anomaly-bar relative volatility predicts HIGHER forward - returns. The sign is fixed BEFORE the run (a validated prototype must - reproduce it). NOTE the report's sample is CSI800 / full-market on a MONTHLY - series while our eval cell is CSI500 daily, so the report numbers are a LOOSE - reference only (disclosed, never mislabeled). is_intraday=False by the module - docstring's reasoning: minute INPUT but a DAILY signal traded close-to-close. - min_history_bars=0: the warm-up is DATA-dependent (a value appears once - >= ``AMP_ANOMALY_MIN_POOL`` valid pooled pairs accumulate in the trailing - window), not a fixed leading count — the honest NaN rate is reported by - data_coverage. - - The description spells out the FIVE pinned interpretations of the - under-specified report (bar freq, lookback, threshold, weighted-vol operator, - within-day lag) so a reader sees exactly what was assumed. - - D1 declarations (D0 pre-assignment table row 3): adjustment= - returns_invariant — amp is the same-day ratio high/low - 1 and the - selected-bar statistic is a std of minute RETURNS (ratios), so the - anchor cancels throughout. overnight_boundary=none — Δamp and the - bar-return are BOTH within-day lagged, "the overnight gap never - contaminates a pair" (data/clean/intraday_amp_anomaly.py:25-26). - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Amplitude marginal-anomaly relative volatility (Changjiang HF-factor " - f"series #19). PINNED interpretations of an under-specified report: " - f"(1) {AMP_ANOMALY_FREQ} bars DERIVED from the 1min cache " - f"(available_time = max source, PIT-faithful); (2) trailing " - f"{self._lookback_days} trading days (PIT-truncated at 14:50 per bar); " - f"(3) select bars with |Δamp| > μ + {AMP_ANOMALY_SIGMA_K:g}σ of the " - f"pooled |Δamp|; (4) factor = ddof=1 std of the RETURNS on the selected " - f"bars; (5) Δamp and bar-return are WITHIN-DAY lagged (each day's first " - f"bar has neither). amp = high/low - 1. Derived from 1min bars but a " - f"DAILY signal traded close-to-close; >= {AMP_ANOMALY_MIN_POOL} valid " - f"pooled pairs and >= {AMP_ANOMALY_MIN_SELECTED} selected bars required " - f"else NaN." - ), - expected_ic_sign=1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar fields the upstream aggregation is derived from. Declared for - # honest provenance disclosure (data_coverage lists them); the daily panel - # surfaces the pre-aggregated column itself. - input_fields=("high", "low", "close"), - requires=_minute_requires("high", "low", "close"), - adjustment="returns_invariant", - overnight_boundary="none", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily amp-marginal-anomaly-vol column off ``panel``. - - The runner runs ``compute_amp_marginal_anomaly_vol`` on the minute cache - upstream and joins the result as ``self.name``; here we only surface it, so this - factor does no temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"AmpMarginalAnomalyVolFactor needs the pre-aggregated '{self.name}' " - f"column on the panel (produced upstream by " - f"compute_amp_marginal_anomaly_vol and joined by the runner); panel has " - f"{list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class VolumePeakCountFactor(Factor): - """Volume-peak-minute-count factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by ``data.clean.intraday_volume_prv.compute_volume_peak_count``); it does - NO minute work of its own, mirroring :class:`JumpAmountCorrFactor` / - :class:`MinuteIdealAmplitudeFactor` / :class:`AmpMarginalAnomalyVolFactor` and the - value / financial factors that surface an enriched column. - - Args: - lookback_days: trailing VALID trading-day count window; part of the factor - DEFINITION (a pinned interpretation of the report), not a tuned knob. It - only names the column so a non-default window cannot silently mislabel it. - """ - - name: str = f"volume_peak_count_{VOLUME_PRV_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = VOLUME_PRV_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"volume-peak-count lookback_days must be a positive integer; got " - f"{lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"volume_peak_count_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=+1: the report's IC is POSITIVE (full-market RankIC +10.62% / - RankICIR 4.36; CSI500 sub-domain long-short +14.96%/yr) — more volume peaks - (informed-trading participation) predicts HIGHER forward returns. The sign is - fixed BEFORE the run (a validated prototype must reproduce it). NOTE the report - is a MONTHLY, market-cap + industry neutral series on Wind data while our eval - cell is CSI500 daily with industry + size neutral, so the report numbers are a - LOOSE reference only (disclosed, never mislabeled). is_intraday=False by the - module docstring's reasoning: minute INPUT but a DAILY signal traded - close-to-close. min_history_bars=0: the warm-up is DATA-dependent (a value - appears once >= ``VOLUME_PRV_MIN_VALID_DAYS`` valid days accumulate in the - trailing window, and a day is valid only once its same-slot baselines fill in), - not a fixed leading count — the honest NaN rate is reported by data_coverage. - - The description spells out the pinned interpretations of the under-specified - report (PIT truncation, strictly-prior same-slot baseline, μ+σ eruptive - threshold, mild-neighbour peak rule, valid-day gate) so a reader sees exactly - what was assumed. - - D1 declarations (D0 pre-assignment table row 4 + note 3): adjustment= - none — the factor reads the pure volume channel, never a price - (data/clean/intraday_volume_prv.py:52-55). overnight_boundary=none — - no raw-price comparison exists. The KNOWN raw-volume magnitude jump - across split days is disclosed at the definition site and deliberately - NOT corrected (report alignment); per D0 note 3 it belongs to NEITHER - taxonomy axis — do not "fix" it via these declarations. - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Volume-peak-minute count (Kaiyuan microstructure series #27). PINNED " - f"interpretations of an under-specified report: (1) 1min bars " - f"PIT-truncated at 14:50 per bar; (2) same-slot baseline = μ/σ (ddof=1) " - f"of the STRICTLY-PRIOR {VOLUME_PRV_BASELINE_DAYS} trading days' " - f"same-slot volume, needing >= {VOLUME_PRV_BASELINE_MIN_OBS} obs else " - f"unclassifiable; (3) a minute is ERUPTIVE if vol > μ + " - f"{VOLUME_PRV_SIGMA_K:g}σ else MILD; (4) a PEAK is an eruptive minute " - f"whose both 1-minute same-session neighbours exist and are mild " - f"(ridge / session-boundary / unclassifiable-neighbour minutes are not " - f"peaks); (5) factor = peak-minute count over the trailing " - f"{self._lookback_days} VALID days (>= {VOLUME_PRV_MIN_CLASSIFIABLE} " - f"classifiable bars) including d, NaN below " - f"{VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived from 1min bars but a " - f"DAILY signal traded close-to-close." - ), - expected_ic_sign=1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar field the upstream aggregation is derived from. Declared for - # honest provenance disclosure (data_coverage lists it); the daily panel - # surfaces the pre-aggregated column itself. - input_fields=("volume",), - requires=_minute_requires("volume"), - adjustment="none", - overnight_boundary="none", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily volume-peak-count column off ``panel``. - - The runner runs ``compute_volume_peak_count`` on the minute cache upstream and - joins the result as ``self.name``; here we only surface it, so this factor does - no temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"VolumePeakCountFactor needs the pre-aggregated '{self.name}' column " - f"on the panel (produced upstream by compute_volume_peak_count and " - f"joined by the runner); panel has {list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class IntradayAmpCutFactor(Factor): - """Intraday amplitude-cut factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by ``data.clean.intraday_amp_cut.compute_amp_cut_stats`` + - ``combine_amp_cut_cross_section``); it does NO minute work of its own, mirroring - :class:`JumpAmountCorrFactor` / :class:`MinuteIdealAmplitudeFactor` / - :class:`AmpMarginalAnomalyVolFactor` / :class:`VolumePeakCountFactor` and the value / - financial factors that surface an enriched column. - - Args: - lookback_days: trailing VALID trading-day window; part of the factor DEFINITION - (reproduced from the report), not a tuned knob. It only names the column so a - non-default window cannot silently mislabel it. - """ - - name: str = f"intraday_amp_cut_{AMP_CUT_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = AMP_CUT_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"intraday-amp-cut lookback_days must be a positive integer; got " - f"{lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"intraday_amp_cut_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=-1: the report's rankIC mean is -0.067 (rankICIR -3.82, quintile - long-short 16.7%/yr at N=10, lambda=20%, 1-minute-return indicator) — a HIGH - intraday amplitude-cut value predicts LOWER forward returns; the V_mean and V_std - sub-factors are each negative too. The sign is fixed BEFORE the run (a validated - prototype must reproduce it). is_intraday=False by the module docstring's - reasoning: minute INPUT but a DAILY signal traded close-to-close. min_history_bars= - 0: the warm-up is DATA-dependent (a value appears once >= ``AMP_CUT_MIN_VALID_DAYS`` - valid days accumulate AND the cross-section has >= ``AMP_CUT_MIN_CROSS_SECTION`` - finite pairs), not a fixed leading count — the honest NaN rate is reported by - data_coverage. - - The description spells out the DISTINCTION FROM PR-D (``minute_ideal_amp``): PR-D - pools the 10-day minutes into ONE set and cuts by minute CLOSE PRICE; this factor - cuts EACH DAY by the 1-MINUTE RETURN, then takes the trailing-10-valid-day mean / - std of the daily cut and combines them cross-sectionally (the report finds the two - only ~30% correlated). - - D1 declarations (D0 pre-assignment table row 5): adjustment= - returns_invariant — amp is the same-day ratio high/low - 1 and the - cut key r = close_t/close_{t-1} - 1 is a within-day ratio, so the - anchor cancels in both. overnight_boundary=none — r is WITHIN-DAY - lagged, "no overnight gap; PR-E precedent" - (data/clean/intraday_amp_cut.py:27-28). - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Intraday amplitude cut (Kaiyuan microstructure series #30, SECOND " - f"factor 日内振幅切割). DISTINCT FROM PR-D minute_ideal_amp, which pools the " - f"trailing days' minutes into ONE set and cuts by minute CLOSE PRICE: " - f"this factor cuts EACH DAY independently by the 1-MINUTE RETURN " - f"r=close_t/close_{{t-1}}-1 (within-day lagged, first bar of each day has " - f"no r), taking V_day = V_high - V_low where V_high/V_low are the mean " - f"amp (high/low-1) of the top/bottom floor({AMP_CUT_LAMBDA:g}*n_day) bars " - f"by return (day valid iff >= {AMP_CUT_MIN_DAY_MINUTES} valid bars). " - f"Trailing {self._lookback_days} VALID days give V_mean / V_std (>= " - f"{AMP_CUT_MIN_VALID_DAYS} valid days else NaN); per date they are each " - f"cross-sectionally z-scored over the covered universe (>= " - f"{AMP_CUT_MIN_CROSS_SECTION} finite pairs else NaN) and averaged: factor " - f"= (z(V_mean) + z(V_std))/2. Derived from 1min bars but a DAILY signal " - f"traded close-to-close (report finds ~30% corr with minute_ideal_amp)." - ), - expected_ic_sign=-1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar fields the upstream aggregation is derived from. Declared for - # honest provenance disclosure (data_coverage lists them); the daily panel - # surfaces the pre-aggregated column itself. - input_fields=("high", "low", "close"), - requires=_minute_requires("high", "low", "close"), - adjustment="returns_invariant", - overnight_boundary="none", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily intraday-amp-cut column off ``panel``. - - The runner runs ``compute_amp_cut_stats`` per symbol on the minute cache upstream, - assembles the full-universe ``(V_mean, V_std)`` panel, applies - ``combine_amp_cut_cross_section``, and joins the result as ``self.name``; here we - only surface it, so this factor does no temporal logic and cannot introduce - lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"IntradayAmpCutFactor needs the pre-aggregated '{self.name}' column on " - f"the panel (produced upstream by compute_amp_cut_stats + " - f"combine_amp_cut_cross_section and joined by the runner); panel has " - f"{list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class PeakIntervalKurtosisFactor(Factor): - """Volume-peak interval-kurtosis factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by ``data.clean.intraday_peak_interval.compute_peak_interval_kurtosis``); - it does NO minute work of its own, mirroring :class:`JumpAmountCorrFactor` / - :class:`MinuteIdealAmplitudeFactor` / :class:`AmpMarginalAnomalyVolFactor` / - :class:`VolumePeakCountFactor` / :class:`IntradayAmpCutFactor` and the value / - financial factors that surface an enriched column. - - Args: - lookback_days: trailing VALID trading-day window pooled for the kurtosis; part of - the factor DEFINITION (reproduced from the report), not a tuned knob. It only - names the column so a non-default window cannot silently mislabel it. - """ - - name: str = f"peak_interval_kurtosis_{PEAK_INTERVAL_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = PEAK_INTERVAL_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"peak-interval-kurtosis lookback_days must be a positive integer; got " - f"{lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"peak_interval_kurtosis_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=+1: the report's full-market RankIC is +7.19% (RankICIR 4.63, - long-short 23.3%/yr, IR 3.39, max drawdown 7.37%, 13/13 positive years — the most - stable factor in the report). A peaky, fat-tailed peak-interval distribution means - informed trading arrives in BURSTS rather than evenly through the session. The - sign is fixed BEFORE the run (a validated prototype must reproduce it). - is_intraday=False for the same reason as the siblings: minute INPUT but a DAILY - signal traded close-to-close. min_history_bars=0: the warm-up is DATA-dependent (a - value appears once enough VALID days AND enough pooled intervals accumulate), not - a fixed leading count — the honest NaN rate is reported by data_coverage. - - The description spells out the DISTINCTION FROM PR-F (``volume_peak_count``): the - SAME peak identification (reused, not re-implemented) reduced by a DIFFERENT - statistic — the shape of the gap distribution rather than the peak count — plus - the two interpretations the report leaves open (interval unit, kurtosis - convention). - - D1 declarations (D0 pre-assignment table row 6): adjustment=none — the - peak identification is PR-F's pure volume machinery (reused, not - re-implemented; data/clean/intraday_peak_interval.py:8-9) and the - statistic is a kurtosis of trading-minute POSITION gaps (:20-21), so - no price is ever read. overnight_boundary=none — no raw-price - comparison exists. The split-day raw-volume σ pollution is disclosed - (same as PR-F, :51-53) and per D0 note 3 belongs to neither axis. - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Volume-peak interval kurtosis (Kaiyuan microstructure series #27, " - f"SECOND factor 量峰间隔峰度). SAME peak identification as PR-F " - f"volume_peak_count (REUSED from data.clean.intraday_volume_prv, not " - f"re-implemented): 1min bars PIT-truncated at 14:50, a minute is ERUPTIVE " - f"if vol > μ + {VOLUME_PRV_SIGMA_K:g}σ of its SAME-SLOT strictly-prior " - f"{VOLUME_PRV_BASELINE_DAYS}-day baseline, and a PEAK is an eruptive " - f"minute whose both 1-minute same-session neighbours are mild. DIFFERENT " - f"STATISTIC: the gaps between consecutive same-day peaks, pooled over the " - f"trailing {self._lookback_days} VALID days (a day with < 2 peaks " - f"contributes 0 intervals but is still valid), reduced to their kurtosis. " - f"PINNED interpretations of an under-specified report: (1) an interval is " - f"measured in TRADING MINUTES — the tradable-slot difference inside the " - f"day's visible bar sequence, so the lunch break costs nothing (11:29 and " - f"13:02 peaks are 3 apart, not 93) and a wall-clock ~90-minute spike can " - f"never dominate the distribution; (2) kurtosis = FISHER excess, " - f"bias-corrected (the pandas .kurt() / scipy fisher=True bias=False " - f"convention; normal = 0). NaN unless >= " - f"{PEAK_INTERVAL_MIN_INTERVALS} intervals pooled and the pool has " - f"non-zero variance. Derived from 1min bars but a DAILY signal traded " - f"close-to-close." - ), - expected_ic_sign=1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar field the upstream aggregation is derived from. Declared for - # honest provenance disclosure (data_coverage lists it); the daily panel - # surfaces the pre-aggregated column itself. - input_fields=("volume",), - requires=_minute_requires("volume"), - adjustment="none", - overnight_boundary="none", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily peak-interval-kurtosis column off ``panel``. - - The runner runs ``compute_peak_interval_kurtosis`` per symbol on the minute cache - upstream and joins the result as ``self.name``; here we only surface it, so this - factor does no temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"PeakIntervalKurtosisFactor needs the pre-aggregated '{self.name}' " - f"column on the panel (produced upstream by " - f"compute_peak_interval_kurtosis and joined by the runner); panel has " - f"{list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class ValleyRelativeVwapFactor(Factor): - """Valley-relative VWAP factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by ``data.clean.intraday_valley_vwap.compute_valley_relative_vwap``); it - does NO minute work of its own, mirroring :class:`JumpAmountCorrFactor` / - :class:`MinuteIdealAmplitudeFactor` / :class:`AmpMarginalAnomalyVolFactor` / - :class:`VolumePeakCountFactor` / :class:`IntradayAmpCutFactor` / - :class:`PeakIntervalKurtosisFactor` and the value / financial factors that surface an - enriched column. - - Args: - lookback_days: trailing VALID trading-day window averaged; part of the factor - DEFINITION (reproduced from the report), not a tuned knob. It only names the - column so a non-default window cannot silently mislabel it. - """ - - name: str = f"valley_relative_vwap_{VALLEY_VWAP_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = VALLEY_VWAP_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"valley-relative-vwap lookback_days must be a positive integer; got " - f"{lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"valley_relative_vwap_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=+1: the report's full-market RankIC is +8.69% (RankICIR 4.44, - long-short 25.35%/yr, IR 3.04, monthly win rate 79.7% — the strongest factor in - the report), and its CSI500 sub-domain long-short is 9.94% / IR 1.26, the closest - comparable to our eval cell. Semantics per the report: valley minutes are moments - of subdued sentiment where prices are unlikely to have over-reacted, so a HIGH - relative valley price predicts HIGHER forward returns. The sign is fixed BEFORE - the run (a validated prototype must reproduce it). NOTE the report is a MONTHLY, - market-cap + industry neutral full-market series on Wind data while our eval cell - is CSI500 daily with industry + size neutral, so the report numbers are a LOOSE - reference only (disclosed, never mislabeled, never written in as an expected - value). is_intraday=False by the module docstring's reasoning: minute INPUT but a - DAILY signal traded close-to-close. min_history_bars=0: the warm-up is - DATA-dependent (a value appears once enough VALID days accumulate), not a fixed - leading count — the honest NaN rate is reported by data_coverage. - - The description spells out the DIFFERENT FAMILY vs PR-F / PR-H (the same reused - classification, but a PRICE LEVEL at the VALLEYS rather than a count or timing of - the PEAKS) plus the pinned choices, including the one place we KNOWINGLY DEVIATE - from the report: our day VWAP covers the PIT-visible window only. - - D1 declarations (D0 pre-assignment table row 7): adjustment= - returns_invariant — both VWAP legs are sums over the SAME trading day - and "a split/dividend adjustment factor is constant within a day, so - it cancels exactly in the ratio" - (data/clean/intraday_valley_vwap.py:33-35). Price information arrives - via Σamount/Σvolume, which is why requires lists no OHLC field — the - anomaly is real and intended. overnight_boundary=none — both legs are - same-day; nothing crosses the overnight boundary. - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Valley-relative VWAP (Kaiyuan microstructure series #27, THIRD factor " - f"量谷相对加权价格). SAME minute classification as PR-F volume_peak_count " - f"and PR-H peak_interval_kurtosis (REUSED from " - f"data.clean.intraday_volume_prv, not re-implemented): 1min bars " - f"PIT-truncated at 14:50, a minute is ERUPTIVE if vol > μ + " - f"{VOLUME_PRV_SIGMA_K:g}σ of its SAME-SLOT strictly-prior " - f"{VOLUME_PRV_BASELINE_DAYS}-day baseline, else it is a VALLEY (量谷). " - f"DIFFERENT FAMILY: instead of counting or timing the PEAKS this factor " - f"prices the VALLEYS — daily ratio = (valley VWAP) / (whole visible day " - f"VWAP), averaged over the trailing {self._lookback_days} VALID days. " - f"PINNED choices: (1) each VWAP uses the aggregation identity Σ(p·v)/Σv " - f"= Σamount/Σvolume, the day's REAL volume-weighted price rather than a " - f"close approximation; (2) bars with non-finite or non-positive volume " - f"or amount are dropped from BOTH sums (guard applied at summation only, " - f"so PR-F's baseline is untouched); (3) RAW unadjusted prices are correct " - f"here because the adjustment factor is constant within a day and cancels " - f"in the ratio; (4) DEVIATION FROM THE REPORT, disclosed: our day VWAP " - f"spans the PIT-VISIBLE window 09:31-14:50 only, not the full session — " - f"reading the close would be lookahead at our 14:50 decision time; " - f"(5) a day is VALID iff it has >= {VOLUME_PRV_MIN_CLASSIFIABLE} " - f"classifiable bars AND >= {VALLEY_VWAP_MIN_VALLEY_BARS} TRADABLE valley " - f"bars (counted after the guard) AND positive volume in both " - f"denominators; NaN below {VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived " - f"from 1min bars but a DAILY signal traded close-to-close." - ), - expected_ic_sign=1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar fields the upstream aggregation is derived from. Declared for - # honest provenance disclosure (data_coverage lists them); the daily panel - # surfaces the pre-aggregated column itself. - input_fields=("volume", "amount"), - requires=_minute_requires("volume", "amount"), - adjustment="returns_invariant", - overnight_boundary="none", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily valley-relative-VWAP column off ``panel``. - - The runner runs ``compute_valley_relative_vwap`` per symbol on the minute cache - upstream and joins the result as ``self.name``; here we only surface it, so this - factor does no temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"ValleyRelativeVwapFactor needs the pre-aggregated '{self.name}' column " - f"on the panel (produced upstream by compute_valley_relative_vwap and " - f"joined by the runner); panel has {list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class ValleyRidgeVwapRatioFactor(Factor): - """Valley/ridge VWAP-ratio factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by - ``data.clean.intraday_valley_ridge_vwap.compute_valley_ridge_vwap_ratio``); it does - NO minute work of its own, mirroring :class:`JumpAmountCorrFactor` / - :class:`MinuteIdealAmplitudeFactor` / :class:`AmpMarginalAnomalyVolFactor` / - :class:`VolumePeakCountFactor` / :class:`IntradayAmpCutFactor` / - :class:`PeakIntervalKurtosisFactor` / :class:`ValleyRelativeVwapFactor` and the value - / financial factors that surface an enriched column. - - Args: - lookback_days: trailing VALID trading-day window averaged; part of the factor - DEFINITION (reproduced from the report), not a tuned knob. It only names the - column so a non-default window cannot silently mislabel it. - """ - - name: str = f"valley_ridge_vwap_ratio_{VALLEY_RIDGE_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = VALLEY_RIDGE_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"valley-ridge-vwap-ratio lookback_days must be a positive integer; got " - f"{lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"valley_ridge_vwap_ratio_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=+1: the report's full-market RankIC is +6.98% (RankICIR 3.56, - long-short 15.83%/yr, IR 1.83, monthly win rate 72.3%, only 2023 negative across - 13 years), and its CSI500 sub-domain long-short is 10.49% / IR 1.34, the closest - comparable to our eval cell. Semantics per the report: a HIGH valley/ridge price - ratio means retail over-reaction pushed the eruptive minutes' price DOWN relative - to the calm ones, so the stock is depressed and performs better going forward. - The sign is fixed BEFORE the run (a validated prototype must reproduce it). NOTE - the report is a MONTHLY, market-cap + industry neutral full-market series on Wind - data while our eval cell is CSI500 daily with industry + size neutral, so the - report numbers are a LOOSE reference only (disclosed, never mislabeled, never - written in as an expected value). is_intraday=False by the module docstring's - reasoning: minute INPUT but a DAILY signal traded close-to-close. - min_history_bars=0: the warm-up is DATA-dependent (a value appears once enough - VALID days accumulate), not a fixed leading count — the honest NaN rate is - reported by data_coverage. - - The description spells out the RELATION TO PR-I (same reused classification, same - VWAP identity, DENOMINATOR swapped from the whole visible day to the ridge bars) - plus the pinned choices — above all the ASYMMETRIC bar floor, which exists - because ridge bars are structurally far scarcer than valley bars. - - D1 declarations (D0 pre-assignment table row 8): adjustment= - returns_invariant — the same same-day two-leg cancellation argument as - PR-I, with the denominator swapped to the ridge VWAP - (data/clean/intraday_valley_ridge_vwap.py:42-45); price arrives via - Σamount/Σvolume, so requires lists no OHLC field. overnight_boundary= - none — both legs are same-day. - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Valley/ridge VWAP ratio (Kaiyuan microstructure series #27, FOURTH " - f"factor 谷岭加权价格比). SAME minute classification as PR-F " - f"volume_peak_count / PR-H peak_interval_kurtosis / PR-I " - f"valley_relative_vwap (REUSED from data.clean.intraday_volume_prv, not " - f"re-implemented): 1min bars PIT-truncated at 14:50, a minute is ERUPTIVE " - f"if vol > μ + {VOLUME_PRV_SIGMA_K:g}σ of its SAME-SLOT strictly-prior " - f"{VOLUME_PRV_BASELINE_DAYS}-day baseline, else it is a VALLEY (量谷). " - f"DENOMINATOR SWAPPED vs PR-I: instead of the whole visible day's VWAP " - f"the divisor is the RIDGE (量岭) VWAP, so the two behavioural groups are " - f"contrasted head-on — daily ratio = (valley VWAP) / (ridge VWAP), " - f"averaged over the trailing {self._lookback_days} VALID days. PINNED " - f"choices: (1) the RIDGE mask is 'eruptive AND NOT an isolated peak', " - f"which is WIDER than 'eruptive next to an eruptive' — it also covers " - f"session-boundary eruptions and eruptions with an unclassifiable " - f"neighbour, keeping valley|peak|ridge an exact partition of the " - f"classifiable bars; an isolated PEAK contributes to NEITHER leg; " - f"(2) each VWAP uses the aggregation identity Σ(p·v)/Σv = Σamount/Σvolume; " - f"(3) bars with non-finite or non-positive volume or amount are dropped " - f"from BOTH sums (guard applied at summation only, so PR-F's baseline is " - f"untouched); (4) RAW unadjusted prices are correct here because the " - f"adjustment factor is constant within a day and cancels in the ratio; " - f"(5) DEVIATION FROM THE REPORT, disclosed: both legs span the " - f"PIT-VISIBLE window 09:31-14:50 only, not the full session — reading the " - f"close would be lookahead at our 14:50 decision time; (6) a day is VALID " - f"iff it has >= {VOLUME_PRV_MIN_CLASSIFIABLE} classifiable bars AND >= " - f"{VALLEY_RIDGE_MIN_VALLEY_BARS} TRADABLE valley bars AND >= " - f"{VALLEY_RIDGE_MIN_RIDGE_BARS} TRADABLE ridge bars (both counted AFTER " - f"the guard) AND positive volume in both denominators — the ridge floor " - f"is deliberately LOWER because a ridge bar must erupt AND fail the " - f"isolation test, making ridges structurally far scarcer than valleys, " - f"and the realized ridge-bar distribution plus day-validity rate are " - f"REPORTED by the runner rather than left implicit; NaN below " - f"{VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived from 1min bars but a " - f"DAILY signal traded close-to-close." - ), - expected_ic_sign=1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar fields the upstream aggregation is derived from. Declared for - # honest provenance disclosure (data_coverage lists them); the daily panel - # surfaces the pre-aggregated column itself. - input_fields=("volume", "amount"), - requires=_minute_requires("volume", "amount"), - adjustment="returns_invariant", - overnight_boundary="none", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily valley/ridge VWAP-ratio column off ``panel``. - - The runner runs ``compute_valley_ridge_vwap_ratio`` per symbol on the minute - cache upstream and joins the result as ``self.name``; here we only surface it, so - this factor does no temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"ValleyRidgeVwapRatioFactor needs the pre-aggregated '{self.name}' " - f"column on the panel (produced upstream by " - f"compute_valley_ridge_vwap_ratio and joined by the runner); panel has " - f"{list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class RidgeMinuteReturnFactor(Factor): - """Ridge minute-return factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by ``data.clean.intraday_ridge_return.compute_ridge_minute_return``); it does - NO minute work of its own, mirroring :class:`JumpAmountCorrFactor` / - :class:`MinuteIdealAmplitudeFactor` / :class:`AmpMarginalAnomalyVolFactor` / - :class:`VolumePeakCountFactor` / :class:`IntradayAmpCutFactor` / - :class:`PeakIntervalKurtosisFactor` / :class:`ValleyRelativeVwapFactor` / - :class:`ValleyRidgeVwapRatioFactor` and the value / financial factors that surface an - enriched column. - - Args: - lookback_days: trailing VALID trading-day window summed; part of the factor - DEFINITION (reproduced from the report), not a tuned knob. It only names the - column so a non-default window cannot silently mislabel it. - """ - - name: str = f"ridge_minute_return_{RIDGE_RETURN_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = RIDGE_RETURN_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"ridge-minute-return lookback_days must be a positive integer; got " - f"{lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"ridge_minute_return_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=-1: this is the report's ONLY NEGATIVE peak/ridge/valley factor - (full-market RankIC -6.29%, RankICIR -3.55, long leg 7.47%/yr, long-short - 14.98%/yr, IR 1.73, max drawdown 13.84%, monthly win rate 70.3%). The report gives - NO CSI500 sub-domain figure for this factor, so none is quoted here. Semantics per - the report: ridge minutes are retail follow-the-crowd trading and their - accumulated return measures that crowd's OVER-REACTION — the more ridge-minute - return a stock has piled up, the more over-extended it is and the worse it - performs going forward. The sign is fixed BEFORE the run (a validated prototype - must reproduce it). NOTE the report is a MONTHLY, market-cap + industry neutral - full-market series on Wind data while our eval cell is CSI500 daily with industry - + size neutral, so the report numbers are a LOOSE reference only (disclosed, never - mislabeled, never written in as an expected value). is_intraday=False by the - module docstring's reasoning: minute INPUT but a DAILY signal traded - close-to-close. min_history_bars=0: the warm-up is DATA-dependent (a value appears - once enough VALID days accumulate), not a fixed leading count — the honest NaN - rate is reported by data_coverage. - - The description spells out the RELATION TO PR-F..PR-J (same reused classification, - a RETURN statistic instead of a count / timing moment / price level) plus the - pinned choices — above all the SIMPLE-SUM convention, the within-day return lag and - the raw closes, none of which the report specifies. - - D1 declarations (D0 pre-assignment table row 9): adjustment= - returns_invariant — the minute return close_t/close_(t-1) - 1 is a - within-day ratio, and "a split/dividend adjustment factor is constant - WITHIN a day, so it cancels exactly" - (data/clean/intraday_ridge_return.py:39-42). overnight_boundary=none — - the same docstring's own claim, "no return ever straddles an ex-date - boundary" (the within-day lag drops each day's first visible bar). - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Ridge minute-return (Kaiyuan microstructure series #27, FIFTH factor " - f"量岭分钟收益). SAME minute classification as PR-F volume_peak_count / " - f"PR-H peak_interval_kurtosis / PR-I valley_relative_vwap / PR-J " - f"valley_ridge_vwap_ratio (REUSED from data.clean.intraday_volume_prv, " - f"not re-implemented): 1min bars PIT-truncated at 14:50, a minute is " - f"ERUPTIVE if vol > μ + {VOLUME_PRV_SIGMA_K:g}σ of its SAME-SLOT " - f"strictly-prior {VOLUME_PRV_BASELINE_DAYS}-day baseline, and a RIDGE " - f"(量岭) is an eruptive minute that is NOT an isolated peak. NEW " - f"STATISTIC vs PR-F..PR-J: a RETURN rather than a count, a timing moment " - f"or a price level — each day sums the minute returns of its ridge bars " - f"and the factor sums those daily sums over the trailing " - f"{self._lookback_days} VALID days. PINNED choices (the report specifies " - f"none of them): (1) the RIDGE mask is 'eruptive AND NOT an isolated " - f"peak', keeping valley|peak|ridge an exact partition of the classifiable " - f"bars — an isolated PEAK's return is counted on NEITHER side; (2) the " - f"minute return is close_t/close_(t-1) - 1 with a WITHIN-DAY lag against " - f"the previous VISIBLE bar of the same date, so each day's FIRST visible " - f"bar carries no return and no return ever crosses a day boundary; exact " - f"60s adjacency is deliberately NOT required, so a bar opening a new " - f"session block (13:01, after lunch) returns against the last bar before " - f"the gap — a genuine price change rather than a discarded one; (3) RAW " - f"unadjusted closes are correct here because the adjustment factor is " - f"constant within a day and cancels in the ratio, and the within-day lag " - f"already excludes the one bar that could straddle an ex-date; (4) a " - f"return is formed only when both closes are finite and strictly positive " - f"(guard applied at the return step only, so PR-F's baseline is " - f"untouched); (5) the daily aggregate is a SIMPLE SUM Σr, NOT a compound " - f"Π(1+r)-1 — ridge minutes are non-contiguous within the day so a " - f"holding-period reading does not apply, and at minute scale the two " - f"differ negligibly; (6) the trailing aggregate is likewise a SUM across " - f"days; (7) a day is VALID iff it has >= {VOLUME_PRV_MIN_CLASSIFIABLE} " - f"classifiable bars AND >= {RIDGE_RETURN_MIN_RIDGE_BARS} ridge bars " - f"CARRYING A VALID RETURN (counted AFTER the guard) — the same scarcity " - f"floor PR-J pinned for its ridge leg, since a ridge bar must erupt AND " - f"fail the isolation test; the realized ridge-bar distribution and " - f"day-validity rate are REPORTED by the runner rather than left implicit; " - f"(8) DEVIATION FROM THE REPORT, disclosed: everything spans the " - f"PIT-VISIBLE window 09:31-14:50 only, not the full session — reading the " - f"close would be lookahead at our 14:50 decision time. NaN below " - f"{VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived from 1min bars but a " - f"DAILY signal traded close-to-close." - ), - expected_ic_sign=-1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar fields the upstream aggregation is derived from. Declared for - # honest provenance disclosure (data_coverage lists them); the daily panel - # surfaces the pre-aggregated column itself. - input_fields=("volume", "close"), - requires=_minute_requires("volume", "close"), - adjustment="returns_invariant", - overnight_boundary="none", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily ridge-minute-return column off ``panel``. - - The runner runs ``compute_ridge_minute_return`` per symbol on the minute cache - upstream and joins the result as ``self.name``; here we only surface it, so this - factor does no temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"RidgeMinuteReturnFactor needs the pre-aggregated '{self.name}' column " - f"on the panel (produced upstream by compute_ridge_minute_return and " - f"joined by the runner); panel has {list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class ValleyPriceQuantileFactor(Factor): - """Valley weighted-price-quantile factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by - ``data.clean.intraday_valley_quantile.compute_valley_price_quantile_stats`` and then - reversal-neutralized by ``residualize_on_reversal``); it does NO minute work of its - own, mirroring the eight precedents in this module. - - Args: - lookback_days: trailing VALID trading-day window averaged; part of the factor - DEFINITION (reproduced from the report), not a tuned knob. It only names the - column so a non-default window cannot silently mislabel it. - """ - - name: str = f"valley_price_quantile_{VALLEY_QUANTILE_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = VALLEY_QUANTILE_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"valley-price-quantile lookback_days must be a positive integer; got " - f"{lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"valley_price_quantile_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=+1 (report full-market RankIC +6.34%, RankICIR 4.32, long leg - 13.1%/yr, long-short 20.22%/yr, IR 3.29, max drawdown 10.18%, monthly win rate - 80.4%; CSI500 sub-domain long-short 11.71% / IR 1.76 — the closest comparable to - our eval cell). Semantics per the report: a HIGH position of the calm-minute - (valley) price within the day's range means the informed, unhurried part of the - day traded near the top of the range -> higher future return. The sign is fixed - BEFORE the run (a validated prototype must reproduce it). NOTE the report is a - MONTHLY, market-cap + industry neutral full-market series on Wind data while our - eval cell is CSI500 daily with industry + size neutral, so the report numbers are - a LOOSE reference only (disclosed, never mislabeled, never written in as an - expected value). is_intraday=False: minute INPUT but a DAILY signal traded - close-to-close. min_history_bars=0: the warm-up is DATA-dependent, not a fixed - leading count — the honest NaN rate is reported by data_coverage. - - The description spells out the two subtleties that make this factor harder than - its five siblings: the PREV_CLOSE-extended range read at OUR 14:50 visibility - rather than the report's true daily close, and the reversal neutralization taken - at T-1 so day d's 15:00 close is never an input to day d's value. - - D1 declarations (D0 pre-assignment table row 10 + note 2 — the - two-axes-differ exemplar): adjustment=returns_invariant — the rev20 - leg is a ratio of FRONT-ADJUSTED daily closes (anchor cancels; - data/clean/intraday_valley_quantile.py:49-55, :403-404) and the minute - legs are raw but fully same-day, so an ANCHOR perturbation does not - move the value. overnight_boundary=CROSSED_DISCLOSED — prev_close is a - raw close on the PREVIOUS day's basis entering day d's range, the - pinned definition keeps the ex-date values, and the deviation is - MEASURED and disclosed (0.73% true ex-dates / ~0.21% ex-date AND - range-distorted; intraday_valley_quantile.py:56-72 and pinned in - tests/test_valley_price_quantile_factor.py). All three - CROSSED_DISCLOSED requirements are met — unlike PR-D, nothing is owed. - ``requires`` adds market_daily close beyond the minute fields: the T-1 - rev20 neutralization reads the qfq DAILY close panel (a real second - endpoint, declared truthfully rather than mirrored off input_fields). - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Valley weighted-price-quantile (Kaiyuan microstructure series #27, " - f"SIXTH factor 量谷加权价格分位点). SAME minute classification as PR-F " - f"volume_peak_count / PR-H peak_interval_kurtosis / PR-I " - f"valley_relative_vwap / PR-J valley_ridge_vwap_ratio / PR-K " - f"ridge_minute_return (REUSED from data.clean.intraday_volume_prv, not " - f"re-implemented): 1min bars PIT-truncated at 14:50, a minute is ERUPTIVE " - f"if vol > μ + {VOLUME_PRV_SIGMA_K:g}σ of its SAME-SLOT strictly-prior " - f"{VOLUME_PRV_BASELINE_DAYS}-day baseline, and a VALLEY (量谷) is a " - f"classifiable NON-eruptive minute. NEW STATISTIC vs PR-F..PR-K: a price " - f"POSITION rather than a count, a timing moment, a price RATIO or a " - f"return — each valid day scores WHERE the valley VWAP sits inside the " - f"day's price range, and the factor averages that over the trailing " - f"{self._lookback_days} VALID days before a cross-sectional reversal " - f"neutralization. PINNED choices: (1) the range is [min(visible low, " - f"prev_close), max(visible high, prev_close)] where prev_close is the " - f"LAST VISIBLE (<=14:50) RAW CLOSE of the PREVIOUS trading day — a " - f"DISCLOSED DEVIATION from the report, which uses the true previous " - f"daily close; the previous 15:00 close is not itself a lookahead at a " - f"14:50 decision, so this is a SINGLE-VISIBILITY-DEFINITION choice (every " - f"price in the factor comes from one source under one cutoff), not a " - f"leakage fix; (2) the daily quantile (valley VWAP - lo)/(hi - lo) is NOT " - f"clipped — a VWAP outside the range means the range is wrong and must be " - f"visible rather than sanitized; (3) THE REVERSAL NEUTRALIZATION IS TAKEN " - f"AT T-1: rev20 = -(close_(d-1)/close_(d-{VALLEY_QUANTILE_REVERSAL_DAYS + 1}) " - f"- 1) on FRONT-ADJUSTED daily closes from the panel (NO new data source). " - f"The report's naive form uses close_d, which is 15:00 information and " - f"WOULD be a lookahead at our 14:50 decision — day d's close is therefore " - f"never an input to day d's factor value; front-adjusted closes are " - f"required because the ratio spans {VALLEY_QUANTILE_REVERSAL_DAYS} " - f"trading days; (4) RAW minute prices for the range and the VWAP, with a " - f"DISCLOSED imperfection: prev_close crosses the overnight boundary, so on " - f"an ex-dividend / split date the previous raw close sits on the OLD " - f"(higher) scale while the day's own bars sit on the new one. The bias is " - f"ONE-SIDED: prev_close pulls hi UP (measured 42 top-widening vs 4 " - f"bottom-widening among true ex-dates), inflating hi - lo while the valley " - f"VWAP and lo stay on the new scale, which compresses that day's quantile " - f"toward the LOW END of an artificially wide range — NOT toward the middle. " - f"Measured: 15.6% of valid days show any prev_close widening (overwhelmingly " - f"ordinary overnight gaps, which is INTENDED), only 0.73% are true ex-dates " - f"and only ~0.21% are both a true ex-date AND range-distorted, before the " - f"{self._lookback_days}-day mean dilutes it; disclosed rather than silently " - f"corrected; (5) a PRICE guard (finite, high >= low > 0) admits a bar to " - f"the range while a TRADE guard (finite positive volume AND amount) " - f"admits it to the VWAP, both applied at the summation step only so PR-F's " - f"baseline stays bit-identical; (6) a day is VALID iff it has >= " - f"{VOLUME_PRV_MIN_CLASSIFIABLE} classifiable bars AND >= " - f"{VALLEY_QUANTILE_MIN_VALLEY_BARS} TRADABLE valley bars AND positive " - f"valley volume AND an available prev_close AND hi > lo — so a symbol's " - f"FIRST visible day never produces a value; (7) the residualization is " - f"PER DATE on the covered cross-section, with < " - f"{VALLEY_QUANTILE_MIN_CROSS_SECTION} paired symbols, a missing rev20, or " - f"a degenerate (zero-variance) rev20 all yielding NaN rather than a " - f"zero-filled or unresidualized value passed off as neutralized; (8) " - f"DEVIATION FROM THE REPORT, disclosed: everything spans the PIT-VISIBLE " - f"window 09:31-14:50 only, not the full session. NaN below " - f"{VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived from 1min bars but a " - f"DAILY signal traded close-to-close." - ), - expected_ic_sign=1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar fields the upstream aggregation is derived from, plus the daily - # close the T-1 reversal neutralization regresses against. Declared for honest - # provenance disclosure (data_coverage lists them); the daily panel surfaces - # the pre-aggregated column itself. - input_fields=("volume", "amount", "high", "low", "close"), - requires=( - *_minute_requires("volume", "amount", "high", "low", "close"), - # The T-1 rev20 neutralization reads the front-adjusted DAILY - # close panel — a genuine market_daily requirement on top of - # the minute fields (see the docstring's D1 paragraph). - PanelField("close", source=MARKET_DAILY), - ), - adjustment="returns_invariant", - overnight_boundary="crossed_disclosed", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily valley-price-quantile column off ``panel``. - - The runner runs ``compute_valley_price_quantile_stats`` per symbol on the minute - cache and ``residualize_on_reversal`` once on the assembled panel upstream, then - joins the result as ``self.name``; here we only surface it, so this factor does no - temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"ValleyPriceQuantileFactor needs the pre-aggregated '{self.name}' " - f"column on the panel (produced upstream by " - f"compute_valley_price_quantile_stats + residualize_on_reversal and " - f"joined by the runner); panel has {list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - - -class PeakRidgeAmountRatioFactor(Factor): - """Peak/ridge traded-amount ratio factor (daily signal, minute-derived). - - ``compute`` reads the pre-aggregated daily column the runner placed on the panel - (produced by - ``data.clean.intraday_amount_ratio.compute_peak_ridge_amount_ratio``); it does NO - minute work of its own, mirroring :class:`JumpAmountCorrFactor` / - :class:`MinuteIdealAmplitudeFactor` / :class:`AmpMarginalAnomalyVolFactor` / - :class:`VolumePeakCountFactor` / :class:`IntradayAmpCutFactor` / - :class:`PeakIntervalKurtosisFactor` / :class:`ValleyRelativeVwapFactor` / - :class:`ValleyRidgeVwapRatioFactor` / :class:`RidgeMinuteReturnFactor` / - :class:`ValleyPriceQuantileFactor` and the value / financial factors that surface an - enriched column. - - Args: - lookback_days: trailing VALID trading-day window POOLED by both amount legs; part - of the factor DEFINITION (reproduced from the report), not a tuned knob. It - only names the column so a non-default window cannot silently mislabel it. - """ - - name: str = f"peak_ridge_amount_ratio_{PEAK_RIDGE_LOOKBACK_DAYS}" - - def __init__(self, lookback_days: int = PEAK_RIDGE_LOOKBACK_DAYS) -> None: - if not isinstance(lookback_days, int) or lookback_days < 1: - raise ValueError( - f"peak-ridge-amount-ratio lookback_days must be a positive integer; got " - f"{lookback_days!r}." - ) - self._lookback_days = lookback_days - self.name = f"peak_ridge_amount_ratio_{lookback_days}" - - @property - def lookback_days(self) -> int: - return self._lookback_days - - @property - def spec(self) -> FactorSpec: - """Evaluation contract; a property so ``factor_id`` tracks the window. - - expected_ic_sign=+1 READ FROM THE REPORT, not inferred from data and not merely - argued from semantics: §7.2 states "峰岭成交比因子 RankIC 均值 10.28%，RankICIR - 4.07" — an explicitly POSITIVE RankIC — with a long leg of 16.06%/yr, long-short - 27.13%/yr, IR 2.89, max drawdown 7.88%, 74.3% monthly win rate and every year - positive 2013–2025. The report's §1 taxonomy summary agrees independently ("对于 - 量峰时点，分钟数、成交额类因子更加有效，反映知情交易参与度，为正向因子", with the - ridge leg's amount factors marked negative-alpha), so peak-over-ridge is positive - on both counts. The sign is fixed BEFORE the run (a validated prototype must - reproduce it). NOTE the report is a MONTHLY, market-cap + industry neutral - full-market series on Wind data while our eval cell is CSI500 daily with industry - + size neutral, so the report numbers are a LOOSE reference only (disclosed, never - mislabeled, never written in as an expected value). is_intraday=False by the module - docstring's reasoning: minute INPUT but a DAILY signal traded close-to-close. - min_history_bars=0: the warm-up is DATA-dependent (a value appears once enough - VALID days accumulate), not a fixed leading count — the honest NaN rate is reported - by data_coverage. - - The description spells out the AGGREGATION FORM, which is the one place this - factor's definition departs from its siblings: the report specifies a RATIO OF - 20-DAY SUMS, not the mean of daily ratios that §7.1 (PR-J) specifies. - - D1 declarations (D0 pre-assignment table row 11): adjustment=none — - "``amount`` is traded VALUE in RMB, which no split or dividend - adjustment factor rescales ... there is nothing to cancel" - (data/clean/intraday_amount_ratio.py:53-57); volume only drives the - reused classification, and no price is ever read (zero price content — - one disclaimer covers both axes). overnight_boundary=none — no - raw-price comparison exists. - """ - return FactorSpec( - factor_id=self.name, - version="1.0", - description=( - f"Peak/ridge traded-amount ratio (Kaiyuan microstructure series #27, " - f"SEVENTH factor 峰岭成交比, §7.2). SAME minute classification as PR-F " - f"volume_peak_count / PR-H peak_interval_kurtosis / PR-I " - f"valley_relative_vwap / PR-J valley_ridge_vwap_ratio (REUSED from " - f"data.clean.intraday_volume_prv, not re-implemented): 1min bars " - f"PIT-truncated at 14:50, a minute is ERUPTIVE if vol > μ + " - f"{VOLUME_PRV_SIGMA_K:g}σ of its SAME-SLOT strictly-prior " - f"{VOLUME_PRV_BASELINE_DAYS}-day baseline; a PEAK is an ISOLATED eruption " - f"and a RIDGE is 'eruptive AND NOT a peak'. This is the first factor of " - f"the loop carrying NO price information: both legs are pure traded VALUE, " - f"so it separates 'only price information survives in this taxonomy' from " - f"'the peak/ridge split itself carries alpha'. AGGREGATION = RATIO OF " - f"SUMS: Σ peak amount over the trailing {self._lookback_days} VALID days " - f"divided by Σ ridge amount over the SAME days — the report's literal " - f"wording ('计算 20 日量峰总成交额与量岭总成交额，二者做比'), which " - f"DIFFERS from §7.1's mean-of-daily-ratios and is followed as written; it " - f"is also the better-behaved estimator, since a single day with a nearly " - f"vanishing ridge amount would dominate a mean of daily ratios. PINNED " - f"choices: (1) PEAK is the numerator, RIDGE the denominator, fixed both by " - f"the name 峰岭 and by the report's stated semantics (informed trading " - f"RELATIVE TO retail participation); (2) the masks are PR-J's exactly, so " - f"valley|peak|ridge stays an exact partition of the classifiable bars and " - f"a VALLEY bar enters NEITHER leg — this factor reads only the two " - f"ERUPTIVE groups; (3) bars with non-finite or non-positive amount are " - f"dropped from both legs and both bar counts (guard applied at summation " - f"only, so PR-F's baseline is untouched); volume is deliberately NOT in " - f"the guard because this factor never divides by volume; (4) RAW " - f"unadjusted amounts are exactly correct: traded VALUE in RMB is not " - f"rescaled by any split/dividend adjustment factor, so unlike PR-L there " - f"is no ex-date caveat and unlike PR-I/PR-J no within-day cancellation " - f"argument is even needed; (5) DEVIATION FROM THE REPORT, disclosed: both " - f"legs span the PIT-VISIBLE window 09:31-14:50 only, not the full session " - f"— reading the close would be lookahead at our 14:50 decision time; (6) a " - f"day is VALID iff it has >= {VOLUME_PRV_MIN_CLASSIFIABLE} classifiable " - f"bars AND >= {PEAK_RIDGE_MIN_PEAK_BARS} TRADABLE peak bars AND >= " - f"{PEAK_RIDGE_MIN_RIDGE_BARS} TRADABLE ridge bars (both counted AFTER the " - f"guard) AND positive amount in both legs — the PEAK floor is deliberately " - f"LOWER, the REVERSE of PR-J's asymmetry, because a peak must erupt AND be " - f"ISOLATED, making peaks the scarcer leg of THIS pair; the realized " - f"peak-bar distribution, the day-validity rate and the counterfactual " - f"valid-day count at a peak floor of 10 are all REPORTED by the runner " - f"rather than left implicit; NaN below {VOLUME_PRV_MIN_VALID_DAYS} valid " - f"days. Derived from 1min bars but a DAILY signal traded close-to-close." - ), - expected_ic_sign=1, - is_intraday=False, - forward_return_horizon=1, - return_basis="close_to_close", - # The 1min bar fields the upstream aggregation is derived from: volume drives - # the REUSED classification, amount is the factor's whole quantity. Declared - # for honest provenance disclosure (data_coverage lists them); the daily panel - # surfaces the pre-aggregated column itself. - input_fields=("volume", "amount"), - requires=_minute_requires("volume", "amount"), - adjustment="none", - overnight_boundary="none", - family="microstructure", - min_history_bars=0, - ) - - def compute(self, panel: pd.DataFrame) -> pd.Series: - """Select the pre-aggregated daily peak/ridge amount-ratio column off ``panel``. - - The runner runs ``compute_peak_ridge_amount_ratio`` per symbol on the minute cache - upstream and joins the result as ``self.name``; here we only surface it, so this - factor does no temporal logic and cannot introduce lookahead. - """ - if self.name not in panel.columns: - raise ValueError( - f"PeakRidgeAmountRatioFactor needs the pre-aggregated '{self.name}' " - f"column on the panel (produced upstream by " - f"compute_peak_ridge_amount_ratio and joined by the runner); panel has " - f"{list(panel.columns)}." - ) - return panel[self.name].rename(self.name) - +from factors.compute.minute.amp_marginal_anomaly_vol import AmpMarginalAnomalyVolFactor +from factors.compute.minute.intraday_amp_cut import IntradayAmpCutFactor +from factors.compute.minute.jump_amount_corr import JumpAmountCorrFactor +from factors.compute.minute.minute_ideal_amplitude import MinuteIdealAmplitudeFactor +from factors.compute.minute.peak_interval_kurtosis import PeakIntervalKurtosisFactor +from factors.compute.minute.peak_ridge_amount_ratio import PeakRidgeAmountRatioFactor +from factors.compute.minute.ridge_minute_return import RidgeMinuteReturnFactor +from factors.compute.minute.valley_price_quantile import ValleyPriceQuantileFactor +from factors.compute.minute.valley_relative_vwap import ValleyRelativeVwapFactor +from factors.compute.minute.valley_ridge_vwap_ratio import ValleyRidgeVwapRatioFactor +from factors.compute.minute.volume_peak_count import VolumePeakCountFactor __all__ = [ "AmpMarginalAnomalyVolFactor", diff --git a/factors/compute/minute/__init__.py b/factors/compute/minute/__init__.py new file mode 100644 index 0000000..3a35c9c --- /dev/null +++ b/factors/compute/minute/__init__.py @@ -0,0 +1,8 @@ +"""``factors.compute.minute``: the minute-factor engine (D2, design v3.2 §3.2). + +One factor per file, every file a small pure-function layer on top of +:mod:`factors.compute.minute.primitives` (the shared visible-bar preparation + +peak/ridge/valley taxonomy + trailing-window machinery). The ``data.clean`` +intraday factor modules are re-export shims of these files until D6d deletes +them (§六.4: the single definition point lives HERE from D2 on). +""" diff --git a/factors/compute/minute/amp_marginal_anomaly_vol.py b/factors/compute/minute/amp_marginal_anomaly_vol.py new file mode 100644 index 0000000..bbf3780 --- /dev/null +++ b/factors/compute/minute/amp_marginal_anomaly_vol.py @@ -0,0 +1,356 @@ +"""Amplitude marginal-anomaly relative-volatility factor (PR-E): math + surface (D2). + +Reproduces the Changjiang high-frequency-factor series #19 (长江证券《高频因子（十九）》, +2026-06-03, reportId 5462994) "振幅边际异常相对波动因子" as a daily PIT-safe column +derived from 5min bars (themselves DERIVED from the 1min cache). + +The source report is UNDER-SPECIFIED; the five choices below are deliberate, +DISCLOSED interpretations pinned in the task card (task_card_pr_e_*.md §0), not tuned +knobs. They are reproduced verbatim on the factor spec so a reader can see exactly +what was assumed: + + 1. bar frequency = 5min, DERIVED from the 1min cache via + :func:`data.clean.intraday_aggregate.resample_intraday_bars` (a derived bar + inherits ``available_time = max(source_1min.available_time)`` — PIT-faithful). + 2. lookback window N = 20 trading days (the symbol's own days that have bars, + trailing and INCLUDING ``d``). + 3. anomaly threshold = ``1{|Δamp_t| > μ + σ}`` with μ / σ = mean / ddof=1 std of + ALL pooled ``|Δamp|`` in the 20-day pool (k = 1). + 4. weighted volatility = the ddof=1 SAMPLE std of the RETURNS on the selected + (weight-1) bars. + 5. bar return ``r_t = close_t/close_{t-1} - 1`` and ``Δamp_t = amp_t - amp_{t-1}`` + are BOTH WITHIN-DAY lagged — each day's FIRST bar has no return / no Δamp — so + the overnight gap never contaminates a pair. + +Definition (per symbol, per panel date ``d``): + + 1. Take the symbol's most recent ``N`` (=20) trading days INCLUDING ``d`` of 5min + bars. PIT truncation (standing authorization): keep only bars with + ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50), + so every day is truncated to its own [session-open, 14:50]. + 2. Per bar: ``amp = high/low - 1`` (drop a bar unless ``low > 0`` and + ``high >= low``). WITHIN each (symbol, day), sorted by ``bar_end``, form + ``Δamp_t = amp_t - amp_{t-1}`` and ``r_t = close_t/close_{t-1} - 1``; the first + surviving bar of each day has neither (no cross-day lag). + 3. Pool ALL surviving ``(|Δamp|, r)`` pairs of the ``N``-day window into ONE set. + If the pool has fewer than ``min_pool`` (=460 ≈ 46 x 20 / 2) valid pairs, the + value is NaN (honest missing — coverage is disclosed by the runner). + 4. Pool ``μ = mean(|Δamp|)`` and ``σ = std(|Δamp|, ddof=1)``; select the bars with + ``|Δamp_t| > μ + k*σ`` (k = 1). If fewer than ``min_selected`` (=20) bars are + selected, the value is NaN (an anomaly std over too few bars is unreliable). + 5. factor(d, s) = ``std(r_t | selected, ddof=1)`` (column + ``amp_marginal_anomaly_vol_{N}``). + +Pre-registered sign = +1 (the report's IC is positive across its universes: +raw CSI800 +4.47% / full-market +4.92%; market-cap + industry neutral +4.11% / ++5.56%, ICIR 65.85% / 108.71%). NOTE the report's sample is CSI800 / full-market on +a MONTHLY series; our eval cell is CSI500 daily — a LOOSE reference only. The value +at ``(d, s)`` uses only bars at dates <= d, so a factor value never sees a future +bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_aggregate import resample_intraday_bars +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + empty_factor_series, + guarded_amplitude, + pooled_trailing_reduce, + visible_minute_frame, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). +AMP_ANOMALY_LOOKBACK_DAYS = 20 # trailing trading-day window (N), includes date d +AMP_ANOMALY_FREQ = "5min" # bar frequency, DERIVED from the 1min cache +AMP_ANOMALY_SIGMA_K = 1.0 # anomaly threshold multiplier k in |Δamp| > μ + k*σ +AMP_ANOMALY_MIN_POOL = 460 # minimum valid pooled (|Δamp|, r) pairs for a finite value +AMP_ANOMALY_MIN_SELECTED = 20 # minimum selected (anomaly) bars for a finite std + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def _anomaly_vol_cut( + dabs: np.ndarray, + ret: np.ndarray, + min_pool: int, + min_selected: int, + sigma_k: float, +) -> float: + """Selected-bar return std over one pooled window; NaN if a gate fails. + + ``dabs`` / ``ret`` are equal-length arrays of the VALID ``(|Δamp|, r)`` pairs in + ONE pooled window (already NaN-free). Fewer than ``min_pool`` pairs -> NaN; select + the bars whose ``|Δamp|`` exceeds ``mean + sigma_k * std(ddof=1)`` of the pooled + ``|Δamp|``; fewer than ``min_selected`` selected -> NaN (an unreliable std). + """ + n = dabs.size + if n < min_pool: + return float("nan") + mu = float(dabs.mean()) + sigma = float(dabs.std(ddof=1)) + threshold = mu + sigma_k * sigma + mask = dabs > threshold + n_sel = int(np.count_nonzero(mask)) + if n_sel < min_selected: + return float("nan") + return float(ret[mask].std(ddof=1)) + + +def _anomaly_vol_for_symbol( + g: pd.DataFrame, + lookback_days: int, + min_pool: int, + min_selected: int, + sigma_k: float, +) -> tuple[list[pd.Timestamp], list[float]]: + """Daily factor values for ONE symbol from its within-day-lagged bars. + + ``g`` holds columns ``trade_date`` / ``dabs`` (``|Δamp_t|``) / ``ret`` (``r_t``) + for a single symbol; ``dabs`` / ``ret`` are NaN on each day's first surviving bar + (no cross-day lag). Per day the VALID (finite) pairs are collected once, then each + date pools the trailing ``lookback_days`` days (including that date) via the shared + :func:`~factors.compute.minute.primitives.pooled_trailing_reduce` — no + cross-symbol leakage because ``g`` is a single symbol's slice, and no cross-day + leakage because each day's first-bar pair is already NaN and dropped here. + """ + days: list[pd.Timestamp] = [] + day_dabs: list[np.ndarray] = [] + day_ret: list[np.ndarray] = [] + for day, sub in g.groupby("trade_date", sort=True): + days.append(pd.Timestamp(day).normalize()) + d = sub["dabs"].to_numpy(dtype=float) + r = sub["ret"].to_numpy(dtype=float) + valid = np.isfinite(d) & np.isfinite(r) + day_dabs.append(d[valid]) + day_ret.append(r[valid]) + + values = pooled_trailing_reduce( + (day_dabs, day_ret), + lookback_days=lookback_days, + reducer=lambda dabs, ret: _anomaly_vol_cut( + dabs, ret, min_pool, min_selected, sigma_k + ), + ) + return days, values + + +def compute_amp_marginal_anomaly_vol( + bars: pd.DataFrame, + *, + lookback_days: int = AMP_ANOMALY_LOOKBACK_DAYS, + min_pool: int = AMP_ANOMALY_MIN_POOL, + min_selected: int = AMP_ANOMALY_MIN_SELECTED, + sigma_k: float = AMP_ANOMALY_SIGMA_K, + decision_time: str = DEFAULT_DECISION_TIME, + freq: str = AMP_ANOMALY_FREQ, + name: str = "amp_marginal_anomaly_vol", +) -> pd.Series: + """PIT-safe daily "amplitude marginal-anomaly relative-volatility" factor. + + Takes normalized 1min ``bars``, DERIVES ``freq`` (default 5min) bars from them via + :func:`resample_intraday_bars` (so a derived bar is usable only once EVERY + constituent 1min bar is available), PIT-truncates them at ``decision_time``, forms + the within-day ``|Δamp|`` / ``r`` pairs, and returns the trailing-``lookback_days`` + anomaly-bar return std. See the module docstring for the LOCKED definition. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping + is strictly per symbol (no cross-symbol leakage). + lookback_days: trailing trading-day window length (part of the definition). + min_pool: minimum valid pooled ``(|Δamp|, r)`` pairs for a finite value. + min_selected: minimum selected (anomaly) bars for a finite return std. + sigma_k: anomaly threshold multiplier ``k`` in ``|Δamp| > μ + k*σ``. + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + freq: derived bar frequency (default 5min); part of the definition. + name: the returned Series name (the factor-panel column name). + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily + factor value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if min_pool < 2: + # Need >= 2 pooled points so a ddof=1 std of |Δamp| is defined. + raise ValueError(f"min_pool must be >= 2; got {min_pool!r}.") + if min_selected < 2: + # Need >= 2 selected bars so a ddof=1 std of their returns is defined. + raise ValueError(f"min_selected must be >= 2; got {min_selected!r}.") + if sigma_k < 0.0: + raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") + if len(bars) == 0: + return empty_factor_series(name) + + # DERIVE the coarse (5min) bars from 1min FIRST: available_time = max source, so a + # coarse bar enters only once all its 1min constituents are available. + coarse = resample_intraday_bars(bars, freq) + if len(coarse) == 0: + return empty_factor_series(name) + + visible = visible_minute_frame( + coarse, columns=("high", "low", "close"), decision_time=decision_time + ) + if visible.empty: + return empty_factor_series(name) + + visible = guarded_amplitude(visible) + if visible.empty: + return empty_factor_series(name) + + # Sort so the within-day lag sees bars in chronological order within each + # (symbol, day); mergesort keeps it stable. + visible = visible.sort_values([SYMBOL_LEVEL, "bar_end"], kind="mergesort") + # Δamp and r are WITHIN-DAY lagged: grouping by (symbol, trade_date) makes each + # day's FIRST surviving bar NaN, so no pair ever crosses the overnight gap. + by_session = visible.groupby([SYMBOL_LEVEL, "trade_date"], sort=False) + visible["dabs"] = by_session["amp"].diff().abs() + visible["ret"] = by_session["close"].pct_change() + + index_tuples: list[tuple] = [] + values: list[float] = [] + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + days, vals = _anomaly_vol_for_symbol( + g, lookback_days, min_pool, min_selected, sigma_k + ) + for day, val in zip(days, vals): + index_tuples.append((day, str(sym))) + values.append(val) + + if not index_tuples: + return empty_factor_series(name) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +class AmpMarginalAnomalyVolFactor(Factor): + """Amplitude marginal-anomaly relative-volatility factor (daily, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_amp_marginal_anomaly_vol`); it does NO minute work of + its own, mirroring its siblings and the value / financial factors that surface an + enriched column. + + Args: + lookback_days: trailing trading-day window; part of the factor DEFINITION + (a pinned interpretation of the report), not a tuned knob. It only names + the column so a non-default window cannot silently mislabel it. + """ + + name: str = f"amp_marginal_anomaly_vol_{AMP_ANOMALY_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = AMP_ANOMALY_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"amp-marginal-anomaly-vol lookback_days must be a positive integer; " + f"got {lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"amp_marginal_anomaly_vol_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=+1: the report's IC is POSITIVE across its universes (raw + CSI800 +4.47% / full-market +4.92%; market-cap + industry neutral +4.11% / + +5.56%) — a HIGH anomaly-bar relative volatility predicts HIGHER forward + returns. The sign is fixed BEFORE the run (a validated prototype must + reproduce it). NOTE the report's sample is CSI800 / full-market on a MONTHLY + series while our eval cell is CSI500 daily, so the report numbers are a LOOSE + reference only (disclosed, never mislabeled). is_intraday=False by the module + docstring's reasoning: minute INPUT but a DAILY signal traded close-to-close. + min_history_bars=0: the warm-up is DATA-dependent (a value appears once + >= ``AMP_ANOMALY_MIN_POOL`` valid pooled pairs accumulate in the trailing + window), not a fixed leading count — the honest NaN rate is reported by + data_coverage. + + The description spells out the FIVE pinned interpretations of the + under-specified report (bar freq, lookback, threshold, weighted-vol operator, + within-day lag) so a reader sees exactly what was assumed. + + D1 declarations (D0 pre-assignment table row 3): adjustment= + returns_invariant — amp is the same-day ratio high/low - 1 and the + selected-bar statistic is a std of minute RETURNS (ratios), so the + anchor cancels throughout. overnight_boundary=none — Δamp and the + bar-return are BOTH within-day lagged, "the overnight gap never + contaminates a pair" (module docstring pinned choice 5). + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Amplitude marginal-anomaly relative volatility (Changjiang HF-factor " + f"series #19). PINNED interpretations of an under-specified report: " + f"(1) {AMP_ANOMALY_FREQ} bars DERIVED from the 1min cache " + f"(available_time = max source, PIT-faithful); (2) trailing " + f"{self._lookback_days} trading days (PIT-truncated at 14:50 per bar); " + f"(3) select bars with |Δamp| > μ + {AMP_ANOMALY_SIGMA_K:g}σ of the " + f"pooled |Δamp|; (4) factor = ddof=1 std of the RETURNS on the selected " + f"bars; (5) Δamp and bar-return are WITHIN-DAY lagged (each day's first " + f"bar has neither). amp = high/low - 1. Derived from 1min bars but a " + f"DAILY signal traded close-to-close; >= {AMP_ANOMALY_MIN_POOL} valid " + f"pooled pairs and >= {AMP_ANOMALY_MIN_SELECTED} selected bars required " + f"else NaN." + ), + expected_ic_sign=1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from. Declared for + # honest provenance disclosure (data_coverage lists them); the daily panel + # surfaces the pre-aggregated column itself. + input_fields=("high", "low", "close"), + requires=_minute_requires("high", "low", "close"), + adjustment="returns_invariant", + overnight_boundary="none", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily amp-marginal-anomaly-vol column off ``panel``. + + The runner runs ``compute_amp_marginal_anomaly_vol`` on the minute cache + upstream and joins the result as ``self.name``; here we only surface it, so this + factor does no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"AmpMarginalAnomalyVolFactor needs the pre-aggregated '{self.name}' " + f"column on the panel (produced upstream by " + f"compute_amp_marginal_anomaly_vol and joined by the runner); panel has " + f"{list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "AMP_ANOMALY_FREQ", + "AMP_ANOMALY_LOOKBACK_DAYS", + "AMP_ANOMALY_MIN_POOL", + "AMP_ANOMALY_MIN_SELECTED", + "AMP_ANOMALY_SIGMA_K", + "AmpMarginalAnomalyVolFactor", + "compute_amp_marginal_anomaly_vol", +] diff --git a/factors/compute/minute/intraday_amp_cut.py b/factors/compute/minute/intraday_amp_cut.py new file mode 100644 index 0000000..ba10de0 --- /dev/null +++ b/factors/compute/minute/intraday_amp_cut.py @@ -0,0 +1,484 @@ +"""Intraday amplitude-cut factor (PR-G): math + surface (D2). + +Reproduces the SECOND factor of the Kaiyuan market-microstructure series #30 (开源证券 +《高频振幅因子的内部切割——市场微观结构系列（30）》, 2025-08-09, reportId 4988549) — the +"日内振幅切割因子" (§3, Table 3, thought (2)) — as a daily PIT-safe column derived from +the 1min cache. + +DISTINCTION FROM PR-D (must be stated on the factor spec): PR-D (``minute_ideal_amp``, +the report's FLAGSHIP, thought (1)) POOLS the trailing 10 days' 1min bars into ONE set +and cuts by minute CLOSE PRICE. THIS factor (thought (2)) cuts EACH DAY independently by +the 1-MINUTE RETURN, produces a daily ``V_day`` series, then takes its trailing-10-valid- +day mean / std and combines them cross-sectionally. The report finds the two are only +~30% correlated — they are mathematically distinct constructions, not variants. + +Definition (report Table 3 four steps + §3 terminal parameters). For each symbol and each +panel date ``d`` (a DAILY signal, close-to-close, ``is_intraday=False``): + + 1. PIT truncation (standing authorization): each day keeps only the 1min bars with + ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50), so + every day (history AND signal day) is truncated to its own [session-open, 14:50]. + 2. PER-DAY CUT (each trading day in the window, independent): + - per-bar amplitude ``amp = high/low - 1`` (drop a bar unless ``low > 0`` and + ``high >= low``); + - the sentiment indicator is the 1-MINUTE RETURN ``r_t = close_t/close_{t-1} - 1`` + (the report's terminal pick), WITHIN-DAY lagged: each day's FIRST surviving bar + has no ``r`` and is NOT in that day's cut (no overnight gap; PR-E precedent); + - a bar is VALID iff both ``amp`` and ``r`` are finite; ``n_day`` = the valid-bar + count; ``n_day < min_day_minutes`` (=100) -> the day is INVALID (no ``V_day``); + - ``k = floor(lam * n_day)`` (lam=0.20, report terminal), ``k >= 1``; a stable + total order ``(r, bar_end)`` makes the selection deterministic; + - ``V_high`` = mean ``amp`` of the ``k`` HIGHEST-``r`` bars, ``V_low`` = mean + ``amp`` of the ``k`` LOWEST-``r`` bars; the day's cut value is + ``V_day = V_high - V_low``. + 3. TIME-SERIES aggregation: take the most recent ``lookback_days`` (=10) VALID trading + days INCLUDING ``d``; fewer than ``min_valid_days`` (=6) valid days -> NaN (honest + missing). ``V_mean = mean(V_day)`` and ``V_std = std(V_day, ddof=1)``. + 4. CROSS-SECTIONAL standardization (report step 4): for each panel date, z-score + ``V_mean`` and ``V_std`` SEPARATELY over the covered-universe cross-section (ddof=1 + std denominator; a date whose finite-pair cross-section is smaller than + ``min_cross_section`` (=10) -> all NaN that date), then the factor value is + ``(z(V_mean) + z(V_std)) / 2`` (column ``intraday_amp_cut_{N}``). + +Pre-registered sign = -1 (report: rankIC mean -0.067, rankICIR -3.82, quintile +long-short 16.7%/yr at N=10, lambda=20%, 1-minute-return indicator; the V_mean and V_std +sub-factors are each negative too). Low-volatility family reading: when high-return +minutes have a MUCH larger amplitude than low-return minutes, future returns are lower. + +PINNED interpretation (disclosed, not a tuned knob): the report cross-standardizes on the +FULL market; our eval cell cross-standardizes on the CSI500 covered set. The factor value +at ``(d, s)`` uses only bars at dates <= d, so a value never sees a future bar +(invariant #1). + +The heavy per-symbol work (steps 1-3, producing the ``(V_mean, V_std)`` panel) is split +from the cross-sectional combine (step 4) on purpose: the runner streams one symbol at a +time through :func:`compute_amp_cut_stats` (memory-bounded), assembles the full-universe +two-column panel, then calls :func:`combine_amp_cut_cross_section` ONCE — because step 4 +needs every symbol's ``(V_mean, V_std)`` present before it can z-score a date's +cross-section. :func:`compute_intraday_amp_cut` chains the two for a single-call path. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DATE_LEVEL, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + add_bar_end_ns, + empty_factor_series, + guarded_amplitude, + visible_minute_frame, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (report terminal parameters; NOT tuned knobs). +AMP_CUT_LOOKBACK_DAYS = 10 # trailing VALID trading-day window (N), includes date d +AMP_CUT_LAMBDA = 0.20 # top/bottom fraction by 1-min return that forms V_high/V_low +AMP_CUT_MIN_DAY_MINUTES = 100 # a day is valid iff it has >= this many valid (amp & r) bars +AMP_CUT_MIN_VALID_DAYS = 6 # min valid days in the trailing window for a finite V_mean/V_std +AMP_CUT_MIN_CROSS_SECTION = 10 # min finite-pair cross-section for a finite z-score on a date + +# Internal two-column stats panel labels (V_mean / V_std of the daily V_day series). +V_MEAN_COL = "v_mean" +V_STD_COL = "v_std" + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def _empty_stats() -> pd.DataFrame: + """Schema-shaped empty ``MultiIndex(date, symbol)`` ``(v_mean, v_std)`` panel.""" + index = pd.MultiIndex.from_arrays( + [pd.DatetimeIndex([]), pd.Index([], dtype=object)], + names=DAILY_INDEX_NAMES, + ) + return pd.DataFrame( + {V_MEAN_COL: pd.Series([], dtype=float), V_STD_COL: pd.Series([], dtype=float)}, + index=index, + ) + + +def _day_cut( + amps: np.ndarray, rets: np.ndarray, bar_ends: np.ndarray, lam: float, min_day_minutes: int +) -> float: + """``V_high - V_low`` for ONE day's valid bars; NaN if the day is invalid or k < 1. + + ``amps`` / ``rets`` / ``bar_ends`` are equal-length arrays of the VALID (amp & r + finite) bars of ONE trading day. The bars are ranked by the 1-minute return ``r`` + with ``(r, bar_end)`` as a stable total order (lexsort with ``r`` primary), so the + top-k / bottom-k selection is deterministic even when two bars share a return. + """ + n = amps.size + if n < min_day_minutes: + return float("nan") + k = int(np.floor(lam * n)) + if k < 1: + return float("nan") + order = np.lexsort((bar_ends, rets)) # r primary, bar_end tie-break + a = amps[order] + return float(a[-k:].mean() - a[:k].mean()) # V_high (top-r) - V_low (bottom-r) + + +def _amp_cut_stats_for_symbol( + g: pd.DataFrame, + lookback_days: int, + lam: float, + min_day_minutes: int, + min_valid_days: int, +) -> tuple[list[pd.Timestamp], list[float], list[float]]: + """Trailing V_mean / V_std for ONE symbol from its within-day-lagged bars. + + ``g`` holds columns ``trade_date`` / ``amp`` / ``ret`` (the within-day 1-minute + return, NaN on each day's first surviving bar) / ``bar_end_ns`` for a single symbol. + Per day the VALID bars are cut into ``V_day``; only VALID days (``V_day`` finite) + enter the trailing series, so the rolling window spans the most recent + ``lookback_days`` VALID days (report "取最近 10 个有效交易日"). Values are emitted only + on valid days; a window with fewer than ``min_valid_days`` valid days -> NaN (both + stats). No cross-symbol leakage — ``g`` is one symbol's slice — and no cross-day + leakage — each day's first-bar return is already NaN and excluded from that day's cut. + """ + days: list[pd.Timestamp] = [] + vdays: list[float] = [] + for day, sub in g.groupby("trade_date", sort=True): + amp = sub["amp"].to_numpy(dtype=float) + ret = sub["ret"].to_numpy(dtype=float) + be = sub["bar_end_ns"].to_numpy(dtype="int64") + valid = np.isfinite(amp) & np.isfinite(ret) + vdays.append(_day_cut(amp[valid], ret[valid], be[valid], lam, min_day_minutes)) + days.append(pd.Timestamp(day).normalize()) + + series = pd.Series(vdays, index=pd.DatetimeIndex(days)).sort_index() + valid_series = series.dropna() # only VALID days carry a V_day + if valid_series.empty: + return [], [], [] + roll = valid_series.rolling(lookback_days, min_periods=min_valid_days) + vmean = roll.mean() + vstd = roll.std() # ddof=1 (pandas rolling default) + out_days = [pd.Timestamp(d).normalize() for d in vmean.index] + return out_days, list(vmean.to_numpy(dtype=float)), list(vstd.to_numpy(dtype=float)) + + +def compute_amp_cut_stats( + bars: pd.DataFrame, + *, + lookback_days: int = AMP_CUT_LOOKBACK_DAYS, + lam: float = AMP_CUT_LAMBDA, + min_day_minutes: int = AMP_CUT_MIN_DAY_MINUTES, + min_valid_days: int = AMP_CUT_MIN_VALID_DAYS, + decision_time: str = DEFAULT_DECISION_TIME, +) -> pd.DataFrame: + """Per-symbol trailing ``(V_mean, V_std)`` panel (steps 1-3; NO cross-section yet). + + Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, forms + the within-day 1-minute returns, cuts each valid day by return into ``V_day``, and + returns the trailing-``lookback_days``-VALID-day mean / std of ``V_day``. The + cross-sectional standardization (step 4) is deliberately NOT applied here — it needs + the full-universe panel and is done by :func:`combine_amp_cut_cross_section`. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the per-day cut + and trailing aggregation are strictly per symbol (no cross-symbol leakage). + lookback_days: trailing VALID trading-day window (definition). + lam: top/bottom 1-minute-return fraction defining V_high/V_low (0 < lam <= 0.5). + min_day_minutes: a day is valid iff it has >= this many valid (amp & r) bars. + min_valid_days: minimum valid days in the trailing window for a finite value. + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + + Returns: + ``MultiIndex(date, symbol)`` DataFrame with columns ``v_mean`` / ``v_std`` + (midnight-normalized dates), sorted. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if not (0.0 < lam <= 0.5): + raise ValueError(f"lam must be in (0, 0.5]; got {lam!r}.") + if min_day_minutes < 2: + # Need at least 2 valid minutes so a non-empty top/bottom cut can exist. + raise ValueError(f"min_day_minutes must be >= 2; got {min_day_minutes!r}.") + if min_valid_days < 2: + # Need >= 2 valid days so a ddof=1 std of the V_day series is defined. + raise ValueError(f"min_valid_days must be >= 2; got {min_valid_days!r}.") + if len(bars) == 0: + return _empty_stats() + + visible = visible_minute_frame( + bars, columns=("high", "low", "close"), decision_time=decision_time + ) + if visible.empty: + return _empty_stats() + + visible = guarded_amplitude(visible) + if visible.empty: + return _empty_stats() + + # int64 nanoseconds for a deterministic lexsort tie-break. + visible = add_bar_end_ns(visible) + # Sort so the within-day lag sees bars in chronological order within each + # (symbol, day); mergesort keeps it stable. + visible = visible.sort_values([SYMBOL_LEVEL, "bar_end"], kind="mergesort") + # r is WITHIN-DAY lagged: grouping by (symbol, trade_date) makes each day's FIRST + # surviving bar NaN, so no return ever crosses the overnight gap. + visible["ret"] = visible.groupby([SYMBOL_LEVEL, "trade_date"], sort=False)[ + "close" + ].pct_change() + + index_tuples: list[tuple] = [] + vmean_vals: list[float] = [] + vstd_vals: list[float] = [] + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + days, vmean, vstd = _amp_cut_stats_for_symbol( + g, lookback_days, lam, min_day_minutes, min_valid_days + ) + for day, m, s in zip(days, vmean, vstd): + index_tuples.append((day, str(sym))) + vmean_vals.append(m) + vstd_vals.append(s) + + if not index_tuples: + return _empty_stats() + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.DataFrame( + {V_MEAN_COL: vmean_vals, V_STD_COL: vstd_vals}, index=index + ).sort_index() + + +def combine_amp_cut_cross_section( + stats: pd.DataFrame, + *, + min_cross_section: int = AMP_CUT_MIN_CROSS_SECTION, + name: str = "intraday_amp_cut", +) -> pd.Series: + """Cross-sectional z-score combine of the ``(V_mean, V_std)`` panel (step 4). + + For each panel date, the cross-section is the symbols with BOTH ``V_mean`` and + ``V_std`` finite. If that count is below ``min_cross_section`` (=10), every symbol on + that date is NaN (honest missing). Otherwise ``V_mean`` and ``V_std`` are each + z-scored (ddof=1 denominator) across the cross-section and the factor value is + ``(z(V_mean) + z(V_std)) / 2``. A degenerate date (a column's cross-sectional std is + zero or non-finite) is likewise NaN. The pre-registered sign (-1) lives on the factor + spec, not here — this returns the RAW combined score. + + Args: + stats: ``MultiIndex(date, symbol)`` panel with ``v_mean`` / ``v_std`` columns, + typically the full covered universe assembled by the runner. + min_cross_section: minimum finite-pair cross-section for a finite z-score. + name: the returned Series name (the factor-panel column name). + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the combined + factor value, sorted, named ``name``. Pure: never mutates ``stats``. + """ + if min_cross_section < 2: + # Need >= 2 cross-section members so a ddof=1 std across the section is defined. + raise ValueError(f"min_cross_section must be >= 2; got {min_cross_section!r}.") + if stats is None or stats.empty: + return empty_factor_series(name) + if V_MEAN_COL not in stats.columns or V_STD_COL not in stats.columns: + raise ValueError( + f"stats must carry '{V_MEAN_COL}' and '{V_STD_COL}' columns; got " + f"{list(stats.columns)}." + ) + + vm_all = stats[V_MEAN_COL].to_numpy(dtype=float) + vs_all = stats[V_STD_COL].to_numpy(dtype=float) + finite_pair = np.isfinite(vm_all) & np.isfinite(vs_all) + valid = stats.loc[finite_pair, [V_MEAN_COL, V_STD_COL]] + if valid.empty: + return empty_factor_series(name) + + index_tuples: list[tuple] = [] + values: list[float] = [] + for date, grp in valid.groupby(level=DATE_LEVEL, sort=True): + syms = grp.index.get_level_values(SYMBOL_LEVEL) + n = len(grp) + if n < min_cross_section: + vals = np.full(n, np.nan) + else: + vm = grp[V_MEAN_COL].to_numpy(dtype=float) + vs = grp[V_STD_COL].to_numpy(dtype=float) + sm = float(vm.std(ddof=1)) + ss = float(vs.std(ddof=1)) + if not (np.isfinite(sm) and np.isfinite(ss)) or sm == 0.0 or ss == 0.0: + vals = np.full(n, np.nan) + else: + zm = (vm - vm.mean()) / sm + zs = (vs - vs.mean()) / ss + vals = (zm + zs) / 2.0 + day = pd.Timestamp(date).normalize() + for sym, v in zip(syms, vals): + index_tuples.append((day, str(sym))) + values.append(float(v)) + + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +def compute_intraday_amp_cut( + bars: pd.DataFrame, + *, + lookback_days: int = AMP_CUT_LOOKBACK_DAYS, + lam: float = AMP_CUT_LAMBDA, + min_day_minutes: int = AMP_CUT_MIN_DAY_MINUTES, + min_valid_days: int = AMP_CUT_MIN_VALID_DAYS, + min_cross_section: int = AMP_CUT_MIN_CROSS_SECTION, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "intraday_amp_cut", +) -> pd.Series: + """Single-call intraday amplitude-cut factor: stats (steps 1-3) then combine (step 4). + + Convenience that chains :func:`compute_amp_cut_stats` and + :func:`combine_amp_cut_cross_section`. The runner does NOT use this (it needs the + per-symbol stats loop for memory-boundedness) but tests and any all-in-memory caller + can. The cross-section is whatever ``bars`` carry — see the module docstring's PINNED + interpretation (the runner scopes it to the CSI500 covered set). + + See the module docstring for the LOCKED definition. Pure: never mutates ``bars``. + """ + stats = compute_amp_cut_stats( + bars, + lookback_days=lookback_days, + lam=lam, + min_day_minutes=min_day_minutes, + min_valid_days=min_valid_days, + decision_time=decision_time, + ) + return combine_amp_cut_cross_section( + stats, min_cross_section=min_cross_section, name=name + ) + + +class IntradayAmpCutFactor(Factor): + """Intraday amplitude-cut factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_amp_cut_stats` + :func:`combine_amp_cut_cross_section`); + it does NO minute work of its own, mirroring its siblings and the value / financial + factors that surface an enriched column. + + Args: + lookback_days: trailing VALID trading-day window; part of the factor DEFINITION + (reproduced from the report), not a tuned knob. It only names the column so a + non-default window cannot silently mislabel it. + """ + + name: str = f"intraday_amp_cut_{AMP_CUT_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = AMP_CUT_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"intraday-amp-cut lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"intraday_amp_cut_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=-1: the report's rankIC mean is -0.067 (rankICIR -3.82, quintile + long-short 16.7%/yr at N=10, lambda=20%, 1-minute-return indicator) — a HIGH + intraday amplitude-cut value predicts LOWER forward returns; the V_mean and V_std + sub-factors are each negative too. The sign is fixed BEFORE the run (a validated + prototype must reproduce it). is_intraday=False by the module docstring's + reasoning: minute INPUT but a DAILY signal traded close-to-close. min_history_bars= + 0: the warm-up is DATA-dependent (a value appears once >= ``AMP_CUT_MIN_VALID_DAYS`` + valid days accumulate AND the cross-section has >= ``AMP_CUT_MIN_CROSS_SECTION`` + finite pairs), not a fixed leading count — the honest NaN rate is reported by + data_coverage. + + The description spells out the DISTINCTION FROM PR-D (``minute_ideal_amp``): PR-D + pools the 10-day minutes into ONE set and cuts by minute CLOSE PRICE; this factor + cuts EACH DAY by the 1-MINUTE RETURN, then takes the trailing-10-valid-day mean / + std of the daily cut and combines them cross-sectionally (the report finds the two + only ~30% correlated). + + D1 declarations (D0 pre-assignment table row 5): adjustment= + returns_invariant — amp is the same-day ratio high/low - 1 and the + cut key r = close_t/close_{t-1} - 1 is a within-day ratio, so the + anchor cancels in both. overnight_boundary=none — r is WITHIN-DAY + lagged, "no overnight gap; PR-E precedent" (module docstring step 2). + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Intraday amplitude cut (Kaiyuan microstructure series #30, SECOND " + f"factor 日内振幅切割). DISTINCT FROM PR-D minute_ideal_amp, which pools the " + f"trailing days' minutes into ONE set and cuts by minute CLOSE PRICE: " + f"this factor cuts EACH DAY independently by the 1-MINUTE RETURN " + f"r=close_t/close_{{t-1}}-1 (within-day lagged, first bar of each day has " + f"no r), taking V_day = V_high - V_low where V_high/V_low are the mean " + f"amp (high/low-1) of the top/bottom floor({AMP_CUT_LAMBDA:g}*n_day) bars " + f"by return (day valid iff >= {AMP_CUT_MIN_DAY_MINUTES} valid bars). " + f"Trailing {self._lookback_days} VALID days give V_mean / V_std (>= " + f"{AMP_CUT_MIN_VALID_DAYS} valid days else NaN); per date they are each " + f"cross-sectionally z-scored over the covered universe (>= " + f"{AMP_CUT_MIN_CROSS_SECTION} finite pairs else NaN) and averaged: factor " + f"= (z(V_mean) + z(V_std))/2. Derived from 1min bars but a DAILY signal " + f"traded close-to-close (report finds ~30% corr with minute_ideal_amp)." + ), + expected_ic_sign=-1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from. Declared for + # honest provenance disclosure (data_coverage lists them); the daily panel + # surfaces the pre-aggregated column itself. + input_fields=("high", "low", "close"), + requires=_minute_requires("high", "low", "close"), + adjustment="returns_invariant", + overnight_boundary="none", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily intraday-amp-cut column off ``panel``. + + The runner runs ``compute_amp_cut_stats`` per symbol on the minute cache upstream, + assembles the full-universe ``(V_mean, V_std)`` panel, applies + ``combine_amp_cut_cross_section``, and joins the result as ``self.name``; here we + only surface it, so this factor does no temporal logic and cannot introduce + lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"IntradayAmpCutFactor needs the pre-aggregated '{self.name}' column on " + f"the panel (produced upstream by compute_amp_cut_stats + " + f"combine_amp_cut_cross_section and joined by the runner); panel has " + f"{list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "AMP_CUT_LAMBDA", + "AMP_CUT_LOOKBACK_DAYS", + "AMP_CUT_MIN_CROSS_SECTION", + "AMP_CUT_MIN_DAY_MINUTES", + "AMP_CUT_MIN_VALID_DAYS", + "IntradayAmpCutFactor", + "V_MEAN_COL", + "V_STD_COL", + "combine_amp_cut_cross_section", + "compute_amp_cut_stats", + "compute_intraday_amp_cut", +] diff --git a/factors/compute/minute/jump_amount_corr.py b/factors/compute/minute/jump_amount_corr.py new file mode 100644 index 0000000..8109d24 --- /dev/null +++ b/factors/compute/minute/jump_amount_corr.py @@ -0,0 +1,287 @@ +"""Price-jump turnover-correlation factor (PR-C): math + surface (D2). + +Reproduces the Kaiyuan report §6 factor (``价格跳跃成交额相关性``, full-A RankIC +-10.23%): the trailing-``lookback_days``-trading-day lagged Pearson correlation +between the traded ``amount`` at price-JUMP minutes and the amount at the +STRICTLY-next minute. The daily value at ``(date d, symbol s)`` uses ONLY bars +at dates ``<= d`` (the trailing window ending at d), so a factor value never +sees a future bar (invariant #1); it is meant to trade close-to-close from d+1. + +Definition (LOCKED, per bar of one (symbol, day) session): + * ``amplitude = (high - low) / open`` (guard open > 0, amount finite); + * ``jump`` = within-(symbol, day) amplitude z-score (ddof=1) ``> jump_z``; + * pair each jump minute ``t`` with the STRICTLY-next minute (same session, + ``bar_end`` gap exactly 60s — this excludes the lunch break AND the close); + * ``factor(s, d)`` = Pearson corr(amount[jump t], amount[t+1]) over ALL + jump-pairs whose date is in the trailing ``lookback_days`` TRADING DAYS + (the symbol's own minute-trading days) ending at d; NaN when fewer than + ``min_pairs`` pairs fall in the window (or the correlation is undefined). + +Vectorized (no per-rebalance-date python loop): the trailing-window correlation +is a rolling sum of per-day sufficient statistics (n, sum x, sum y, sum x^2, +sum y^2, sum xy) over the trading-day axis, then Pearson's closed form. Rolling +over ROWS of the per-day-sorted stats == a trailing window of trading days, +because consecutive rows are consecutive trading days. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_schema import ( + DATE_LEVEL, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + ONE_MINUTE_SECONDS, + empty_factor_series, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (reproduced from the report; NOT tuned knobs). The +# daily value is a trailing-``JUMP_LOOKBACK_DAYS``-trading-day lagged correlation +# between the traded ``amount`` at price-JUMP minutes and the amount at the +# STRICTLY-next minute; a jump minute is one whose within-(symbol, day) amplitude +# z-score exceeds ``JUMP_Z``. Requires at least ``JUMP_MIN_PAIRS`` jump-pairs. +JUMP_LOOKBACK_DAYS = 20 +JUMP_MIN_PAIRS = 10 +JUMP_Z = 1.0 + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def compute_jump_amount_corr( + bars: pd.DataFrame, + *, + lookback_days: int = JUMP_LOOKBACK_DAYS, + min_pairs: int = JUMP_MIN_PAIRS, + jump_z: float = JUMP_Z, + name: str = "jump_amount_corr", +) -> pd.Series: + """PIT-safe daily "price-jump turnover correlation" factor from 1min ``bars``. + + See the module docstring for the LOCKED definition. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the + grouping is strictly per symbol (no cross-symbol leakage). + lookback_days: trailing trading-day window length (part of the definition). + min_pairs: minimum jump-pairs in the window for a finite value. + jump_z: within-day amplitude z-score threshold defining a jump minute. + name: the returned Series name (the factor-panel column name). + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily + factor value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if min_pairs < 2: + # Pearson correlation needs at least 2 points; below that it is undefined. + raise ValueError(f"min_pairs must be >= 2; got {min_pairs!r}.") + if len(bars) == 0: + return empty_factor_series(name) + + work = bars.reset_index()[ + [SYMBOL_LEVEL, "bar_end", "open", "high", "low", "amount"] + ].copy() + # Guard bad rows BEFORE anything else: a non-positive open makes the amplitude + # meaningless and a non-finite amount would poison the correlation. + work = work[(work["open"] > 0.0) & np.isfinite(work["amount"].to_numpy(dtype=float))] + if work.empty: + return empty_factor_series(name) + work[DATE_LEVEL] = work["bar_end"].dt.normalize() + # Sort so the "strictly next minute" shift and the per-symbol trading-day + # rolling both see bars in chronological order within each (symbol, day). + work = work.sort_values([SYMBOL_LEVEL, "bar_end"], kind="mergesort") + + work["amp"] = (work["high"] - work["low"]) / work["open"] + by_session = work.groupby([SYMBOL_LEVEL, DATE_LEVEL], sort=False) + mean_amp = by_session["amp"].transform("mean") + std_amp = by_session["amp"].transform("std") # ddof=1 (pandas default) + zscore = (work["amp"] - mean_amp) / std_amp + next_bar_end = by_session["bar_end"].shift(-1) + amt_next = by_session["amount"].shift(-1) + gap = (next_bar_end - work["bar_end"]).dt.total_seconds() + is_jump = (zscore > jump_z) & (gap == ONE_MINUTE_SECONDS) + + pairs = pd.DataFrame( + { + SYMBOL_LEVEL: work[SYMBOL_LEVEL].to_numpy(), + DATE_LEVEL: work[DATE_LEVEL].to_numpy(), + "x": work["amount"].to_numpy(dtype=float), + "y": amt_next.to_numpy(dtype=float), + } + ).loc[is_jump.to_numpy()] + + # Per-(symbol, day) sufficient statistics of the jump-pairs. + if pairs.empty: + stats = pd.DataFrame( + columns=["cnt", "sx", "sy", "sxx", "syy", "sxy"], dtype=float + ) + stats.index = pd.MultiIndex.from_arrays( + [pd.DatetimeIndex([]), pd.Index([], dtype=object)], + names=[SYMBOL_LEVEL, DATE_LEVEL], + ) + else: + pairs = pairs.assign( + xx=pairs["x"] * pairs["x"], + yy=pairs["y"] * pairs["y"], + xy=pairs["x"] * pairs["y"], + ) + stats = pairs.groupby([SYMBOL_LEVEL, DATE_LEVEL], sort=True).agg( + cnt=("x", "size"), + sx=("x", "sum"), + sy=("y", "sum"), + sxx=("xx", "sum"), + syy=("yy", "sum"), + sxy=("xy", "sum"), + ) + + # Trading-day axis = every (symbol, day) the symbol has minute bars on, so the + # rolling window counts TRADING DAYS (days with no jump still occupy a row, as + # zeros — they consume one of the trailing ``lookback_days`` slots). + axis = ( + work[[SYMBOL_LEVEL, DATE_LEVEL]] + .drop_duplicates() + .sort_values([SYMBOL_LEVEL, DATE_LEVEL], kind="mergesort") + ) + full_index = pd.MultiIndex.from_arrays( + [axis[SYMBOL_LEVEL].to_numpy(), axis[DATE_LEVEL].to_numpy()], + names=[SYMBOL_LEVEL, DATE_LEVEL], + ) + dense = stats.reindex(full_index).fillna(0.0) + rolled = ( + dense.groupby(level=SYMBOL_LEVEL, sort=False) + .rolling(lookback_days, min_periods=1) + .sum() + ) + # groupby.rolling prepends the group key -> drop it, keep (symbol, date). + rolled.index = rolled.index.droplevel(0) + + n = rolled["cnt"].to_numpy(dtype=float) + sx = rolled["sx"].to_numpy(dtype=float) + sy = rolled["sy"].to_numpy(dtype=float) + with np.errstate(invalid="ignore", divide="ignore"): + cov = n * rolled["sxy"].to_numpy(dtype=float) - sx * sy + var_x = n * rolled["sxx"].to_numpy(dtype=float) - sx * sx + var_y = n * rolled["syy"].to_numpy(dtype=float) - sy * sy + den = np.sqrt(var_x * var_y) + corr = np.where((n >= min_pairs) & (den > 0.0), cov / den, np.nan) + corr = np.clip(corr, -1.0, 1.0) + + out = pd.Series(corr, index=rolled.index, name=name) + out.index = out.index.set_names([SYMBOL_LEVEL, DATE_LEVEL]) + return out.reorder_levels([DATE_LEVEL, SYMBOL_LEVEL]).sort_index() + + +class JumpAmountCorrFactor(Factor): + """Price-jump turnover-correlation factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the + panel (produced by :func:`compute_jump_amount_corr`); it does NO minute work + of its own, mirroring the value / financial factors that surface an enriched + column. + + Args: + lookback_days: trailing trading-day window; part of the factor DEFINITION + (reproduced from the report), not a tuned knob. It only names the + column so a non-default window cannot silently mislabel it. + """ + + name: str = f"jump_amount_corr_{JUMP_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = JUMP_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"jump-amount-corr lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"jump_amount_corr_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=-1: the report's RankIC mean is -10.23% (full A, market-cap + + industry neutral) — high jump-amount-correlation predicts LOWER forward + returns. The sign is fixed BEFORE the run; a validated prototype reproduced + it (mean RankIC -0.074 on 2022-2024 sampled names). is_intraday=False by the + module docstring's reasoning (daily signal traded close-to-close). + min_history_bars=0: the warm-up is DATA-dependent (a value appears once + >= ``JUMP_MIN_PAIRS`` jump-pairs accumulate in the trailing window), not a + fixed leading count — the honest NaN rate is reported by data_coverage + rather than hidden behind a fabricated warm-up window. + + D1 declarations (D0 pre-assignment table row 1): adjustment= + returns_invariant — the jump is a within-(symbol, day) amplitude + z-score of the SAME-DAY ratio (high-low)/open (this module's + ``compute_jump_amount_corr``) and the correlated quantity is + ``amount`` (anchor-free), so the adjustment anchor cancels. + overnight_boundary=none — jump/amount pairs are strictly same-session + adjacent minutes (the exact-60s gap test); no raw-price comparison + crosses the overnight boundary. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Price-jump turnover correlation (Kaiyuan report §6): trailing " + f"{self._lookback_days}-trading-day lagged Pearson corr between the " + f"traded amount at price-JUMP minutes (within-day amplitude z-score " + f">1) and the amount at the strictly-next minute. Derived from 1min " + f"bars but a DAILY signal traded close-to-close; >= {JUMP_MIN_PAIRS} " + f"jump-pairs required else NaN." + ), + expected_ic_sign=-1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from. These + # are declared for honest provenance disclosure (data_coverage lists + # them); the daily panel surfaces the pre-aggregated column itself. + input_fields=("high", "low", "open", "amount"), + requires=_minute_requires("high", "low", "open", "amount"), + adjustment="returns_invariant", + overnight_boundary="none", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily jump-amount-corr column off ``panel``. + + The runner runs ``compute_jump_amount_corr`` on the minute cache upstream + and joins the result as ``self.name``; here we only surface it, so this + factor does no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"JumpAmountCorrFactor needs the pre-aggregated '{self.name}' column " + f"on the panel (produced upstream by compute_jump_amount_corr and " + f"joined by the runner); panel has {list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "JUMP_LOOKBACK_DAYS", + "JUMP_MIN_PAIRS", + "JUMP_Z", + "JumpAmountCorrFactor", + "compute_jump_amount_corr", +] diff --git a/factors/compute/minute/minute_ideal_amplitude.py b/factors/compute/minute/minute_ideal_amplitude.py new file mode 100644 index 0000000..95c8a96 --- /dev/null +++ b/factors/compute/minute/minute_ideal_amplitude.py @@ -0,0 +1,293 @@ +"""Minute "ideal amplitude" factor (PR-D): math + surface (D2). + +Reproduces the Kaiyuan report §30 (市场微观结构系列 30) 分钟理想振幅因子 as a daily +PIT-safe column derived from 1min bars. + +Definition (LOCKED — reproduced from the report; N/lambda/min-minutes are part of +the factor DEFINITION, not tuned knobs). For each symbol and each panel date ``d``: + + 1. Take the symbol's most recent ``N`` (=10) trading days INCLUDING ``d`` (the + symbol's own minute-trading days — mirrors the jump factor's trailing window). + 2. PIT truncation (standing authorization): keep only bars with + ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50). + This reuses the I3 per-bar cutoff path, so EVERY day in the window is truncated + to its own [session-open, 14:50] and post-14:50 / close data is never touched. + 3. Per-bar minute amplitude ``amp = high/low - 1``; a bar is dropped unless + ``low > 0`` and ``high >= low``. + 4. Pool ALL surviving bars of the window into ONE set ("merged cut", not a + per-day cut). If the pool has fewer than ``min_minutes`` (=1150 ≈ half of + 10 x ~230) valid minutes, the value is NaN (honest missing — no fabricated + warm-up; coverage is disclosed by the runner). + 5. Rank the pooled minutes by RAW minute close (unadjusted — amplitude is a ratio + so it needs no adjustment, and the report ranks on the raw minute price), with + ``(close, bar_end)`` as a stable total order. With ``k = floor(lambda * n)`` + (lambda=0.25): + V_high = mean amp of the ``k`` HIGHEST-close minutes + V_low = mean amp of the ``k`` LOWEST-close minutes + 6. factor(d, s) = ``V_high - V_low`` (column ``minute_ideal_amp_{N}``). + +Pre-registered sign = -1 (report full-market IC -0.059 / ICIR -3.1 / rankIC -0.076). +The value at ``(d, s)`` uses only bars at dates <= d, so a factor value never sees a +future bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + add_bar_end_ns, + empty_factor_series, + guarded_amplitude, + pooled_trailing_reduce, + visible_minute_frame, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (report terminal parameters; NOT tuned knobs). +IDEAL_AMP_LOOKBACK_DAYS = 10 # trailing trading-day window (N), includes date d +IDEAL_AMP_LAMBDA = 0.25 # top/bottom fraction by close (lambda) that forms V_high/V_low +IDEAL_AMP_MIN_MINUTES = 1150 # minimum valid pooled minutes for a finite value + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def _rank_cut( + closes: np.ndarray, amps: np.ndarray, bar_ends: np.ndarray, lam: float, min_minutes: int +) -> float: + """V_high - V_low over one pooled window; NaN if too few minutes or k < 1. + + ``closes``/``amps``/``bar_ends`` are equal-length arrays for ONE pooled window. + Ranking is a stable total order ``(close, bar_end)`` (lexsort with close as the + primary key), so the top-k / bottom-k selection is fully deterministic even when + two minutes share a close. + """ + n = closes.size + if n < min_minutes: + return float("nan") + k = int(np.floor(lam * n)) + if k < 1: + return float("nan") + order = np.lexsort((bar_ends, closes)) # close primary, bar_end tie-break + a = amps[order] + return float(a[-k:].mean() - a[:k].mean()) + + +def _amplitude_for_symbol( + g: pd.DataFrame, lookback_days: int, lam: float, min_minutes: int +) -> tuple[list[pd.Timestamp], list[float]]: + """Daily factor values for ONE symbol from its PIT-filtered, guarded bars. + + ``g`` holds columns ``trade_date`` / ``close`` / ``amp`` / ``bar_end_ns`` for a + single symbol. Per-day arrays are built once, then each date pools the trailing + ``lookback_days`` days (including that date) via the shared + :func:`~factors.compute.minute.primitives.pooled_trailing_reduce` — no + cross-symbol leakage because ``g`` is a single symbol's slice. + """ + days: list[pd.Timestamp] = [] + day_close: list[np.ndarray] = [] + day_amp: list[np.ndarray] = [] + day_be: list[np.ndarray] = [] + for day, sub in g.groupby("trade_date", sort=True): + days.append(pd.Timestamp(day).normalize()) + day_close.append(sub["close"].to_numpy(dtype=float)) + day_amp.append(sub["amp"].to_numpy(dtype=float)) + day_be.append(sub["bar_end_ns"].to_numpy(dtype="int64")) + + values = pooled_trailing_reduce( + (day_close, day_amp, day_be), + lookback_days=lookback_days, + reducer=lambda closes, amps, bes: _rank_cut(closes, amps, bes, lam, min_minutes), + ) + return days, values + + +def compute_minute_ideal_amplitude( + bars: pd.DataFrame, + *, + lookback_days: int = IDEAL_AMP_LOOKBACK_DAYS, + lam: float = IDEAL_AMP_LAMBDA, + min_minutes: int = IDEAL_AMP_MIN_MINUTES, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "minute_ideal_amp", +) -> pd.Series: + """PIT-safe daily "minute ideal amplitude" factor from 1min ``bars``. + + See the module docstring for the LOCKED definition. The heavy per-symbol loop is + memory-bounded (the runner feeds one symbol at a time), but this function also + accepts a multi-symbol frame and keeps symbols strictly isolated. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. + lookback_days: trailing trading-day window length (part of the definition). + lam: top/bottom close fraction defining V_high/V_low (0 < lam <= 0.5). + min_minutes: minimum valid pooled minutes for a finite value. + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + name: the returned Series name (the factor-panel column name). + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily + factor value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if not (0.0 < lam <= 0.5): + raise ValueError(f"lam must be in (0, 0.5]; got {lam!r}.") + if min_minutes < 2: + # Need at least 2 minutes so a non-empty top/bottom cut can exist. + raise ValueError(f"min_minutes must be >= 2; got {min_minutes!r}.") + if len(bars) == 0: + return empty_factor_series(name) + + visible = visible_minute_frame( + bars, columns=("high", "low", "close"), decision_time=decision_time + ) + if visible.empty: + return empty_factor_series(name) + + visible = guarded_amplitude(visible) + if visible.empty: + return empty_factor_series(name) + + # int64 nanoseconds for a deterministic lexsort tie-break. + visible = add_bar_end_ns(visible) + visible = visible.sort_values([SYMBOL_LEVEL, "bar_end"], kind="mergesort") + + index_tuples: list[tuple] = [] + values: list[float] = [] + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + days, vals = _amplitude_for_symbol(g, lookback_days, lam, min_minutes) + for day, val in zip(days, vals): + index_tuples.append((day, str(sym))) + values.append(val) + + if not index_tuples: + return empty_factor_series(name) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +class MinuteIdealAmplitudeFactor(Factor): + """Minute ideal-amplitude factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_minute_ideal_amplitude`); it does NO minute work of its + own, mirroring its siblings and the value / financial factors that surface an + enriched column. + + Args: + lookback_days: trailing trading-day window; part of the factor DEFINITION + (reproduced from the report), not a tuned knob. It only names the column + so a non-default window cannot silently mislabel it. + """ + + name: str = f"minute_ideal_amp_{IDEAL_AMP_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = IDEAL_AMP_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"minute-ideal-amplitude lookback_days must be a positive integer; " + f"got {lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"minute_ideal_amp_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=-1: the report's RankIC mean is -7.6% (full market, N=10, + lambda=25%) — a HIGH minute ideal amplitude predicts LOWER forward returns. + The sign is fixed BEFORE the run (a validated prototype must reproduce it). + is_intraday=False by the module docstring's reasoning: minute INPUT but a + DAILY signal traded close-to-close. min_history_bars=0: the warm-up is + DATA-dependent (a value appears once >= ``IDEAL_AMP_MIN_MINUTES`` valid + pooled minutes accumulate in the trailing window), not a fixed leading + count — the honest NaN rate is reported by data_coverage. + + D1 declarations (D0 pre-assignment table row 2 + note 1, the table's + flagged judgment call): adjustment=returns_invariant — the amplitude + itself is the same-day ratio high/low - 1 (anchor cancels; module + docstring step 3). overnight_boundary=CROSSED_DISCLOSED — the pooled + ranking key is the RAW minute close across the trailing multi-day + window, so when the window contains an ex-date the pooled ordering + interleaves bars on two different price bases; the definition is + pinned to the report (Wind ranks on the raw price) and deliberately + kept. OPEN OBLIGATION, tracked here per D0 note 1: of the three + CROSSED_DISCLOSED requirements (crossing is real / values kept by + definition / deviation MEASURED and disclosed) the third — a + PR-L-style ex-date deviation measurement (share of pooling windows + containing a true ex-date + realized rank-perturbation magnitude) — + is still MISSING and is owed in D2 alongside that stage's + overnight-boundary property tests. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Minute ideal amplitude (Kaiyuan report §30): pool the 1min bars of " + f"the trailing {self._lookback_days} trading days (PIT-truncated at " + f"14:50 per bar), rank the pooled minutes by RAW close, and return " + f"V_high - V_low where V_high/V_low are the mean per-minute amplitude " + f"(high/low - 1) of the top / bottom floor({IDEAL_AMP_LAMBDA:g}*n) " + f"minutes by close. Derived from 1min bars but a DAILY signal traded " + f"close-to-close; >= {IDEAL_AMP_MIN_MINUTES} valid pooled minutes " + f"required else NaN." + ), + expected_ic_sign=-1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from. Declared + # for honest provenance disclosure (data_coverage lists them); the daily + # panel surfaces the pre-aggregated column itself. + input_fields=("high", "low", "close"), + requires=_minute_requires("high", "low", "close"), + adjustment="returns_invariant", + overnight_boundary="crossed_disclosed", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily minute-ideal-amplitude column off ``panel``. + + The runner runs ``compute_minute_ideal_amplitude`` on the minute cache + upstream and joins the result as ``self.name``; here we only surface it, so + this factor does no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"MinuteIdealAmplitudeFactor needs the pre-aggregated '{self.name}' " + f"column on the panel (produced upstream by " + f"compute_minute_ideal_amplitude and joined by the runner); panel has " + f"{list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "IDEAL_AMP_LAMBDA", + "IDEAL_AMP_LOOKBACK_DAYS", + "IDEAL_AMP_MIN_MINUTES", + "MinuteIdealAmplitudeFactor", + "compute_minute_ideal_amplitude", +] diff --git a/factors/compute/minute/mmp.py b/factors/compute/minute/mmp.py new file mode 100644 index 0000000..c7ba8bc --- /dev/null +++ b/factors/compute/minute/mmp.py @@ -0,0 +1,194 @@ +"""Minute Microstructure Pressure (MMP, I5c): the per-bar factor math (D2). + +EXPLORATORY factor (never promoted; I5d's CSI500 monotonicity degraded on the +corrected engine and I5e's CSI300 generalization failed — MMP is on hold). The +math lives HERE since D2 (moved from ``data.clean.intraday_aggregate``, which +re-exports it); the daily equal-weight aggregation is consumed through +``asof_daily_features(features=["mmp_ew"])`` in the aggregate module's generic +core, which imports :func:`mmp_ew_daily` from this module. + +Layering note: this module must NEVER import ``data.clean.intraday_aggregate`` +— the aggregate module imports THIS one (re-export + feature hook), so an +import back would be a genuine cycle. Everything shared lives in +``data.clean.intraday_schema`` / ``factors.compute.minute.primitives``. + +The window is part of the factor definition, not a tuned parameter. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + DEFAULT_SESSION_OPEN, + SYMBOL_LEVEL, + validate_intraday_bars, +) + +# Minute Microstructure Pressure (MMP, I5c): rolling baseline window (prior bars +# t-MMP_LOOKBACK..t-1) and the default denominator epsilon. EXPLORATORY factor; +# the window is part of the factor definition, not a tuned parameter. +MMP_LOOKBACK = 20 +DEFAULT_EPSILON = 1e-6 + + +def compute_minute_mmp( + open_: np.ndarray, + high: np.ndarray, + low: np.ndarray, + close: np.ndarray, + volume: np.ndarray, + *, + lookback: int = MMP_LOOKBACK, + epsilon: float = DEFAULT_EPSILON, +) -> np.ndarray: + """Per-bar Minute Microstructure Pressure ``MMP_t`` for ONE symbol/day session. + + Inputs are equal-length 1D arrays ORDERED by ``bar_end`` ascending and + belonging to a SINGLE ``(symbol, trade_date)`` session. The rolling baselines + use ONLY the prior ``lookback`` bars (``t-lookback..t-1``) — never bar ``t`` + itself, never a later bar, never the prior day's tail — so the first + ``lookback`` bars have NaN ``MMP``. + + mid_t = (high_t + low_t) / 2 + S_t = (close_t - mid_t) / mid_t (NaN if mid_t <= 0) + V_t = sqrt(volume_t / median(volume[t-lookback:t])) + (NaN if baseline <= 0 / NaN) + B_t = |close_t - open_t| / (high_t - low_t + epsilon) + R_t = (high_t - low_t) / (mean(hl[t-lookback:t]) + epsilon) + (NaN if baseline is NaN) + MMP_t = S_t * V_t * B_t * R_t + + Invalid denominators yield NaN, never ``inf``. Pure: reads no returns / no + future bars / no token. + """ + open_ = np.asarray(open_, dtype=float) + high = np.asarray(high, dtype=float) + low = np.asarray(low, dtype=float) + close = np.asarray(close, dtype=float) + volume = np.asarray(volume, dtype=float) + + hl = high - low + mid = (high + low) / 2.0 + + # Prior-`lookback` baselines: rolling over t-lookback+1..t THEN shift(1) so + # position t holds the statistic of bars t-lookback..t-1 (excludes bar t). + med_vol = pd.Series(volume).rolling(lookback).median().shift(1).to_numpy() + ma_hl = pd.Series(hl).rolling(lookback).mean().shift(1).to_numpy() + + with np.errstate(divide="ignore", invalid="ignore"): + s_t = np.where(mid > 0.0, (close - mid) / mid, np.nan) + ratio = np.where(med_vol > 0.0, volume / med_vol, np.nan) + v_t = np.sqrt(ratio) + b_t = np.abs(close - open_) / (hl + epsilon) + r_t = hl / (ma_hl + epsilon) + mmp = s_t * v_t * b_t * r_t + return mmp + + +def in_session_bars( + g: pd.DataFrame, trade_date: pd.Timestamp, session_open: str +) -> pd.DataFrame: + """Bars whose ``bar_end`` is on/after ``session_open`` (the MMP window lower bound). + + The MMP daily score aggregates over ``[session_open, decision_time]`` (the upper + bound is the available_time cutoff already applied upstream). Restricting to the + in-session bars keeps PRE-session bars out of BOTH the rolling baseline and the + daily mean, so the first in-session bar correctly has no prior-20 baseline. + """ + session_start = trade_date + pd.Timedelta(session_open) + return g[g["bar_end"] >= session_start] + + +def mmp_ew_daily( + g: pd.DataFrame, epsilon: float, trade_date: pd.Timestamp, session_open: str +) -> float: + """Equal-weight mean of valid per-minute ``MMP_t`` over one PIT-filtered group. + + ``g`` is one ``(date, symbol)`` session's visible bars, sorted by ``bar_end``; + only the in-session bars (``bar_end >= session_open``) enter, so the rolling + baseline starts at the session open and the first 20 in-session bars are NaN. + Every valid minute ``MMP_t`` gets EQUAL weight (no extra volume weighting — the + volume term already lives inside ``MMP_t``). No valid minute -> NaN. + """ + gs = in_session_bars(g, trade_date, session_open) + if gs.empty: + return float("nan") + mmp = compute_minute_mmp( + gs["open"].to_numpy(dtype=float), + gs["high"].to_numpy(dtype=float), + gs["low"].to_numpy(dtype=float), + gs["close"].to_numpy(dtype=float), + gs["volume"].to_numpy(dtype=float), + epsilon=epsilon, + ) + valid = mmp[~np.isnan(mmp)] + return float(np.mean(valid)) if valid.size else float("nan") + + +def mmp_valid_minute_counts( + bars: pd.DataFrame, + *, + decision_time: str = DEFAULT_DECISION_TIME, + session_open: str = DEFAULT_SESSION_OPEN, + epsilon: float = DEFAULT_EPSILON, +) -> pd.Series: + """Per-``(date, symbol)`` count of valid (non-NaN) ``MMP_t`` minutes (I5c report). + + Report-only diagnostic: applies the SAME window as the daily MMP score — + ``available_time <= trade_date + decision_time`` (upper bound) AND + ``bar_end >= trade_date + session_open`` (lower bound) — then counts the + in-session minutes that yielded a valid ``MMP_t`` (the first ``MMP_LOOKBACK`` + in-session bars never do). Reuses :func:`compute_minute_mmp` so there is a + single MMP source of truth. + """ + validate_intraday_bars(bars) + empty = pd.Series( + [], dtype=int, + index=pd.MultiIndex.from_arrays( + [pd.DatetimeIndex([]), pd.Index([], dtype=object)], + names=DAILY_INDEX_NAMES, + ), + ) + if len(bars) == 0: + return empty + work = bars.reset_index() + work["trade_date"] = work["bar_end"].dt.normalize() + cutoff = work["trade_date"] + pd.Timedelta(decision_time) + visible = work.loc[work["available_time"] <= cutoff].copy() + if visible.empty: + return empty + visible = visible.sort_values([SYMBOL_LEVEL, "bar_end"]) + index_tuples: list[tuple] = [] + counts: list[int] = [] + for (date, sym), g in visible.groupby(["trade_date", SYMBOL_LEVEL], sort=True): + gs = in_session_bars(g, pd.Timestamp(date).normalize(), session_open) + if gs.empty: + index_tuples.append((date, str(sym))) + counts.append(0) + continue + mmp = compute_minute_mmp( + gs["open"].to_numpy(dtype=float), + gs["high"].to_numpy(dtype=float), + gs["low"].to_numpy(dtype=float), + gs["close"].to_numpy(dtype=float), + gs["volume"].to_numpy(dtype=float), + epsilon=epsilon, + ) + index_tuples.append((date, str(sym))) + counts.append(int(np.count_nonzero(~np.isnan(mmp)))) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(counts, index=index, dtype=int).sort_index() + + +__all__ = [ + "DEFAULT_EPSILON", + "MMP_LOOKBACK", + "compute_minute_mmp", + "in_session_bars", + "mmp_ew_daily", + "mmp_valid_minute_counts", +] diff --git a/factors/compute/minute/peak_interval_kurtosis.py b/factors/compute/minute/peak_interval_kurtosis.py new file mode 100644 index 0000000..d804069 --- /dev/null +++ b/factors/compute/minute/peak_interval_kurtosis.py @@ -0,0 +1,429 @@ +"""Volume-peak INTERVAL-KURTOSIS factor (PR-H): math + surface (D2). + +Reproduces the SECOND factor of the Kaiyuan market-microstructure series #27 (开源证券 +《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, +§6): "针对两个量峰之间的时间间隔计算统计指标，对过去 20 日同日前后两个量峰之间的时间间隔 +分布，计算其峰度". + +Same MACHINE as PR-F, different STATISTIC. The volume-peak identification is the +shared taxonomy in :mod:`factors.compute.minute.primitives` +(``prepare_visible_minute_bars`` + ``peak_mask_for_symbol``) — same-slot +strictly-prior μ+kσ eruptive/mild classification, 1-minute same-session +mild-neighbour peak rule, session-boundary and missing-minute bars are never peaks, +a day is VALID iff it has enough classifiable bars. Nothing about the taxonomy is +re-implemented here, so the factors can never drift apart; this module only measures +the GAPS between consecutive peaks and reduces them to a kurtosis. + +The report is silent on two operative details; both are deliberate, DISCLOSED choices +PINNED in the task card (task_card_pr_h_*.md §1), not tuned knobs, and both are +reproduced on the factor spec: + + 1. INTERVAL UNIT = TRADING MINUTES, i.e. the difference in TRADABLE SLOT POSITION + within the day's PIT-visible bar sequence — NOT wall-clock minutes. The lunch break + therefore costs nothing: a peak at 11:29 and a peak at 13:02 are 3 trading minutes + apart (11:29 -> 11:30 -> 13:01 -> 13:02), not 93. Measuring wall clock would inject + a fixed ~90-minute spike into every interval distribution that happens to straddle + lunch and would dominate the kurtosis. A consequence, disclosed rather than + corrected: a minute MISSING from the cache is not a tradable slot in our sequence, + so an interval spanning it is measured one shorter (the same stance PR-F takes on + gaps — it consults no exchange calendar). + 2. KURTOSIS = FISHER EXCESS, BIAS-CORRECTED — the ``pandas.Series.kurt()`` / + ``scipy.stats.kurtosis(fisher=True, bias=False)`` convention (a normal sample sits + near 0, not 3). ``excess_kurtosis`` implements it in numpy for speed and is locked + against ``pandas.kurt()`` by test. + +Factor value: ``peak_interval_kurtosis_20`` = the kurtosis of the POOLED interval +multiset over the symbol's most recent ``lookback_days`` (=20) VALID trading days +INCLUDING ``d``. A day with fewer than 2 peaks contributes ZERO intervals but is still a +valid day (it is not skipped and does not poison the window). Two NaN gates, both honest +missing rather than a fabricated number: fewer than ``min_intervals`` (=20) pooled +intervals (kurtosis is wildly unstable on small samples), or a zero-variance pool +(kurtosis is undefined). The PR-F valid-day floor is kept as well (``min_valid_days``). +Note the reused taxonomy makes two peaks 1 minute apart impossible — adjacent eruptive +minutes are RIDGES — so the smallest attainable interval is 2. + +Pre-registered sign = +1 (the report's full-market RankIC is +7.19% / RankICIR 4.63, +long-short 23.3%/yr, IR 3.39, 13/13 positive years — the most stable factor in the +report). Semantics: a peaky, fat-tailed interval distribution means informed trading +arrives in BURSTS rather than spread evenly through the session -> higher future return. +NOTE the report is a MONTHLY, market-cap + industry neutral full-market series on Wind +data; our eval cell is CSI500 daily with industry + size neutralization, so its numbers +are a LOOSE reference only (the report gives no CSI500 sub-domain figure for THIS +factor, and none is invented). Raw minute volume (cached as-is) has split-day magnitude +jumps that pollute the 20-day σ; the report (Wind) does not adjust for this either, so +it is disclosed and NOT corrected. + +The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future +bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + VOLUME_PRV_BASELINE_DAYS, + VOLUME_PRV_BASELINE_MIN_OBS, + VOLUME_PRV_MIN_CLASSIFIABLE, + VOLUME_PRV_MIN_VALID_DAYS, + VOLUME_PRV_SIGMA_K, + empty_factor_series, + peak_mask_for_symbol, + prepare_visible_minute_bars, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). +# The peak-identification constants live with the taxonomy in ``primitives``. +PEAK_INTERVAL_LOOKBACK_DAYS = 20 # trailing VALID trading-day pool window, includes d +PEAK_INTERVAL_MIN_INTERVALS = 20 # min pooled intervals for a finite kurtosis + +# Kurtosis needs at least 4 observations for the bias-corrected estimator to exist. +_KURTOSIS_MIN_N = 4 + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def excess_kurtosis(x: np.ndarray) -> float: + """Fisher excess, bias-corrected kurtosis of ``x`` — the ``pandas.kurt()`` convention. + + Equivalent to ``pandas.Series(x).kurt()`` and to + ``scipy.stats.kurtosis(x, fisher=True, bias=False)`` (locked by test), computed in + numpy because the caller evaluates it once per symbol-day:: + + G2 = n(n+1)(n-1)·M4 / ((n-2)(n-3)·M2²) - 3(n-1)² / ((n-2)(n-3)) + + with ``M2 = Σ(x-x̄)²`` and ``M4 = Σ(x-x̄)⁴`` (central sums, NOT means). Central sums + are accumulated after subtracting the mean, so no catastrophic power-sum + cancellation. + + Returns: + The excess kurtosis, or NaN when it is undefined: fewer than 4 observations, or + a zero-variance sample. Never raises, never returns ±inf. + """ + n = x.size + if n < _KURTOSIS_MIN_N: + return float("nan") + d = x - x.mean() + m2 = float(np.dot(d, d)) + if not m2 > 0.0: # zero variance (or a non-finite that made it NaN) -> undefined + return float("nan") + m4 = float(np.dot(d * d, d * d)) + left = n * (n + 1.0) * (n - 1.0) * m4 / ((n - 2.0) * (n - 3.0) * m2 * m2) + adj = 3.0 * (n - 1.0) ** 2 / ((n - 2.0) * (n - 3.0)) + return left - adj + + +def peak_intervals_by_day(work: pd.DataFrame) -> pd.Series: + """Trading-minute gaps between consecutive same-day peaks, per trade date. + + ``work`` is one symbol's frame as returned by + :func:`~factors.compute.minute.primitives.peak_mask_for_symbol` (sorted by + ``(trade_date, bar_end)`` with a boolean ``peak`` column). + + The interval unit is the TRADABLE SLOT POSITION difference inside the day's visible + bar sequence (PINNED §1 of the module docstring): position 0, 1, 2, ... is assigned + to the day's PIT-visible bars in time order, so consecutive tradable minutes are 1 + apart REGARDLESS of the lunch break sitting between them. Intervals are never taken + ACROSS days (the positions restart every trade date). + + Returns: + Series indexed by ``trade_date`` (every day present in ``work``, in order) whose + values are integer numpy arrays of that day's peak-to-peak intervals — an EMPTY + array for a day with fewer than 2 peaks (zero intervals, still a real day). + """ + # position within the day's visible bar sequence == "trading minute" coordinate + pos = work.groupby("trade_date", sort=False).cumcount().to_numpy() + peak = work["peak"].to_numpy(dtype=bool) + dates = work["trade_date"].to_numpy() + + out_dates: list[pd.Timestamp] = [] + out_intervals: list[np.ndarray] = [] + for day, idx in pd.Series(np.arange(len(work))).groupby(dates, sort=True): + sel = idx.to_numpy() + day_pos = pos[sel][peak[sel]] + out_dates.append(pd.Timestamp(day)) + out_intervals.append(np.diff(day_pos).astype(np.int64)) + return pd.Series(out_intervals, index=pd.Index(out_dates, name="trade_date")) + + +def _kurtosis_for_symbol( + g: pd.DataFrame, + *, + baseline_days: int, + baseline_min_obs: int, + sigma_k: float, + lookback_days: int, + min_valid_days: int, + min_classifiable: int, + min_intervals: int, +) -> tuple[list[pd.Timestamp], list[float]]: + """Daily peak-interval-kurtosis values for ONE symbol from its PIT-visible bars. + + Identifies the peaks with the SHARED :func:`peak_mask_for_symbol`, measures each + valid day's peak-to-peak trading-minute intervals, then pools the trailing + ``lookback_days`` valid days and reduces the pool to its excess kurtosis. No + cross-symbol leakage (``g`` is one symbol's slice) and no lookahead (the baseline is + strictly prior, the pool is trailing). + """ + work = peak_mask_for_symbol( + g, + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + ) + + classifiable_count = work.groupby("trade_date")["classifiable"].sum() + valid_days = classifiable_count.index[classifiable_count >= min_classifiable] + if len(valid_days) == 0: + return [], [] + + intervals = peak_intervals_by_day(work) + # Keep ONLY valid days, in order: a day below the classifiable floor contributes + # nothing and does not occupy a slot in the trailing window (same rule as PR-F). + valid = [pd.Timestamp(d) for d in sorted(valid_days)] + per_day = [intervals.loc[d] for d in valid] + + days: list[pd.Timestamp] = [] + values: list[float] = [] + for i, day in enumerate(valid): + window = per_day[max(0, i - lookback_days + 1) : i + 1] + days.append(day.normalize()) + if len(window) < min_valid_days: + values.append(float("nan")) + continue + pool = np.concatenate(window) if window else np.empty(0, dtype=np.int64) + # Gate 1: too few pooled intervals -> honest NaN (kurtosis is wild on small + # samples). Gate 2 (zero variance / n < 4) lives inside excess_kurtosis. + if pool.size < min_intervals: + values.append(float("nan")) + continue + values.append(excess_kurtosis(pool.astype(float))) + return days, values + + +def compute_peak_interval_kurtosis( + bars: pd.DataFrame, + *, + lookback_days: int = PEAK_INTERVAL_LOOKBACK_DAYS, + baseline_days: int = VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, + sigma_k: float = VOLUME_PRV_SIGMA_K, + min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + min_intervals: int = PEAK_INTERVAL_MIN_INTERVALS, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "peak_interval_kurtosis", +) -> pd.Series: + """PIT-safe daily "volume-peak interval kurtosis" factor from 1min ``bars``. + + Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, + identifies the volume peaks with the SHARED PR-F taxonomy, measures the + trading-minute gaps between consecutive same-day peaks, pools the trailing + ``lookback_days`` VALID days and returns the pool's Fisher excess (bias-corrected) + kurtosis. See the module docstring for the LOCKED definition and the two pinned + interpretations (interval unit, kurtosis convention). + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping + is strictly per symbol (no cross-symbol leakage). + lookback_days: trailing VALID trading-day window pooled for the kurtosis. + baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). + baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). + sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). + min_valid_days: minimum valid days in the trailing window for a finite value. + min_classifiable: a day is VALID iff it has at least this many classifiable bars. + min_intervals: minimum POOLED intervals for a finite kurtosis (>= 4, the point + below which the bias-corrected estimator does not exist). + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + name: the returned Series name (the factor-panel column name). + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily + factor value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if baseline_days < 2: + # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is + # defined. + raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") + if baseline_min_obs < 2: + raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") + if sigma_k < 0.0: + raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") + if min_valid_days < 1: + raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") + if min_classifiable < 1: + raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") + if min_intervals < _KURTOSIS_MIN_N: + raise ValueError( + f"min_intervals must be >= {_KURTOSIS_MIN_N} (the bias-corrected kurtosis " + f"does not exist below that); got {min_intervals!r}." + ) + if len(bars) == 0: + return empty_factor_series(name) + + visible = prepare_visible_minute_bars(bars, decision_time=decision_time) + if visible.empty: + return empty_factor_series(name) + + index_tuples: list[tuple] = [] + values: list[float] = [] + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + days, vals = _kurtosis_for_symbol( + g.reset_index(drop=True), + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + lookback_days=lookback_days, + min_valid_days=min_valid_days, + min_classifiable=min_classifiable, + min_intervals=min_intervals, + ) + for day, val in zip(days, vals): + index_tuples.append((day, str(sym))) + values.append(val) + + if not index_tuples: + return empty_factor_series(name) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +class PeakIntervalKurtosisFactor(Factor): + """Volume-peak interval-kurtosis factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_peak_interval_kurtosis`); it does NO minute work of its + own, mirroring its siblings and the value / financial factors that surface an + enriched column. + + Args: + lookback_days: trailing VALID trading-day window pooled for the kurtosis; part of + the factor DEFINITION (reproduced from the report), not a tuned knob. It only + names the column so a non-default window cannot silently mislabel it. + """ + + name: str = f"peak_interval_kurtosis_{PEAK_INTERVAL_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = PEAK_INTERVAL_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"peak-interval-kurtosis lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"peak_interval_kurtosis_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=+1: the report's full-market RankIC is +7.19% (RankICIR 4.63, + long-short 23.3%/yr, IR 3.39, max drawdown 7.37%, 13/13 positive years — the most + stable factor in the report). A peaky, fat-tailed peak-interval distribution means + informed trading arrives in BURSTS rather than evenly through the session. The + sign is fixed BEFORE the run (a validated prototype must reproduce it). + is_intraday=False for the same reason as the siblings: minute INPUT but a DAILY + signal traded close-to-close. min_history_bars=0: the warm-up is DATA-dependent (a + value appears once enough VALID days AND enough pooled intervals accumulate), not + a fixed leading count — the honest NaN rate is reported by data_coverage. + + The description spells out the DISTINCTION FROM PR-F (``volume_peak_count``): the + SAME peak identification (shared, not re-implemented) reduced by a DIFFERENT + statistic — the shape of the gap distribution rather than the peak count — plus + the two interpretations the report leaves open (interval unit, kurtosis + convention). + + D1 declarations (D0 pre-assignment table row 6): adjustment=none — the + peak identification is the shared pure volume machinery (factors/ + compute/minute/primitives.py, not re-implemented) and the statistic is + a kurtosis of trading-minute POSITION gaps (``peak_intervals_by_day``), + so no price is ever read. overnight_boundary=none — no raw-price + comparison exists. The split-day raw-volume σ pollution is disclosed + (same as PR-F) and per D0 note 3 belongs to neither axis. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Volume-peak interval kurtosis (Kaiyuan microstructure series #27, " + f"SECOND factor 量峰间隔峰度). SAME peak identification as PR-F " + f"volume_peak_count (SHARED taxonomy in factors.compute.minute." + f"primitives, not re-implemented): 1min bars PIT-truncated at 14:50, a " + f"minute is ERUPTIVE if vol > μ + {VOLUME_PRV_SIGMA_K:g}σ of its " + f"SAME-SLOT strictly-prior {VOLUME_PRV_BASELINE_DAYS}-day baseline, and " + f"a PEAK is an eruptive minute whose both 1-minute same-session " + f"neighbours are mild. DIFFERENT STATISTIC: the gaps between consecutive " + f"same-day peaks, pooled over the trailing {self._lookback_days} VALID " + f"days (a day with < 2 peaks contributes 0 intervals but is still " + f"valid), reduced to their kurtosis. PINNED interpretations of an " + f"under-specified report: (1) an interval is measured in TRADING MINUTES " + f"— the tradable-slot difference inside the day's visible bar sequence, " + f"so the lunch break costs nothing (11:29 and 13:02 peaks are 3 apart, " + f"not 93) and a wall-clock ~90-minute spike can never dominate the " + f"distribution; (2) kurtosis = FISHER excess, bias-corrected (the pandas " + f".kurt() / scipy fisher=True bias=False convention; normal = 0). NaN " + f"unless >= {PEAK_INTERVAL_MIN_INTERVALS} intervals pooled and the pool " + f"has non-zero variance. Derived from 1min bars but a DAILY signal " + f"traded close-to-close." + ), + expected_ic_sign=1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar field the upstream aggregation is derived from. Declared for + # honest provenance disclosure (data_coverage lists it); the daily panel + # surfaces the pre-aggregated column itself. + input_fields=("volume",), + requires=_minute_requires("volume"), + adjustment="none", + overnight_boundary="none", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily peak-interval-kurtosis column off ``panel``. + + The runner runs ``compute_peak_interval_kurtosis`` per symbol on the minute cache + upstream and joins the result as ``self.name``; here we only surface it, so this + factor does no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"PeakIntervalKurtosisFactor needs the pre-aggregated '{self.name}' " + f"column on the panel (produced upstream by " + f"compute_peak_interval_kurtosis and joined by the runner); panel has " + f"{list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "PEAK_INTERVAL_LOOKBACK_DAYS", + "PEAK_INTERVAL_MIN_INTERVALS", + "PeakIntervalKurtosisFactor", + "compute_peak_interval_kurtosis", + "excess_kurtosis", + "peak_intervals_by_day", +] diff --git a/factors/compute/minute/peak_ridge_amount_ratio.py b/factors/compute/minute/peak_ridge_amount_ratio.py new file mode 100644 index 0000000..f606969 --- /dev/null +++ b/factors/compute/minute/peak_ridge_amount_ratio.py @@ -0,0 +1,560 @@ +"""PEAK/RIDGE AMOUNT-RATIO factor (PR-M): math + surface (D2). + +Reproduces the SEVENTH factor of the Kaiyuan market-microstructure series #27 (开源证券 +《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, +§7.2 「峰岭成交比因子多空年化收益 27.13%」): "本小节通过计算 20 日量峰总成交额与量岭总成交额， +二者做比作为峰岭成交比因子，衡量知情交易相对个人投资者交易的相对参与程度". + +This factor carries NO price information at all — it is a pure TRADED-VALUE mix between +the two eruptive groups — so it is the one test that separates "only price information +survives in this taxonomy" from "the peak/ridge split itself carries alpha". + +The classification is the SHARED taxonomy in :mod:`factors.compute.minute.primitives` +(``prepare_visible_minute_bars`` + ``peak_mask_for_symbol``) — same same-slot +strictly-prior μ+kσ eruptive test, same classifiable rule. Nothing about the taxonomy is +re-implemented here, so the family can never drift apart. + +AGGREGATION — THE REPORT'S FORM, WHICH IS *NOT* THE MEAN OF DAILY RATIOS +------------------------------------------------------------------------ +The report says "计算 20 日量峰总成交额与量岭总成交额，二者做比": sum the peak amount over +20 days, sum the ridge amount over 20 days, THEN divide. That is a RATIO OF SUMS. Contrast +§7.1 (PR-J), which explicitly says "计算 20 日价格比均值" — a MEAN OF RATIOS. The report +draws the distinction itself in adjacent sections, so it is a deliberate difference in the +source and is followed here. + +The two forms are genuinely different, not a rounding detail: a ratio of sums is +AMOUNT-WEIGHTED (busy days dominate) while a mean of ratios weights every valid day +equally. The ratio of sums is also structurally far better behaved for THIS quantity — +``peak_amt / ridge_amt`` on a single day has a small, noisy denominator and a heavy right +tail, and a single day whose ridge amount nearly vanishes would dominate a 20-day mean of +ratios. Pooling both legs first is the natural estimator of a 20-day participation MIX, +which is exactly what the report says the factor measures. + +PINNED choices (deliberate and DISCLOSED, not tuned knobs; reproduced on the factor spec): + + 1. PEAK IS THE NUMERATOR, RIDGE THE DENOMINATOR — "峰岭成交比", peak first, and the + report's stated semantics ("衡量知情交易相对个人投资者交易的相对参与程度", informed + trading RELATIVE TO retail) fix the direction independently of the name's word order. + 2. THE RIDGE MASK IS ``eruptive & ~peak`` and the PEAK MASK is PR-F's isolated-eruption + test — the identical masks PR-J used, so ``valley | peak | ridge == classifiable`` + stays an exact partition and no eruptive bar is silently counted on both sides. A + VALLEY bar enters NEITHER leg: this factor reads only the two ERUPTIVE groups. + 3. POSITIVE-TRADE GUARD. A bar with non-finite or non-positive ``amount`` carries no + traded value, so it is dropped from both legs and from both bar counts. The guard runs + at the summation step only — never before classification — because the same-slot μ/σ + baseline is the shared taxonomy's and must stay bit-identical. Unlike PR-I/PR-J this + factor never divides by volume, so ``volume`` is NOT part of the guard: a bar with a + real amount contributes its amount regardless of how its volume is recorded. + 4. RAW (UNADJUSTED) AMOUNTS. ``amount`` is traded VALUE in RMB, which no split or + dividend adjustment factor rescales — the adjustment moves prices and share counts in + compensating directions. Both legs are therefore free of the ex-date caveat PR-L had + to disclose, and free even of PR-I/PR-J's weaker "cancels within the day" argument: + there is nothing to cancel. + 5. BOTH LEGS COVER THE PIT-VISIBLE WINDOW ONLY (09:31–14:50), not the full session. A + NECESSARY DEVIATION from the report, which uses the whole day: the standing 14:50 + decision cutoff truncates history days and the signal day identically, and reading the + closing auction would be lookahead at our decision time. + 6. AN ASYMMETRIC BAR FLOOR — ``min_peak_bars`` (=5) against ``min_ridge_bars`` (=10). + PEAKS ARE THE SCARCER LEG HERE, the reverse of PR-J's valley/ridge asymmetry: a peak + must erupt AND be ISOLATED, and isolation is a strong condition on a liquid name whose + eruptions cluster. Holding the peak leg to the ridge floor would discard sound days + and bias the surviving sample toward names whose eruptions happen to arrive alone. + The floor is therefore set LOW, deliberately, and both the realized peak-bar + distribution and the resulting day-validity rate are REPORTED + (``with_diagnostics=True``) rather than left implicit — together with the + counterfactual valid-day count at a peak floor of 10. + 7. BOTH BAR COUNTS ARE TAKEN AFTER THE GUARD, so a day cannot qualify on the strength of + bars that traded nothing. + +Factor value: ``peak_ridge_amount_ratio_20`` = ``Σ peak_amt / Σ ridge_amt`` where both sums +run over the symbol's most recent ``lookback_days`` (=20) VALID trading days INCLUDING +``d``. A day is VALID iff it clears four gates: at least ``min_classifiable`` (=100) +classifiable bars (PR-F's gate, unchanged), at least ``min_peak_bars`` (=5) tradable peak +bars, at least ``min_ridge_bars`` (=10) tradable ridge bars, and strictly positive amount in +BOTH legs. Fewer than ``min_valid_days`` (=10) valid days in the trailing window -> NaN +(honest missing; the runner discloses the coverage). Values are emitted only on valid days. + +Pre-registered sign = +1, READ FROM THE REPORT, not from data and not from semantics: §7.2 +states "峰岭成交比因子 RankIC 均值 10.28%，RankICIR 4.07" — an explicitly POSITIVE RankIC — +with a long leg of 16.06%/yr, long-short 27.13%/yr, IR 2.89, max drawdown 7.88% and a 74.3% +monthly win rate (every year positive 2013–2025). NOTE the report is a MONTHLY, market-cap ++ industry neutral full-market series on Wind data while our eval cell is CSI500 daily with +industry + size neutralization, so its numbers are a LOOSE reference only. + +The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future bar +(invariant #1); it is a DAILY signal traded close-to-close from d+1. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + VOLUME_PRV_BASELINE_DAYS, + VOLUME_PRV_BASELINE_MIN_OBS, + VOLUME_PRV_MIN_CLASSIFIABLE, + VOLUME_PRV_MIN_VALID_DAYS, + VOLUME_PRV_SIGMA_K, + empty_factor_series, + peak_mask_for_symbol, + prepare_visible_minute_bars, + symbol_frames, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). +# The classification constants live with the taxonomy in ``primitives``. +PEAK_RIDGE_LOOKBACK_DAYS = 20 # trailing VALID trading-day window for BOTH sums, includes d +# PINNED LOWER than the ridge floor (module docstring §6): a peak must erupt AND be +# ISOLATED, which makes peaks the structurally scarcer leg of this pair. +PEAK_RIDGE_MIN_PEAK_BARS = 5 # min TRADABLE peak bars for a valid day +PEAK_RIDGE_MIN_RIDGE_BARS = 10 # min TRADABLE ridge bars for a valid day + +# The extra 1min column this factor needs on top of the taxonomy's (volume): the traded +# value. Unlike PR-I / PR-J this factor never divides by volume — amount is the whole +# quantity. +_AMOUNT = "amount" + +# Per-day diagnostic columns (the peak-scarcity disclosure the task card requires). +DIAGNOSTIC_COLUMNS = ("classifiable_bars", "peak_bars", "ridge_bars", "valid") + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +# --------------------------------------------------------------------------- # +# The three load-bearing steps, each a NAMED function. +# +# They are small enough to inline, but each one carries a correctness property that the +# test-suite has to be able to BREAK on purpose: the positive-trade guard, the trailing +# (never forward-looking) window, and the per-symbol split. A test that asserts "perturbing +# X changes nothing" proves nothing unless the defective implementation can be substituted +# and shown to FAIL it, so each property gets its own substitutable seam. The per-symbol +# split is the shared ``primitives.symbol_frames``, aliased into THIS module's globals so +# the seam stays patchable here. +# --------------------------------------------------------------------------- # +def _tradable_amount(amt: np.ndarray) -> np.ndarray: + """Positive-trade guard (module docstring §3): finite, strictly positive amount. + + ``volume`` is deliberately absent — this factor never divides by volume, so a bar with + a real traded value contributes it regardless of how its volume is recorded. + """ + return np.isfinite(amt) & (amt > 0.0) + + +def _trailing_ratio_of_sums( + legs: pd.DataFrame, *, lookback_days: int, min_valid_days: int +) -> pd.Series: + """The report's ratio of 20-day sums over a STRICTLY TRAILING valid-day window. + + Both legs are pooled with the SAME window and the SAME ``min_periods``, so numerator + and denominator always cover exactly the same days. The window is trailing and the + index is sorted ascending first, so a value at ``d`` can never absorb a later day. + """ + ordered = legs.sort_index() + roll = ordered.rolling(lookback_days, min_periods=min_valid_days).sum() + return roll["peak_amt"] / roll["ridge_amt"] + + +# The per-symbol split seam (see the block comment above): the shared primitive bound to +# a module-local name so tests can substitute a defective split HERE. +_symbol_frames = symbol_frames + + +def peak_ridge_amount_by_day( + work: pd.DataFrame, + *, + min_peak_bars: int = PEAK_RIDGE_MIN_PEAK_BARS, + min_ridge_bars: int = PEAK_RIDGE_MIN_RIDGE_BARS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + with_diagnostics: bool = False, +) -> pd.DataFrame | tuple[pd.DataFrame, pd.DataFrame]: + """Daily peak / ridge traded-AMOUNT totals for ONE symbol, on VALID days only. + + ``work`` is one symbol's frame as returned by + :func:`~factors.compute.minute.primitives.peak_mask_for_symbol`, which must have been + built from bars prepared with ``extra_columns=("amount",)`` so the traded value is + available alongside the ``peak`` / ``ridge`` / ``classifiable`` masks. + + Returns the two LEGS rather than their daily ratio, because the factor is the report's + RATIO OF 20-DAY SUMS (module docstring), not the mean of daily ratios — the caller + pools each leg over the trailing window first and divides once at the end. + + Args: + work: one symbol's classified minute frame (see above). + min_peak_bars: minimum TRADABLE peak bars for a valid day (PINNED lower than the + ridge floor — see §6; peaks are the scarcer leg of this pair). + min_ridge_bars: minimum TRADABLE ridge bars for a valid day. + min_classifiable: minimum classifiable bars for a valid day (PR-F's gate). + with_diagnostics: also return the per-day bar-count frame, so the caller can + REPORT the peak-scarcity distribution instead of only gating on it. + + Returns: + DataFrame indexed by ``trade_date`` (ascending) with ``peak_amt`` / ``ridge_amt`` + columns for the days that clear all gates — invalid days are ABSENT, not NaN, so + they do not occupy a slot in the caller's trailing window (the same rule + PR-F / PR-H / PR-I / PR-J use). With ``with_diagnostics=True``, a + ``(legs, diagnostics)`` pair where ``diagnostics`` is indexed by EVERY day present + in ``work`` and carries :data:`DIAGNOSTIC_COLUMNS`. + """ + amt = work[_AMOUNT].to_numpy(dtype=float) + # Positive-trade guard: a bar with no finite positive traded value contributes nothing. + # Applied HERE, at the summation step, never before classification — the shared + # same-slot baseline must stay bit-identical. + tradable = _tradable_amount(amt) + peak = work["peak"].to_numpy(dtype=bool) & tradable + ridge = work["ridge"].to_numpy(dtype=bool) & tradable + + per_bar = pd.DataFrame( + { + "trade_date": work["trade_date"].to_numpy(), + "peak_amt": np.where(peak, amt, 0.0), + "ridge_amt": np.where(ridge, amt, 0.0), + "peak_bars": peak.astype(np.int64), + "ridge_bars": ridge.astype(np.int64), + "classifiable_bars": work["classifiable"] + .to_numpy(dtype=bool) + .astype(np.int64), + } + ) + agg = per_bar.groupby("trade_date", sort=True).sum() + + # Four validity gates (module docstring): PR-F's classifiable floor (unchanged), + # enough TRADABLE peak bars, enough TRADABLE ridge bars, and strictly positive amount + # on both legs (a day contributing 0 to the denominator pool is never fabricated). + valid = ( + (agg["classifiable_bars"] >= min_classifiable) + & (agg["peak_bars"] >= min_peak_bars) + & (agg["ridge_bars"] >= min_ridge_bars) + & (agg["peak_amt"] > 0.0) + & (agg["ridge_amt"] > 0.0) + ) + legs = agg.loc[valid, ["peak_amt", "ridge_amt"]].astype(float) + + if not with_diagnostics: + return legs + diagnostics = agg[["classifiable_bars", "peak_bars", "ridge_bars"]].copy() + diagnostics["valid"] = valid + return legs, diagnostics + + +def _peak_ridge_amount_ratio_for_symbol( + g: pd.DataFrame, + *, + baseline_days: int, + baseline_min_obs: int, + sigma_k: float, + lookback_days: int, + min_valid_days: int, + min_classifiable: int, + min_peak_bars: int, + min_ridge_bars: int, + collect_diagnostics: bool = False, +) -> tuple[list[pd.Timestamp], list[float], pd.DataFrame | None]: + """Daily peak/ridge amount-ratio values for ONE symbol from its PIT-visible bars. + + Classifies the minutes with the SHARED :func:`peak_mask_for_symbol`, reduces each valid + day to its two amount legs, then divides the trailing-``lookback_days``-valid-day SUM of + the peak leg by that of the ridge leg (the report's ratio-of-sums form). No cross-symbol + leakage (``g`` is one symbol's slice) and no lookahead (the baseline is strictly prior, + both rolling windows are trailing). + """ + work = peak_mask_for_symbol( + g, + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + ) + result = peak_ridge_amount_by_day( + work, + min_peak_bars=min_peak_bars, + min_ridge_bars=min_ridge_bars, + min_classifiable=min_classifiable, + with_diagnostics=collect_diagnostics, + ) + if collect_diagnostics: + legs, diagnostics = result + else: + legs, diagnostics = result, None + if legs.empty: + return [], [], diagnostics + + # RATIO OF SUMS over the trailing lookback_days VALID days (including d); NaN until + # min_valid_days valid days have accumulated. + ratio = _trailing_ratio_of_sums( + legs, lookback_days=lookback_days, min_valid_days=min_valid_days + ) + days = [pd.Timestamp(d).normalize() for d in ratio.index] + return days, list(ratio.to_numpy(dtype=float)), diagnostics + + +def compute_peak_ridge_amount_ratio( + bars: pd.DataFrame, + *, + lookback_days: int = PEAK_RIDGE_LOOKBACK_DAYS, + baseline_days: int = VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, + sigma_k: float = VOLUME_PRV_SIGMA_K, + min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + min_peak_bars: int = PEAK_RIDGE_MIN_PEAK_BARS, + min_ridge_bars: int = PEAK_RIDGE_MIN_RIDGE_BARS, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "peak_ridge_amount_ratio", + diagnostics_out: list | None = None, +) -> pd.Series: + """PIT-safe daily "peak/ridge traded-amount ratio" factor from 1min ``bars``. + + Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, classifies + every visible minute with the SHARED PR-F taxonomy, totals each valid day's peak and + ridge traded amount, and returns the ratio of the trailing-``lookback_days``-VALID-day + SUMS of the two legs. See the module docstring for the LOCKED definition, the report's + ratio-of-sums wording, and the seven pinned choices. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is + strictly per symbol (no cross-symbol leakage). + lookback_days: trailing VALID trading-day window POOLED by both legs (definition). + baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). + baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). + sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). + min_valid_days: minimum valid days in the trailing window for a finite value. + min_classifiable: a day needs at least this many classifiable bars (PR-F). + min_peak_bars: a day needs at least this many TRADABLE peak bars (PINNED lower than + the ridge floor — see the module docstring §6). + min_ridge_bars: a day needs at least this many TRADABLE ridge bars. + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + name: the returned Series name (the factor-panel column name). + diagnostics_out: optional list the per-symbol day-level bar-count frames are + APPENDED to (each carries a ``symbol`` column), so a caller can report the + peak-scarcity distribution. Purely observational — supplying it does not change + the returned factor. + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily factor + value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if baseline_days < 2: + # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is + # defined. + raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") + if baseline_min_obs < 2: + raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") + if sigma_k < 0.0: + raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") + if min_valid_days < 1: + raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") + if min_classifiable < 1: + raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") + if min_peak_bars < 1: + raise ValueError(f"min_peak_bars must be >= 1; got {min_peak_bars!r}.") + if min_ridge_bars < 1: + raise ValueError(f"min_ridge_bars must be >= 1; got {min_ridge_bars!r}.") + if len(bars) == 0: + return empty_factor_series(name) + + # extra_columns=("amount",) is the ONLY difference from the PR-F / PR-H entry points: + # the traded value rides along on the surviving rows, and the truncation / volume guard + # / slot assignment are untouched. + visible = prepare_visible_minute_bars( + bars, decision_time=decision_time, extra_columns=(_AMOUNT,) + ) + if visible.empty: + return empty_factor_series(name) + + collect = diagnostics_out is not None + index_tuples: list[tuple] = [] + values: list[float] = [] + for sym, g in _symbol_frames(visible): + days, vals, diagnostics = _peak_ridge_amount_ratio_for_symbol( + g, + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + lookback_days=lookback_days, + min_valid_days=min_valid_days, + min_classifiable=min_classifiable, + min_peak_bars=min_peak_bars, + min_ridge_bars=min_ridge_bars, + collect_diagnostics=collect, + ) + if collect and diagnostics is not None and not diagnostics.empty: + frame = diagnostics.copy() + frame[SYMBOL_LEVEL] = sym + diagnostics_out.append(frame) + for day, val in zip(days, vals): + index_tuples.append((day, sym)) + values.append(val) + + if not index_tuples: + return empty_factor_series(name) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +class PeakRidgeAmountRatioFactor(Factor): + """Peak/ridge traded-amount ratio factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_peak_ridge_amount_ratio`); it does NO minute work of its + own, mirroring its siblings and the value / financial factors that surface an + enriched column. + + Args: + lookback_days: trailing VALID trading-day window POOLED by both amount legs; part + of the factor DEFINITION (reproduced from the report), not a tuned knob. It + only names the column so a non-default window cannot silently mislabel it. + """ + + name: str = f"peak_ridge_amount_ratio_{PEAK_RIDGE_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = PEAK_RIDGE_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"peak-ridge-amount-ratio lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"peak_ridge_amount_ratio_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=+1 READ FROM THE REPORT, not inferred from data and not merely + argued from semantics: §7.2 states "峰岭成交比因子 RankIC 均值 10.28%，RankICIR + 4.07" — an explicitly POSITIVE RankIC — with a long leg of 16.06%/yr, long-short + 27.13%/yr, IR 2.89, max drawdown 7.88%, 74.3% monthly win rate and every year + positive 2013–2025. The report's §1 taxonomy summary agrees independently ("对于 + 量峰时点，分钟数、成交额类因子更加有效，反映知情交易参与度，为正向因子", with the + ridge leg's amount factors marked negative-alpha), so peak-over-ridge is positive + on both counts. The sign is fixed BEFORE the run (a validated prototype must + reproduce it). NOTE the report is a MONTHLY, market-cap + industry neutral + full-market series on Wind data while our eval cell is CSI500 daily with industry + + size neutral, so the report numbers are a LOOSE reference only (disclosed, never + mislabeled, never written in as an expected value). is_intraday=False by the module + docstring's reasoning: minute INPUT but a DAILY signal traded close-to-close. + min_history_bars=0: the warm-up is DATA-dependent (a value appears once enough + VALID days accumulate), not a fixed leading count — the honest NaN rate is reported + by data_coverage. + + The description spells out the AGGREGATION FORM, which is the one place this + factor's definition departs from its siblings: the report specifies a RATIO OF + 20-DAY SUMS, not the mean of daily ratios that §7.1 (PR-J) specifies. + + D1 declarations (D0 pre-assignment table row 11): adjustment=none — + "``amount`` is traded VALUE in RMB, which no split or dividend + adjustment factor rescales ... there is nothing to cancel" (module + docstring PINNED §4); volume only drives the shared classification, + and no price is ever read (zero price content — one disclaimer covers + both axes). overnight_boundary=none — no raw-price comparison exists. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Peak/ridge traded-amount ratio (Kaiyuan microstructure series #27, " + f"SEVENTH factor 峰岭成交比, §7.2). SAME minute classification as PR-F " + f"volume_peak_count / PR-H peak_interval_kurtosis / PR-I " + f"valley_relative_vwap / PR-J valley_ridge_vwap_ratio (SHARED taxonomy " + f"in factors.compute.minute.primitives, not re-implemented): 1min bars " + f"PIT-truncated at 14:50, a minute is ERUPTIVE if vol > μ + " + f"{VOLUME_PRV_SIGMA_K:g}σ of its SAME-SLOT strictly-prior " + f"{VOLUME_PRV_BASELINE_DAYS}-day baseline; a PEAK is an ISOLATED eruption " + f"and a RIDGE is 'eruptive AND NOT a peak'. This is the first factor of " + f"the loop carrying NO price information: both legs are pure traded VALUE, " + f"so it separates 'only price information survives in this taxonomy' from " + f"'the peak/ridge split itself carries alpha'. AGGREGATION = RATIO OF " + f"SUMS: Σ peak amount over the trailing {self._lookback_days} VALID days " + f"divided by Σ ridge amount over the SAME days — the report's literal " + f"wording ('计算 20 日量峰总成交额与量岭总成交额，二者做比'), which " + f"DIFFERS from §7.1's mean-of-daily-ratios and is followed as written; it " + f"is also the better-behaved estimator, since a single day with a nearly " + f"vanishing ridge amount would dominate a mean of daily ratios. PINNED " + f"choices: (1) PEAK is the numerator, RIDGE the denominator, fixed both by " + f"the name 峰岭 and by the report's stated semantics (informed trading " + f"RELATIVE TO retail participation); (2) the masks are PR-J's exactly, so " + f"valley|peak|ridge stays an exact partition of the classifiable bars and " + f"a VALLEY bar enters NEITHER leg — this factor reads only the two " + f"ERUPTIVE groups; (3) bars with non-finite or non-positive amount are " + f"dropped from both legs and both bar counts (guard applied at summation " + f"only, so PR-F's baseline is untouched); volume is deliberately NOT in " + f"the guard because this factor never divides by volume; (4) RAW " + f"unadjusted amounts are exactly correct: traded VALUE in RMB is not " + f"rescaled by any split/dividend adjustment factor, so unlike PR-L there " + f"is no ex-date caveat and unlike PR-I/PR-J no within-day cancellation " + f"argument is even needed; (5) DEVIATION FROM THE REPORT, disclosed: both " + f"legs span the PIT-VISIBLE window 09:31-14:50 only, not the full session " + f"— reading the close would be lookahead at our 14:50 decision time; (6) a " + f"day is VALID iff it has >= {VOLUME_PRV_MIN_CLASSIFIABLE} classifiable " + f"bars AND >= {PEAK_RIDGE_MIN_PEAK_BARS} TRADABLE peak bars AND >= " + f"{PEAK_RIDGE_MIN_RIDGE_BARS} TRADABLE ridge bars (both counted AFTER the " + f"guard) AND positive amount in both legs — the PEAK floor is deliberately " + f"LOWER, the REVERSE of PR-J's asymmetry, because a peak must erupt AND be " + f"ISOLATED, making peaks the scarcer leg of THIS pair; the realized " + f"peak-bar distribution, the day-validity rate and the counterfactual " + f"valid-day count at a peak floor of 10 are all REPORTED by the runner " + f"rather than left implicit; NaN below {VOLUME_PRV_MIN_VALID_DAYS} valid " + f"days. Derived from 1min bars but a DAILY signal traded close-to-close." + ), + expected_ic_sign=1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from: volume drives + # the SHARED classification, amount is the factor's whole quantity. Declared + # for honest provenance disclosure (data_coverage lists them); the daily panel + # surfaces the pre-aggregated column itself. + input_fields=("volume", "amount"), + requires=_minute_requires("volume", "amount"), + adjustment="none", + overnight_boundary="none", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily peak/ridge amount-ratio column off ``panel``. + + The runner runs ``compute_peak_ridge_amount_ratio`` per symbol on the minute cache + upstream and joins the result as ``self.name``; here we only surface it, so this + factor does no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"PeakRidgeAmountRatioFactor needs the pre-aggregated '{self.name}' " + f"column on the panel (produced upstream by " + f"compute_peak_ridge_amount_ratio and joined by the runner); panel has " + f"{list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "DIAGNOSTIC_COLUMNS", + "PEAK_RIDGE_LOOKBACK_DAYS", + "PEAK_RIDGE_MIN_PEAK_BARS", + "PEAK_RIDGE_MIN_RIDGE_BARS", + "PeakRidgeAmountRatioFactor", + "compute_peak_ridge_amount_ratio", + "peak_ridge_amount_by_day", +] diff --git a/factors/compute/minute/primitives.py b/factors/compute/minute/primitives.py new file mode 100644 index 0000000..14523d8 --- /dev/null +++ b/factors/compute/minute/primitives.py @@ -0,0 +1,405 @@ +"""Shared minute-factor primitives (D2, design v3.2 §3.2). + +The ONE home of the machinery every minute factor shares: + +* ``prepare_visible_minute_bars`` + ``peak_mask_for_symbol`` — the peak/ridge/ + valley taxonomy, moved house VERBATIM from ``data.clean.intraday_volume_prv`` + (it was already author-once there; the move is relocation, not repair — R1); +* ``visible_minute_frame`` / ``guarded_amplitude`` / ``add_bar_end_ns`` — the + PIT front half + amplitude price guard of the non-peak factor family + (jump / ideal-amplitude / amp-anomaly / amp-cut share these blocks); +* ``rolling_valid_days`` / ``pooled_trailing_reduce`` — the two trailing-window + forms (rolling over VALID days only, and pool-then-reduce over trailing + days), each preserving the factors' exact float operation order; +* ``positive_trade_mask`` — the valley/ridge family's trade guard for the + ``Σamount/Σvolume`` VWAP aggregation identity; +* ``symbol_frames`` / ``empty_factor_series`` — the per-symbol isolation split + and the schema-shaped empty output. + +Per-factor DEFINITION parameters (``min_valley_bars=20`` / ``min_ridge_bars=10`` +/ "counted AFTER the guard" / …) are pre-registered definitions and stay +explicit parameters of these primitives — parameterized, never unified (§〇 +总原则: the refactor changes implementations, not definitions). + +Layering: imports ``data.clean.intraday_schema`` (real data-layer code) and +numpy/pandas only. It must NEVER import ``data.clean.intraday_aggregate`` — +that module re-exports the migrated MMP/jump math FROM ``factors.compute. +minute``, so an import back would be a genuine cycle. + +Latency note (§八, load-bearing): this layer exists so the 11 factors share one +bars read + one normalization + one classification (measured ≈17.6s shared vs +≈58s naive per-factor re-reads on the CSI500 tail window); the shared-read +orchestration itself arrives with the D4 materializer. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from typing import Callable + +import numpy as np +import pandas as pd + +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, +) + +# --------------------------------------------------------------------------- # +# Peak/ridge/valley classification constants (factor DEFINITION constants, +# pinned interpretations of the Kaiyuan series-27 report; NOT tuned knobs). +# Moved from data/clean/intraday_volume_prv.py together with the taxonomy — +# they are the classification's parameters, shared by the whole peak family. +# --------------------------------------------------------------------------- # +VOLUME_PRV_BASELINE_DAYS = 20 # strictly-prior same-slot baseline window (trading days) +VOLUME_PRV_BASELINE_MIN_OBS = 10 # min same-slot obs for a classifiable bar +VOLUME_PRV_SIGMA_K = 1.0 # eruptive threshold multiplier k in vol > μ + k*σ +VOLUME_PRV_MIN_VALID_DAYS = 10 # min valid days in a trailing window for a finite value +VOLUME_PRV_MIN_CLASSIFIABLE = 100 # a day is valid iff it has >= this many classifiable bars + +# The "strictly-next minute" test in seconds: two bars are same-session 1-minute +# neighbours iff their bar_end gap is EXACTLY 60s. The lunch break (11:30 -> 13:01), +# the session close and any missing minute all differ, so they are not neighbours. +ONE_MINUTE_SECONDS = 60.0 + +# The columns every peak-family factor needs. ``prepare_visible_minute_bars`` always +# emits EXACTLY these (plus the derived ``trade_date`` / ``slot``); anything else a +# caller needs is opt-in via ``extra_columns``, so the default output stays byte-identical +# for the callers that predate the option. +_BASE_VISIBLE_COLUMNS = [SYMBOL_LEVEL, "bar_end", "available_time", "volume"] + + +def empty_factor_series(name: str) -> pd.Series: + """Schema-shaped empty ``MultiIndex(date, symbol)`` factor Series.""" + index = pd.MultiIndex.from_arrays( + [pd.DatetimeIndex([]), pd.Index([], dtype=object)], + names=DAILY_INDEX_NAMES, + ) + return pd.Series([], index=index, dtype=float, name=name) + + +def visible_minute_frame( + bars: pd.DataFrame, + *, + columns: Sequence[str], + decision_time: str = DEFAULT_DECISION_TIME, +) -> pd.DataFrame: + """PIT front half of the non-peak minute factors (jump/amplitude/anomaly/cut). + + Selects ``symbol`` / ``bar_end`` / ``available_time`` plus ``columns`` off + the normalized bars, derives ``trade_date``, and keeps ONLY the bars whose + ``available_time`` is at or before their own ``trade_date + decision_time`` + (per-bar timestamps FIRST — history days and the signal day are truncated + identically, and a post-cutoff bar can never leak into a decision). + + Returns a fresh frame (possibly empty). Pure: never mutates ``bars``. + """ + work = bars.reset_index()[ + [SYMBOL_LEVEL, "bar_end", "available_time", *columns] + ].copy() + work["trade_date"] = work["bar_end"].dt.normalize() + cutoff = work["trade_date"] + pd.Timedelta(decision_time) + return work.loc[work["available_time"] <= cutoff].copy() + + +def guarded_amplitude(visible: pd.DataFrame) -> pd.DataFrame: + """Amplitude price guard + per-bar amplitude for the amplitude family. + + Drops every bar unless ``low > 0`` and ``high >= low`` (a bar failing the + guard has no meaningful amplitude), then adds ``amp = high/low - 1`` on the + survivors. Returns a fresh frame (possibly empty, in which case no ``amp`` + column is added). Pure: never mutates ``visible``. + """ + low = visible["low"].to_numpy(dtype=float) + high = visible["high"].to_numpy(dtype=float) + guard = (low > 0.0) & (high >= low) + out = visible.loc[guard].copy() + if out.empty: + return out + out["amp"] = out["high"].to_numpy(dtype=float) / out["low"].to_numpy( + dtype=float + ) - 1.0 + return out + + +def add_bar_end_ns(frame: pd.DataFrame) -> pd.DataFrame: + """Add ``bar_end_ns``: int64 nanoseconds for a deterministic lexsort tie-break. + + (The view via ``datetime64[ns]`` avoids the datetime->int astype + deprecation.) Returns ``frame`` itself with the column added — callers own + the frame (it is a fresh copy from the preparation helpers). + """ + frame["bar_end_ns"] = frame["bar_end"].to_numpy(dtype="datetime64[ns]").astype( + "int64" + ) + return frame + + +def prepare_visible_minute_bars( + bars: pd.DataFrame, + *, + decision_time: str = DEFAULT_DECISION_TIME, + extra_columns: Sequence[str] = (), +) -> pd.DataFrame: + """PIT-truncate ``bars`` at ``decision_time`` and add ``trade_date`` / ``slot``. + + The shared front half of every peak-taxonomy factor: keep only the bars whose + ``available_time`` is at or before their own ``trade_date + decision_time`` (so + history days and the signal day are truncated identically), drop non-finite / + negative volumes (invalid data that would poison the same-slot μ/σ), and add the + minute-of-day ``slot`` that aligns the SAME time-of-day across days. + + ``bars`` must ALREADY be schema-validated by the caller + (:func:`data.clean.intraday_schema.validate_intraday_bars`) — this helper is the + post-validation preparation step, kept separate so the entry points keep their own + validate-then-check-params ordering. + + Args: + bars: normalized 1min bars, ``MultiIndex(time, symbol)``; one or many symbols. + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + extra_columns: additional ``bars`` columns to carry through, appended AFTER the + base columns. The truncation and the volume guard are unaffected — the extra + values merely ride along on the surviving rows — so the default ``()`` keeps + the output byte-identical for callers that do not need them (PR-F / PR-H). + PR-I passes ``("amount",)`` to compute a volume-weighted price. + + Returns: + A fresh ``RangeIndex`` frame with columns ``symbol`` / ``bar_end`` / + ``available_time`` / ``volume`` / ``trade_date`` / ``slot`` plus any + ``extra_columns`` (possibly empty). Pure: never mutates ``bars``. + """ + extra = list(extra_columns) + unknown = [c for c in extra if c not in bars.columns] + if unknown: + raise ValueError( + f"prepare_visible_minute_bars extra_columns not present on bars: {unknown}; " + f"bars has {list(bars.columns)}." + ) + clashing = [c for c in extra if c in _BASE_VISIBLE_COLUMNS] + if clashing: + raise ValueError( + f"prepare_visible_minute_bars extra_columns are already emitted: {clashing}." + ) + work = bars.reset_index()[_BASE_VISIBLE_COLUMNS + extra].copy() + work["trade_date"] = work["bar_end"].dt.normalize() + # PIT truncation FIRST (per-bar timestamps): each bar's cutoff is its own + # trade_date + decision_time, so every day is truncated to [open, cutoff]. + cutoff = work["trade_date"] + pd.Timedelta(decision_time) + visible = work.loc[work["available_time"] <= cutoff].copy() + if visible.empty: + return visible + + # Guard bad volume BEFORE anything else: a non-finite / negative volume is invalid + # data that would poison the same-slot μ/σ. (Split-day magnitude jumps are NOT + # corrected — disclosed, matching the report's Wind treatment.) + vol = visible["volume"].to_numpy(dtype=float) + visible = visible.loc[np.isfinite(vol) & (vol >= 0.0)].copy() + if visible.empty: + return visible + + # slot = minute-of-day (minutes since midnight); bars are minute-aligned so this is + # exact and aligns the SAME time-of-day across days for the same-slot baseline. + visible["slot"] = ( + (visible["bar_end"] - visible["trade_date"]) // pd.Timedelta(minutes=1) + ).astype(int) + return visible + + +def peak_mask_for_symbol( + g: pd.DataFrame, + *, + baseline_days: int = VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, + sigma_k: float = VOLUME_PRV_SIGMA_K, +) -> pd.DataFrame: + """Per-minute PEAK / classifiable mask for ONE symbol's PIT-visible bars. + + This is THE volume-peak identification of the report (§2), shared by every factor in + the peak family so the taxonomy is defined in exactly one place: same-slot + strictly-prior μ/σ baseline -> eruptive vs mild -> a peak is an eruptive minute whose + both 1-minute same-session neighbours are mild. + + ``g`` holds columns ``trade_date`` / ``slot`` / ``bar_end`` / ``volume`` for a + SINGLE symbol (a fresh ``RangeIndex``, as produced by + :func:`prepare_visible_minute_bars`). The same-slot STRICTLY-PRIOR baseline is + computed on a day x slot pivot (rolling over the day axis then ``shift(1)``), so no + per-``(day, slot)`` python loop is needed and the current day never enters its own + baseline. Peak detection is done back in long form with a within-day 60s-gap + neighbour test (which enforces "same continuous session" and "the 1-minute + neighbour" at once). No cross-symbol leakage — ``g`` is one symbol's slice — and no + cross-day leakage — the baseline is strictly prior and the neighbour test never + crosses a day boundary (grouped by ``trade_date``). + + Returns: + ``g`` sorted by ``(trade_date, bar_end)`` on a fresh ``RangeIndex`` with the + added boolean columns ``classifiable`` (a strictly-prior baseline existed), + ``valley`` (classifiable and NOT eruptive — the report's 量谷, what PR-I prices), + ``peak`` and ``ridge`` (eruptive but NOT an isolated peak — the report's 量岭, + what PR-J prices). The three masks PARTITION the classifiable bars exactly: + ``valley`` holds the mild ones and ``peak`` / ``ridge`` split the eruptive ones, + so ``valley | peak | ridge == classifiable`` and no two overlap. Pure: never + mutates ``g``. + """ + # day x slot volume matrix: rows are the symbol's trading days, columns are the + # minute-of-day slots; a missing (day, slot) cell is NaN (no bar that minute). + v = g.pivot(index="trade_date", columns="slot", values="volume") + v = v.sort_index().sort_index(axis=1) + + # Strictly-prior same-slot baseline: rolling over the DAY axis (per slot column), + # requiring >= baseline_min_obs actual same-slot observations, THEN shift(1) so + # day t uses days t-baseline_days .. t-1 only (never day t itself). + roll = v.rolling(baseline_days, min_periods=baseline_min_obs) + mu = roll.mean().shift(1) + sigma = roll.std().shift(1) # ddof=1 (pandas rolling default) + thr = mu + sigma_k * sigma + + thr_long = thr.reset_index().melt( + id_vars="trade_date", var_name="slot", value_name="thr" + ) + # melt can hand back the slot column labels as object dtype; keep it int so the + # merge below matches g's int ``slot`` (an object/int mismatch would silently miss). + thr_long["slot"] = thr_long["slot"].astype(int) + work = g.merge(thr_long, on=["trade_date", "slot"], how="left") + work = work.sort_values(["trade_date", "bar_end"], kind="mergesort").reset_index( + drop=True + ) + + vol = work["volume"].to_numpy(dtype=float) + thr_arr = work["thr"].to_numpy(dtype=float) + # Classifiable iff the strictly-prior baseline is finite (>= baseline_min_obs obs); + # eruptive iff strictly above μ + k*σ; mild is any other classifiable bar. + classifiable = np.isfinite(thr_arr) + eruptive = classifiable & (vol > thr_arr) + mild = classifiable & ~eruptive + work["classifiable"] = classifiable + # VALLEY (量谷 in the report's peak/ridge/valley taxonomy) == MILD: a classifiable, + # non-eruptive minute. Exposed as a first-class boolean so the valley-PRICE family + # (PR-I) consumes THE SAME classification instead of re-deriving it and drifting. + # Purely additive — ``classifiable`` and ``peak`` below are computed exactly as + # before and no consumer of this frame reads columns positionally. + work["valley"] = mild + # Carry mild as 0/1 so the within-day neighbour shift stays numeric (a shifted bool + # column would go through an object-dtype fillna and warn). + work["mild_i"] = mild.astype(np.int8) + + by_day = work.groupby("trade_date", sort=False) + prev_end = by_day["bar_end"].shift(1) + next_end = by_day["bar_end"].shift(-1) + gap_prev = (work["bar_end"] - prev_end).dt.total_seconds().to_numpy() + gap_next = (next_end - work["bar_end"]).dt.total_seconds().to_numpy() + prev_mild = by_day["mild_i"].shift(1).fillna(0).to_numpy() > 0 + next_mild = by_day["mild_i"].shift(-1).fillna(0).to_numpy() > 0 + # A peak is an eruptive bar whose BOTH 1-minute neighbours (exactly 60s away, so + # same session) exist and are mild. Missing / lunch-break / cutoff neighbours fail + # the 60s gap; eruptive neighbours (ridge) and unclassifiable ones fail the mild + # test — all correctly block the peak. + peak = ( + eruptive + & (gap_prev == ONE_MINUTE_SECONDS) + & prev_mild + & (gap_next == ONE_MINUTE_SECONDS) + & next_mild + ) + work["peak"] = peak + # RIDGE (量岭) == eruptive AND NOT an isolated peak. Exposed as a first-class boolean + # so the ridge-PRICE family (PR-J) consumes THE SAME classification instead of + # re-deriving it and drifting. Purely additive: ``classifiable`` / ``valley`` / + # ``peak`` above are computed exactly as before and no consumer of this frame reads + # columns positionally. + # + # PINNED, and wider than the literal "eruptive next to an eruptive": an eruptive bar + # is a ridge whenever its isolation is NOT PROVABLE, which also covers the + # session-boundary bars (a neighbour missing across the lunch break / the cutoff / + # a gap) and the bars whose neighbour is unclassifiable. That is the exact complement + # of PR-F's deliberately conservative peak rule, so the taxonomy stays a clean + # partition (valley | peak | ridge == classifiable) with no bar silently dropped. + work["ridge"] = eruptive & ~peak + return work + + +def positive_trade_mask(volume: np.ndarray, amount: np.ndarray) -> np.ndarray: + """The valley/ridge family's POSITIVE-TRADE guard. + + A bar with non-finite or non-positive ``volume`` OR ``amount`` carries no + price information (``amount/volume`` is meaningless or degenerate), so it is + dropped from BOTH sums of a ``Σamount/Σvolume`` VWAP. Applied at the + summation step only, never before classification — the same-slot baseline + must stay bit-identical across the family. + """ + return np.isfinite(volume) & (volume > 0.0) & np.isfinite(amount) & (amount > 0.0) + + +def rolling_valid_days( + obj: pd.Series | pd.DataFrame, *, lookback_days: int, min_valid_days: int +): + """Trailing window over VALID days only (the peak family's reduction form). + + ``obj`` is indexed by ``trade_date`` and carries ONLY the valid days + (invalid days are ABSENT, not NaN, so they never occupy a window slot). + Returns the pandas ``Rolling`` object over the ascending-sorted days with + ``min_periods=min_valid_days`` — the caller applies its own reduction + (``.mean()`` / ``.sum()``), keeping each factor's float operation order + exactly as before the D2 move. + """ + return obj.sort_index().rolling(lookback_days, min_periods=min_valid_days) + + +def pooled_trailing_reduce( + day_channels: Sequence[Sequence[np.ndarray]], + *, + lookback_days: int, + reducer: Callable[..., float], +) -> list[float]: + """Pool-then-reduce over trailing days (the amplitude family's form). + + ``day_channels`` holds one sequence per channel; each sequence has one + ``np.ndarray`` per trading day, all channels aligned on the same day axis. + For every day ``j`` the trailing ``lookback_days`` days (including ``j``) + of each channel are concatenated into ONE pooled array and passed to + ``reducer(*pooled_channels) -> float`` — exactly the original per-factor + loop, so the float operation order is unchanged. + """ + if not day_channels: + return [] + n_days = len(day_channels[0]) + values: list[float] = [] + for j in range(n_days): + lo = max(0, j - lookback_days + 1) + pooled = tuple( + np.concatenate(list(channel[lo : j + 1])) for channel in day_channels + ) + values.append(reducer(*pooled)) + return values + + +def symbol_frames(visible: pd.DataFrame) -> Iterator[tuple[str, pd.DataFrame]]: + """Split the visible bars into ONE FRAME PER SYMBOL, in sorted symbol order. + + The whole cross-symbol isolation guarantee lives here: every downstream step (the + same-slot baseline, the classification, the per-day legs, the trailing window) runs + on a single symbol's rows, so one symbol's bars can never reach another's factor + value. + """ + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + yield str(sym), g.reset_index(drop=True) + + +__all__ = [ + "ONE_MINUTE_SECONDS", + "VOLUME_PRV_BASELINE_DAYS", + "VOLUME_PRV_BASELINE_MIN_OBS", + "VOLUME_PRV_MIN_CLASSIFIABLE", + "VOLUME_PRV_MIN_VALID_DAYS", + "VOLUME_PRV_SIGMA_K", + "add_bar_end_ns", + "empty_factor_series", + "guarded_amplitude", + "peak_mask_for_symbol", + "pooled_trailing_reduce", + "positive_trade_mask", + "prepare_visible_minute_bars", + "rolling_valid_days", + "symbol_frames", + "visible_minute_frame", +] diff --git a/factors/compute/minute/ridge_minute_return.py b/factors/compute/minute/ridge_minute_return.py new file mode 100644 index 0000000..0739569 --- /dev/null +++ b/factors/compute/minute/ridge_minute_return.py @@ -0,0 +1,522 @@ +"""RIDGE MINUTE-RETURN factor (PR-K): math + surface (D2). + +Reproduces the FIFTH factor of the Kaiyuan market-microstructure series #27 (开源证券 +《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, +§3): "计算过去 20 日量岭时点的累计收益作为量岭分钟收益因子，衡量个人投资者交易的收益贡献, +个人投资者交易的过度反应，导致量岭分钟收益因子为显著负向因子". + +Same MACHINE as PR-F / PR-H / PR-I / PR-J — what changes is the STATISTIC. The four prior +reproductions from this report covered a COUNT (weak), a TIMING moment (null) and two +PRICE LEVELS (both passed). This is the RETURN family, a fourth statistic type, and the +report's ONLY NEGATIVE peak/ridge/valley factor — so it also tests whether the SIGN +transfers along with the family. + +The classification is the SHARED taxonomy in :mod:`factors.compute.minute.primitives` +(``prepare_visible_minute_bars`` + ``peak_mask_for_symbol``, whose ``ridge`` column PR-J +already exposed) — same same-slot strictly-prior μ+kσ eruptive test, same classifiable +rule, same valid-day floor. Nothing about the taxonomy is re-implemented or modified here, +so the family can never drift apart. + +PINNED choices (deliberate and DISCLOSED, not tuned knobs; the report is silent on every +one of them, and each is reproduced on the factor spec so a reader sees what was assumed): + + 1. THE RIDGE MASK IS ``eruptive & ~peak`` (PR-J's, unchanged). A RIDGE (量岭) is an + eruptive minute that is NOT an isolated peak — wider than the literal "eruptive next + to an eruptive", because it also covers session-boundary eruptions and eruptions + whose neighbour is unclassifiable. It is the exact COMPLEMENT of PR-F's conservative + peak test, so ``valley | peak | ridge == classifiable`` stays an exact partition and + an isolated PEAK's return is counted on NEITHER side. + 2. THE MINUTE RETURN IS ``close_t / close_{t-1} - 1`` WITH A WITHIN-DAY LAG. The + predecessor is the previous VISIBLE bar of the SAME trade date, so each day's FIRST + visible bar has no return and never enters the sum. The lag deliberately does NOT + require exact 60s adjacency: a bar that opens a new session block (in practice 13:01, + after the lunch break) returns against the last bar before the gap, which is a + genuine price change of the stock over that interval. Dropping such bars instead + would silently discard the post-lunch minute whenever it is a ridge. The obvious + selection worry is defused by the taxonomy itself: the 13:01 slot is compared against + its OWN same-slot history, so a systematically busy post-lunch minute is not + systematically eruptive. + 3. RAW (UNADJUSTED) CLOSES. The cached minute bars are unadjusted, and that is CORRECT + here: a split/dividend adjustment factor is constant WITHIN a day, so it cancels + exactly in ``close_t / close_{t-1}``; and because the within-day lag already excludes + each day's first bar, no return ever straddles an ex-date boundary. (Same reasoning + PR-I / PR-J's ratios, PR-D's amplitude and I5b's price-limit checks rely on.) + 4. POSITIVE-CLOSE GUARD. A return is formed only when BOTH closes are finite and + strictly positive; otherwise the bar contributes nothing and is not counted towards + the ridge floor. The guard runs at the return step only — never before classification + — because the shared same-slot μ/σ baseline must stay bit-identical. + 5. THE DAILY AGGREGATE IS A SIMPLE SUM ``s_day = Σ r_t`` over the selected ridge bars, + NOT ``Π(1+r_t) - 1``. The report says only "累计收益" (cumulative return). Ridge + minutes are NON-CONTIGUOUS within the day — they are scattered eruptive moments, not + a held position — so a holding-period/compounding reading does not apply to them; and + at minute scale the two conventions differ negligibly anyway. The choice is therefore + the simple sum, disclosed rather than silently equated to the report's wording. + 6. THE TRAILING AGGREGATE IS ALSO A SUM. "过去 20 日累计" is read as the sum of the daily + sums over the trailing window, i.e. the factor accumulates across days rather than + averaging (which is what PR-J's ratio did). + 7. A RIDGE-BAR FLOOR OF 10, counted AFTER the return guard. Ridge bars are STRUCTURALLY + scarce: a minute must erupt (a minority event by construction, since the threshold is + its own strictly-prior same-slot μ+σ) AND fail the isolation test. This is the SAME + floor PR-J pinned for its ridge leg, so the two runs' coverage is directly + comparable, and the realized ridge-bar distribution plus the day-validity rate are + REPORTED by the runner rather than left implicit. + 8. BOTH THE SUM AND THE COUNT COVER THE PIT-VISIBLE WINDOW ONLY (09:31–14:50), not the + full session. A NECESSARY DEVIATION from the report, which uses the whole day: the + standing 14:50 decision cutoff truncates history days and the signal day identically, + and reading the closing auction would be lookahead at our decision time. + +Factor value: ``ridge_minute_return_20`` = the SUM of ``s_day`` over the symbol's most +recent ``lookback_days`` (=20) VALID trading days INCLUDING ``d``. A day is VALID iff it +has at least ``min_classifiable`` (=100) classifiable bars (PR-F's gate, unchanged) and at +least ``min_ridge_bars`` (=10) ridge bars carrying a valid return. Fewer than +``min_valid_days`` (=10) valid days in the trailing window -> NaN (honest missing; the +runner discloses the coverage). Values are emitted only on valid days. + +Pre-registered sign = -1. The report's full-market RankIC is -6.29% / RankICIR -3.55 (long +leg 7.47%/yr, long-short 14.98%/yr, IR 1.73, max drawdown 13.84%, monthly win rate 70.3%); +it gives NO CSI500 sub-domain figure for this factor, so none is quoted. Semantics per the +report: ridge minutes are retail follow-the-crowd trading, and their return contribution +measures that crowd's OVER-REACTION — the higher the accumulated ridge-minute return, the +more over-extended the stock and the worse it performs going forward. NOTE the report is a +MONTHLY, market-cap + industry neutral full-market series on Wind data while our eval cell +is CSI500 daily with industry + size neutralization, so its numbers are a LOOSE reference +only (disclosed, never mislabeled, never written in as an expected value). + +The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future bar +(invariant #1); it is a DAILY signal traded close-to-close from d+1. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + VOLUME_PRV_BASELINE_DAYS, + VOLUME_PRV_BASELINE_MIN_OBS, + VOLUME_PRV_MIN_CLASSIFIABLE, + VOLUME_PRV_MIN_VALID_DAYS, + VOLUME_PRV_SIGMA_K, + empty_factor_series, + peak_mask_for_symbol, + prepare_visible_minute_bars, + rolling_valid_days, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). +# The classification constants live with the taxonomy in ``primitives``. +RIDGE_RETURN_LOOKBACK_DAYS = 20 # trailing VALID trading-day SUM window, includes d +# The SAME scarcity floor PR-J pinned for its ridge leg (module docstring §7), so the two +# runs' ridge coverage is directly comparable. +RIDGE_RETURN_MIN_RIDGE_BARS = 10 # min RETURN-CARRYING ridge bars for a valid day + +# The extra 1min column this family needs on top of the taxonomy's (volume): the close, +# which turns consecutive bars into a minute return. +_CLOSE = "close" + +# Per-day diagnostic columns (the ridge-scarcity disclosure the task card requires). +# ``ridge_bars`` is every ridge minute; ``ridge_return_bars`` is the subset that carries a +# valid return — the one the floor actually gates on — so the attrition is visible. +DIAGNOSTIC_COLUMNS = ("classifiable_bars", "ridge_bars", "ridge_return_bars", "valid") + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def ridge_minute_return_by_day( + work: pd.DataFrame, + *, + min_ridge_bars: int = RIDGE_RETURN_MIN_RIDGE_BARS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + with_diagnostics: bool = False, +) -> pd.Series | tuple[pd.Series, pd.DataFrame]: + """Daily summed ridge-minute return for ONE symbol, on VALID days only. + + ``work`` is one symbol's frame as returned by + :func:`~factors.compute.minute.primitives.peak_mask_for_symbol`, which must have been + built from bars prepared with ``extra_columns=("close",)`` so the close is available + alongside the ``ridge`` / ``classifiable`` masks. ``peak_mask_for_symbol`` returns the + rows sorted by ``(trade_date, bar_end)``, which is exactly the order the WITHIN-DAY lag + needs. + + The minute return is ``close_t / close_{t-1} - 1`` against the previous VISIBLE bar of + the SAME day (PINNED §2), guarded so both closes are finite and strictly positive + (§4), and the day's value is the SIMPLE SUM of those returns over the ridge bars (§5) + — an isolated PEAK's return is never included (§1). + + Args: + work: one symbol's classified minute frame (see above). + min_ridge_bars: minimum ridge bars CARRYING A VALID RETURN for a valid day. + min_classifiable: minimum classifiable bars for a valid day (PR-F's gate). + with_diagnostics: also return the per-day bar-count frame, so the caller can + REPORT the ridge-scarcity distribution instead of only gating on it. + + Returns: + Series indexed by ``trade_date`` (ascending) holding the summed ridge-minute + return for the days that clear both gates — invalid days are ABSENT, not NaN, so + they do not occupy a slot in the caller's trailing window (the same rule PR-F / + PR-H / PR-I / PR-J use). With ``with_diagnostics=True``, a ``(daily, diagnostics)`` + pair where ``diagnostics`` is indexed by EVERY day present in ``work`` and carries + :data:`DIAGNOSTIC_COLUMNS`. + """ + close = work[_CLOSE].to_numpy(dtype=float) + # WITHIN-DAY lag: the previous visible bar of the SAME trade date. grouping by + # trade_date is what stops a return from ever crossing a day boundary, so each day's + # first visible bar gets a NaN predecessor and drops out below. + prev_close = ( + work.groupby("trade_date", sort=False)[_CLOSE].shift(1).to_numpy(dtype=float) + ) + # Positive-close guard: a non-finite or non-positive close on EITHER side carries no + # usable return. Applied HERE, at the return step, never before classification — + # the shared same-slot baseline must stay bit-identical. + has_return = ( + np.isfinite(close) & (close > 0.0) + & np.isfinite(prev_close) & (prev_close > 0.0) + ) + # Denominator neutralized where the guard failed, so no divide-by-zero / inf is ever + # formed; those entries are discarded by the np.where immediately after. + safe_prev = np.where(has_return, prev_close, 1.0) + ret = np.where(has_return, close / safe_prev - 1.0, 0.0) + + ridge = work["ridge"].to_numpy(dtype=bool) + ridge_return = ridge & has_return + + per_bar = pd.DataFrame( + { + "trade_date": work["trade_date"].to_numpy(), + # SIMPLE SUM of the selected returns (PINNED §5) — not Π(1+r)-1. + "ridge_return_sum": np.where(ridge_return, ret, 0.0), + "ridge_bars": ridge.astype(np.int64), + "ridge_return_bars": ridge_return.astype(np.int64), + "classifiable_bars": work["classifiable"] + .to_numpy(dtype=bool) + .astype(np.int64), + } + ) + agg = per_bar.groupby("trade_date", sort=True).sum() + + # Two validity gates (§ module docstring / task card §1.5): PR-F's classifiable floor + # (unchanged) and enough RETURN-CARRYING ridge bars. + valid = (agg["classifiable_bars"] >= min_classifiable) & ( + agg["ridge_return_bars"] >= min_ridge_bars + ) + ok = agg.loc[valid] + if ok.empty: + daily = pd.Series([], index=pd.DatetimeIndex([], name="trade_date"), dtype=float) + else: + daily = ok["ridge_return_sum"].astype(float) + + if not with_diagnostics: + return daily + diagnostics = agg[["classifiable_bars", "ridge_bars", "ridge_return_bars"]].copy() + diagnostics["valid"] = valid + return daily, diagnostics + + +def _ridge_return_sum_for_symbol( + g: pd.DataFrame, + *, + baseline_days: int, + baseline_min_obs: int, + sigma_k: float, + lookback_days: int, + min_valid_days: int, + min_classifiable: int, + min_ridge_bars: int, + collect_diagnostics: bool = False, +) -> tuple[list[pd.Timestamp], list[float], pd.DataFrame | None]: + """Daily ridge-minute-return values for ONE symbol from its PIT-visible bars. + + Classifies the minutes with the SHARED :func:`peak_mask_for_symbol`, reduces each valid + day to its summed ridge-minute return, then takes the trailing-``lookback_days``- + valid-day SUM. No cross-symbol leakage (``g`` is one symbol's slice) and no lookahead + (the baseline is strictly prior, the window is trailing). + """ + work = peak_mask_for_symbol( + g, + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + ) + result = ridge_minute_return_by_day( + work, + min_ridge_bars=min_ridge_bars, + min_classifiable=min_classifiable, + with_diagnostics=collect_diagnostics, + ) + if collect_diagnostics: + daily, diagnostics = result + else: + daily, diagnostics = result, None + if daily.empty: + return [], [], diagnostics + + # SUM over the trailing lookback_days VALID days (including d); NaN until + # min_valid_days valid days have accumulated. Emitted only on valid days. + rolled = rolling_valid_days( + daily, lookback_days=lookback_days, min_valid_days=min_valid_days + ).sum() + days = [pd.Timestamp(d).normalize() for d in rolled.index] + return days, list(rolled.to_numpy(dtype=float)), diagnostics + + +def compute_ridge_minute_return( + bars: pd.DataFrame, + *, + lookback_days: int = RIDGE_RETURN_LOOKBACK_DAYS, + baseline_days: int = VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, + sigma_k: float = VOLUME_PRV_SIGMA_K, + min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + min_ridge_bars: int = RIDGE_RETURN_MIN_RIDGE_BARS, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "ridge_minute_return", + diagnostics_out: list | None = None, +) -> pd.Series: + """PIT-safe daily "ridge minute-return" factor from 1min ``bars``. + + Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, classifies + every visible minute with the SHARED PR-F taxonomy, sums each valid day's ridge-minute + returns, and returns the trailing-``lookback_days``-VALID-day SUM of those daily sums. + See the module docstring for the LOCKED definition and the eight pinned choices. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is + strictly per symbol (no cross-symbol leakage). + lookback_days: trailing VALID trading-day window summed (definition). + baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). + baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). + sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). + min_valid_days: minimum valid days in the trailing window for a finite value. + min_classifiable: a day needs at least this many classifiable bars (PR-F). + min_ridge_bars: a day needs at least this many ridge bars CARRYING A VALID RETURN. + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + name: the returned Series name (the factor-panel column name). + diagnostics_out: optional list the per-symbol day-level bar-count frames are + APPENDED to (each carries a ``symbol`` column), so a caller can report the + ridge-scarcity distribution. Purely observational — supplying it does not + change the returned factor. + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily factor + value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if baseline_days < 2: + # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is + # defined. + raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") + if baseline_min_obs < 2: + raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") + if sigma_k < 0.0: + raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") + if min_valid_days < 1: + raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") + if min_classifiable < 1: + raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") + if min_ridge_bars < 1: + raise ValueError(f"min_ridge_bars must be >= 1; got {min_ridge_bars!r}.") + if len(bars) == 0: + return empty_factor_series(name) + + # extra_columns=("close",) is the ONLY difference from the PR-F / PR-H entry points: + # the close rides along on the surviving rows, and the truncation / volume guard / + # slot assignment are untouched. + visible = prepare_visible_minute_bars( + bars, decision_time=decision_time, extra_columns=(_CLOSE,) + ) + if visible.empty: + return empty_factor_series(name) + + collect = diagnostics_out is not None + index_tuples: list[tuple] = [] + values: list[float] = [] + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + days, vals, diagnostics = _ridge_return_sum_for_symbol( + g.reset_index(drop=True), + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + lookback_days=lookback_days, + min_valid_days=min_valid_days, + min_classifiable=min_classifiable, + min_ridge_bars=min_ridge_bars, + collect_diagnostics=collect, + ) + if collect and diagnostics is not None and not diagnostics.empty: + frame = diagnostics.copy() + frame[SYMBOL_LEVEL] = str(sym) + diagnostics_out.append(frame) + for day, val in zip(days, vals): + index_tuples.append((day, str(sym))) + values.append(val) + + if not index_tuples: + return empty_factor_series(name) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +class RidgeMinuteReturnFactor(Factor): + """Ridge minute-return factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_ridge_minute_return`); it does NO minute work of its + own, mirroring its siblings and the value / financial factors that surface an + enriched column. + + Args: + lookback_days: trailing VALID trading-day window summed; part of the factor + DEFINITION (reproduced from the report), not a tuned knob. It only names the + column so a non-default window cannot silently mislabel it. + """ + + name: str = f"ridge_minute_return_{RIDGE_RETURN_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = RIDGE_RETURN_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"ridge-minute-return lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"ridge_minute_return_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=-1: this is the report's ONLY NEGATIVE peak/ridge/valley factor + (full-market RankIC -6.29%, RankICIR -3.55, long leg 7.47%/yr, long-short + 14.98%/yr, IR 1.73, max drawdown 13.84%, monthly win rate 70.3%). The report gives + NO CSI500 sub-domain figure for this factor, so none is quoted here. Semantics per + the report: ridge minutes are retail follow-the-crowd trading and their + accumulated return measures that crowd's OVER-REACTION — the more ridge-minute + return a stock has piled up, the more over-extended it is and the worse it + performs going forward. The sign is fixed BEFORE the run (a validated prototype + must reproduce it). NOTE the report is a MONTHLY, market-cap + industry neutral + full-market series on Wind data while our eval cell is CSI500 daily with industry + + size neutral, so the report numbers are a LOOSE reference only (disclosed, never + mislabeled, never written in as an expected value). is_intraday=False by the + module docstring's reasoning: minute INPUT but a DAILY signal traded + close-to-close. min_history_bars=0: the warm-up is DATA-dependent (a value appears + once enough VALID days accumulate), not a fixed leading count — the honest NaN + rate is reported by data_coverage. + + The description spells out the RELATION TO PR-F..PR-J (same shared classification, + a RETURN statistic instead of a count / timing moment / price level) plus the + pinned choices — above all the SIMPLE-SUM convention, the within-day return lag and + the raw closes, none of which the report specifies. + + D1 declarations (D0 pre-assignment table row 9): adjustment= + returns_invariant — the minute return close_t/close_(t-1) - 1 is a + within-day ratio, and "a split/dividend adjustment factor is constant + WITHIN a day, so it cancels exactly" (module docstring PINNED §3). + overnight_boundary=none — the same pinned choice's own claim, "no + return ever straddles an ex-date boundary" (the within-day lag drops + each day's first visible bar). + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Ridge minute-return (Kaiyuan microstructure series #27, FIFTH factor " + f"量岭分钟收益). SAME minute classification as PR-F volume_peak_count / " + f"PR-H peak_interval_kurtosis / PR-I valley_relative_vwap / PR-J " + f"valley_ridge_vwap_ratio (SHARED taxonomy in factors.compute.minute." + f"primitives, not re-implemented): 1min bars PIT-truncated at 14:50, a " + f"minute is ERUPTIVE if vol > μ + {VOLUME_PRV_SIGMA_K:g}σ of its " + f"SAME-SLOT strictly-prior {VOLUME_PRV_BASELINE_DAYS}-day baseline, and " + f"a RIDGE (量岭) is an eruptive minute that is NOT an isolated peak. NEW " + f"STATISTIC vs PR-F..PR-J: a RETURN rather than a count, a timing moment " + f"or a price level — each day sums the minute returns of its ridge bars " + f"and the factor sums those daily sums over the trailing " + f"{self._lookback_days} VALID days. PINNED choices (the report specifies " + f"none of them): (1) the RIDGE mask is 'eruptive AND NOT an isolated " + f"peak', keeping valley|peak|ridge an exact partition of the classifiable " + f"bars — an isolated PEAK's return is counted on NEITHER side; (2) the " + f"minute return is close_t/close_(t-1) - 1 with a WITHIN-DAY lag against " + f"the previous VISIBLE bar of the same date, so each day's FIRST visible " + f"bar carries no return and no return ever crosses a day boundary; exact " + f"60s adjacency is deliberately NOT required, so a bar opening a new " + f"session block (13:01, after lunch) returns against the last bar before " + f"the gap — a genuine price change rather than a discarded one; (3) RAW " + f"unadjusted closes are correct here because the adjustment factor is " + f"constant within a day and cancels in the ratio, and the within-day lag " + f"already excludes the one bar that could straddle an ex-date; (4) a " + f"return is formed only when both closes are finite and strictly positive " + f"(guard applied at the return step only, so PR-F's baseline is " + f"untouched); (5) the daily aggregate is a SIMPLE SUM Σr, NOT a compound " + f"Π(1+r)-1 — ridge minutes are non-contiguous within the day so a " + f"holding-period reading does not apply, and at minute scale the two " + f"differ negligibly; (6) the trailing aggregate is likewise a SUM across " + f"days; (7) a day is VALID iff it has >= {VOLUME_PRV_MIN_CLASSIFIABLE} " + f"classifiable bars AND >= {RIDGE_RETURN_MIN_RIDGE_BARS} ridge bars " + f"CARRYING A VALID RETURN (counted AFTER the guard) — the same scarcity " + f"floor PR-J pinned for its ridge leg, since a ridge bar must erupt AND " + f"fail the isolation test; the realized ridge-bar distribution and " + f"day-validity rate are REPORTED by the runner rather than left implicit; " + f"(8) DEVIATION FROM THE REPORT, disclosed: everything spans the " + f"PIT-VISIBLE window 09:31-14:50 only, not the full session — reading the " + f"close would be lookahead at our 14:50 decision time. NaN below " + f"{VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived from 1min bars but a " + f"DAILY signal traded close-to-close." + ), + expected_ic_sign=-1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from. Declared for + # honest provenance disclosure (data_coverage lists them); the daily panel + # surfaces the pre-aggregated column itself. + input_fields=("volume", "close"), + requires=_minute_requires("volume", "close"), + adjustment="returns_invariant", + overnight_boundary="none", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily ridge-minute-return column off ``panel``. + + The runner runs ``compute_ridge_minute_return`` per symbol on the minute cache + upstream and joins the result as ``self.name``; here we only surface it, so this + factor does no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"RidgeMinuteReturnFactor needs the pre-aggregated '{self.name}' column " + f"on the panel (produced upstream by compute_ridge_minute_return and " + f"joined by the runner); panel has {list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "DIAGNOSTIC_COLUMNS", + "RIDGE_RETURN_LOOKBACK_DAYS", + "RIDGE_RETURN_MIN_RIDGE_BARS", + "RidgeMinuteReturnFactor", + "compute_ridge_minute_return", + "ridge_minute_return_by_day", +] diff --git a/factors/compute/minute/valley_price_quantile.py b/factors/compute/minute/valley_price_quantile.py new file mode 100644 index 0000000..d344bdd --- /dev/null +++ b/factors/compute/minute/valley_price_quantile.py @@ -0,0 +1,730 @@ +"""VALLEY WEIGHTED-PRICE-QUANTILE factor (PR-L): math + surface (D2). + +Reproduces the SIXTH factor of the Kaiyuan market-microstructure series #27 (开源证券 +《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, +§5): "将『日内最高价、日内最低价、昨日收盘价』三者的最高值与最低值作为区间的价格高低位，并计算 +每日量谷成交量加权价格的相对分位点，将 20 日的分位点均值作为量谷加权价格分位点因子…考虑到因子 +计算中正向暴露 20 日反转因子，我们进一步对该因子做反转中性化处理". + +Same MACHINE as PR-F / PR-H / PR-I / PR-J / PR-K, a new STATISTIC FAMILY. The minute +classification is the SHARED taxonomy in :mod:`factors.compute.minute.primitives` +(``prepare_visible_minute_bars`` + ``peak_mask_for_symbol``) — same same-slot +strictly-prior μ+kσ eruptive test, same classifiable rule. A VALLEY (量谷) is exactly +that taxonomy's ``valley`` column. Where PR-F counted peaks, PR-H measured their timing, +PR-I / PR-J took price RATIOS and PR-K a RETURN, this module measures a price POSITION: +WHERE in the day's price range the valley VWAP sits. + +Definition, per symbol and per panel date ``d`` (a DAILY signal traded close-to-close, +``is_intraday=False`` like all eight precedents): + + 1. PIT truncation (standing authorization): each day keeps only the 1min bars with + ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50). + 2. PRICE RANGE from three prices — the visible day's high, the visible day's low, and + the PREVIOUS trading day's close: ``hi = max(visible high, prev_close)`` and + ``lo = min(visible low, prev_close)``. + 3. VALLEY VWAP = ``Σamount(valley bars) / Σvolume(valley bars)`` (PR-I's identity). + 4. DAILY QUANTILE ``q_day = (valley VWAP - lo) / (hi - lo)``, deliberately NOT clipped. + 5. TRAILING MEAN ``qbar`` over the most recent ``lookback_days`` (=20) VALID days + including ``d``; fewer than ``min_valid_days`` (=10) valid days -> NaN. + 6. REVERSAL NEUTRALIZATION: per panel date, the cross-sectional OLS residual of + ``qbar`` on a 20-day reversal factor. + +PINNED choices (deliberate and DISCLOSED, reproduced on the factor spec): + + 1. PREV_CLOSE IS THE LAST VISIBLE (<=14:50) RAW CLOSE OF THE PREVIOUS TRADING DAY, not + that day's true 15:00 daily close. DEVIATION FROM THE REPORT, which uses the real + previous close. The previous day's 15:00 close is NOT itself a lookahead at a 14:50 + decision on day d — it is already past — so this is not a leakage fix; it is a + SINGLE-VISIBILITY-DEFINITION choice, so that every price entering the factor (the + range, the VWAP, the previous close) comes from one source under one cutoff rather + than mixing a minute-cache window with a daily-close feed. Disclosed, never silently + equated to the report's construction. + 2. NO CLIPPING of ``q_day``. If the valley VWAP falls outside ``[lo, hi]`` the range + construction is wrong, and clipping to [0,1] would hide exactly the defect worth + seeing. Under a correct range a traded price cannot escape it, so an out-of-range + value is a signal to investigate, not a value to sanitize. + 3. THE REVERSAL IS TAKEN AT T-1: ``rev20 = -(close_{d-1}/close_{d-21} - 1)`` on + FRONT-ADJUSTED (qfq) daily closes. The report's naive 20-day reversal uses + ``close_d``, which is 15:00 information and WOULD be a lookahead at our 14:50 + decision time. This is the direct application of the standing authorization that + implicitly-leaking fields be taken from pre-14:50 data. The closes come from the + panel the runner already loads — NO new data source. Front-adjusted closes are + REQUIRED here (unlike the within-day quantities below) because the ratio spans 20 + trading days and would otherwise break across any split or dividend. + 4. RAW (UNADJUSTED) MINUTE PRICES for the range and the VWAP, with ONE DISCLOSED + IMPERFECTION. PR-I/PR-J/PR-K could argue the adjustment factor cancels because both + of their legs sat inside ONE trading day. That argument does NOT fully hold here: + ``prev_close`` crosses the overnight boundary, so on an EX-DIVIDEND / SPLIT date the + previous raw close sits on the OLD (higher) price scale while the day's own bars sit + on the new one. The DIRECTION is therefore one-sided, not symmetric: ``prev_close`` + pulls ``hi`` UP (measured on real cache: 42 top-widening vs 4 bottom-widening among + true ex-dates), inflating the denominator ``hi - lo`` while the valley VWAP and + ``lo`` stay on the new scale — which compresses that day's ``q_day`` toward the LOW + END of an artificially wide range, NOT toward the middle. + MEASURED on the 30-name prototype panel: 15.6% of valid days show ANY range + widening from ``prev_close``, but that is overwhelmingly ordinary overnight gapping, + which is the INTENDED behaviour (the report's range includes the previous close + precisely so a gap counts); only 0.73% of valid days are true ex-dates, and only + ~0.21% are BOTH a true ex-date AND actually range-distorted — before the 20-day mean + dilutes it further. DISCLOSED here rather than silently corrected, because + correcting it would mix an adjusted price into an otherwise raw, single-visibility + construction. + 5. THE RANGE USES A PRICE GUARD, THE VWAP USES A TRADE GUARD. A bar enters the + high/low range if its prices are finite with ``high >= low > 0`` (a zero-volume + minute still quotes a real price); it enters the VWAP only if BOTH ``volume`` and + ``amount`` are finite and strictly positive (PR-I's positive-trade guard). Both + guards run at the summation step only, never before classification, so the shared + same-slot baseline — and therefore all merged factors — stay bit-identical. + 6. DAY VALIDITY needs all of: >= ``min_classifiable`` (=100) classifiable bars (PR-F's + gate, unchanged), >= ``min_valley_bars`` (=20) TRADABLE valley bars (PR-I's floor), + strictly positive valley volume, an available ``prev_close``, and ``hi > lo``. A + symbol's FIRST visible day therefore never produces a value (no previous day). + 7. THE RESIDUALIZATION IS PER DATE ON THE COVERED CROSS-SECTION, mirroring PR-G's + per-date cross-sectional post-processing. A date with fewer than + ``min_cross_section`` (=10) symbols carrying BOTH ``qbar`` and ``rev20`` is all-NaN; + a symbol missing ``rev20`` is dropped from the regression and its residual is NaN + (never a silent zero fill); a degenerate cross-section (zero-variance ``rev20``, + which cannot identify a slope) is likewise NaN rather than an unresidualized + ``qbar`` passed off as neutralized. + 8. EVERYTHING SPANS THE PIT-VISIBLE WINDOW 09:31-14:50 ONLY, while the report uses the + full session. Reading the closing auction would be lookahead at our decision time. + Disclosed, not silently equated. + +Pre-registered sign = +1. The report's full-market RankIC is +6.34% / RankICIR 4.32 +(long leg 13.1%/yr, long-short 20.22%/yr, IR 3.29, max drawdown 10.18%, monthly win rate +80.4%); its CSI500 sub-domain long-short is 11.71% / IR 1.76, the closest comparable to +our eval cell. Semantics per the report: a HIGH position of the calm-minute (valley) +price within the day's range means the informed, unhurried part of the day traded near +the top of the range -> higher future return. + +The value at ``d`` uses only bars at dates <= d and closes at dates <= d-1, so a factor +value never sees a future bar (invariant #1). +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import MARKET_DAILY, STK_MINS_1MIN +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DATE_LEVEL, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + VOLUME_PRV_BASELINE_DAYS, + VOLUME_PRV_BASELINE_MIN_OBS, + VOLUME_PRV_MIN_CLASSIFIABLE, + VOLUME_PRV_MIN_VALID_DAYS, + VOLUME_PRV_SIGMA_K, + empty_factor_series, + peak_mask_for_symbol, + positive_trade_mask, + prepare_visible_minute_bars, + rolling_valid_days, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). +# The classification constants live with the taxonomy in ``primitives``. +VALLEY_QUANTILE_LOOKBACK_DAYS = 20 # trailing VALID trading-day mean window, includes d +VALLEY_QUANTILE_MIN_VALLEY_BARS = 20 # min TRADABLE valley bars for a valid day (PR-I's) +VALLEY_QUANTILE_REVERSAL_DAYS = 20 # span of the reversal factor neutralized against +VALLEY_QUANTILE_MIN_CROSS_SECTION = 10 # min paired cross-section for a finite residual + +# The extra 1min columns this family needs on top of the taxonomy's (volume): the traded +# value for the VWAP, and the prices that form the day's range / the previous close. +_EXTRA_COLUMNS = ("amount", "high", "low", "close") + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def valley_price_quantile_by_day( + work: pd.DataFrame, + *, + min_valley_bars: int = VALLEY_QUANTILE_MIN_VALLEY_BARS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, +) -> pd.Series: + """Daily valley-VWAP price quantile for ONE symbol, on VALID days only. + + ``work`` is one symbol's frame as returned by + :func:`~factors.compute.minute.primitives.peak_mask_for_symbol`, built from bars + prepared with ``extra_columns=("amount", "high", "low", "close")`` so the traded + value and the prices ride alongside the ``valley`` / ``classifiable`` masks. + + Implements pinned choices 1-6 of the module docstring: the range is + ``[min(visible low, prev_close), max(visible high, prev_close)]`` where ``prev_close`` + is the previous trading day's LAST VISIBLE usable close; the valley VWAP is + ``Σamount/Σvolume`` over guarded valley bars; the quantile is NOT clipped. + + Returns: + Series indexed by ``trade_date`` (ascending) holding ``q_day`` for the days that + clear every validity gate — invalid days are ABSENT, not NaN, so they do not + occupy a slot in the caller's trailing window (the rule PR-F..PR-K use). + """ + # Chronological within the symbol so "the day's last visible close" is well defined. + ordered = work.sort_values("bar_end", kind="mergesort") + + vol = ordered["volume"].to_numpy(dtype=float) + amt = ordered["amount"].to_numpy(dtype=float) + high = ordered["high"].to_numpy(dtype=float) + low = ordered["low"].to_numpy(dtype=float) + close = ordered["close"].to_numpy(dtype=float) + + # PINNED §5: a TRADE guard for the VWAP, a PRICE guard for the range. A zero-volume + # minute still quotes a real price, so it may set the day's high/low while + # contributing nothing to the volume-weighted price. + tradable = positive_trade_mask(vol, amt) + priced = np.isfinite(high) & np.isfinite(low) & (low > 0.0) & (high >= low) + valley = ordered["valley"].to_numpy(dtype=bool) & tradable + + per_bar = pd.DataFrame( + { + "trade_date": ordered["trade_date"].to_numpy(), + "valley_amt": np.where(valley, amt, 0.0), + "valley_vol": np.where(valley, vol, 0.0), + "valley_bars": valley.astype(np.int64), + "classifiable_bars": ordered["classifiable"] + .to_numpy(dtype=bool) + .astype(np.int64), + # +/-inf so a day with no priced bar reduces to a non-finite range and fails + # the hi > lo gate rather than fabricating one. + "day_high": np.where(priced, high, -np.inf), + "day_low": np.where(priced, low, np.inf), + } + ) + grouped = per_bar.groupby("trade_date", sort=True) + agg = grouped[ + ["valley_amt", "valley_vol", "valley_bars", "classifiable_bars"] + ].sum() + agg["day_high"] = grouped["day_high"].max() + agg["day_low"] = grouped["day_low"].min() + + # PINNED §1: prev_close = the previous trading day's LAST VISIBLE usable close. Taken + # from the SAME visible (<=14:50) bars as everything else — a post-cutoff bar on the + # previous day can never become it. + usable_close = np.isfinite(close) & (close > 0.0) + closes = pd.DataFrame( + { + "trade_date": ordered["trade_date"].to_numpy()[usable_close], + "close": close[usable_close], + } + ) + last_close = closes.groupby("trade_date", sort=True)["close"].last() + # shift(1) over the symbol's own trading days: day d takes day d-1's last visible + # close, and the FIRST day has none (NaN) -> that day is invalid. + prev_close = last_close.reindex(agg.index).shift(1) + + pc = prev_close.to_numpy(dtype=float) + hi = np.maximum(agg["day_high"].to_numpy(dtype=float), pc) + lo = np.minimum(agg["day_low"].to_numpy(dtype=float), pc) + + valid = ( + (agg["classifiable_bars"].to_numpy() >= min_classifiable) + & (agg["valley_bars"].to_numpy() >= min_valley_bars) + & (agg["valley_vol"].to_numpy(dtype=float) > 0.0) + & np.isfinite(pc) + & np.isfinite(hi) + & np.isfinite(lo) + & (hi > lo) + ) + if not valid.any(): + return pd.Series( + [], index=pd.DatetimeIndex([], name="trade_date"), dtype=float + ) + + valley_vwap = ( + agg["valley_amt"].to_numpy(dtype=float)[valid] + / agg["valley_vol"].to_numpy(dtype=float)[valid] + ) + # PINNED §2: NOT clipped — an out-of-range value must be visible, not sanitized. + q = (valley_vwap - lo[valid]) / (hi[valid] - lo[valid]) + return pd.Series(q, index=agg.index[valid], dtype=float) + + +def _quantile_mean_for_symbol( + g: pd.DataFrame, + *, + baseline_days: int, + baseline_min_obs: int, + sigma_k: float, + lookback_days: int, + min_valid_days: int, + min_classifiable: int, + min_valley_bars: int, +) -> tuple[list[pd.Timestamp], list[float]]: + """Trailing-mean daily quantile for ONE symbol from its PIT-visible bars. + + Classifies the minutes with the SHARED :func:`peak_mask_for_symbol`, reduces each + valid day to its price quantile, then takes the trailing-``lookback_days``-valid-day + mean. No cross-symbol leakage (``g`` is one symbol's slice) and no lookahead (the + baseline is strictly prior, the mean window is trailing, the previous close is the + previous day's). + """ + work = peak_mask_for_symbol( + g, + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + ) + q = valley_price_quantile_by_day( + work, min_valley_bars=min_valley_bars, min_classifiable=min_classifiable + ) + if q.empty: + return [], [] + rolled = rolling_valid_days( + q, lookback_days=lookback_days, min_valid_days=min_valid_days + ).mean() + days = [pd.Timestamp(d).normalize() for d in rolled.index] + return days, list(rolled.to_numpy(dtype=float)) + + +def compute_valley_price_quantile_stats( + bars: pd.DataFrame, + *, + lookback_days: int = VALLEY_QUANTILE_LOOKBACK_DAYS, + baseline_days: int = VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, + sigma_k: float = VOLUME_PRV_SIGMA_K, + min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + min_valley_bars: int = VALLEY_QUANTILE_MIN_VALLEY_BARS, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "valley_price_quantile_raw", +) -> pd.Series: + """RAW trailing-mean price quantile panel (steps 1-5; NO neutralization yet). + + Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, + classifies every visible minute with the SHARED PR-F taxonomy, computes each valid + day's quantile of the valley VWAP within the prev-close-extended price range, and + returns the trailing-``lookback_days``-VALID-day mean of that quantile. + + The REVERSAL NEUTRALIZATION (step 6) is deliberately NOT applied here — it needs the + full-universe panel plus daily closes and is done by :func:`residualize_on_reversal`. + The returned values are therefore the RAW ``qbar``, which is exactly what the + prototype's before/after neutralization comparison needs. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is + strictly per symbol (no cross-symbol leakage). + lookback_days: trailing VALID trading-day window averaged (definition). + baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). + baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). + sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). + min_valid_days: minimum valid days in the trailing window for a finite value. + min_classifiable: a day needs at least this many classifiable bars (PR-F). + min_valley_bars: a day needs at least this many TRADABLE valley bars. + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + name: the returned Series name. + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the raw + trailing-mean quantile, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if baseline_days < 2: + # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is + # defined. + raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") + if baseline_min_obs < 2: + raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") + if sigma_k < 0.0: + raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") + if min_valid_days < 1: + raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") + if min_classifiable < 1: + raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") + if min_valley_bars < 1: + raise ValueError(f"min_valley_bars must be >= 1; got {min_valley_bars!r}.") + if len(bars) == 0: + return empty_factor_series(name) + + visible = prepare_visible_minute_bars( + bars, decision_time=decision_time, extra_columns=_EXTRA_COLUMNS + ) + if visible.empty: + return empty_factor_series(name) + + index_tuples: list[tuple] = [] + values: list[float] = [] + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + days, vals = _quantile_mean_for_symbol( + g.reset_index(drop=True), + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + lookback_days=lookback_days, + min_valid_days=min_valid_days, + min_classifiable=min_classifiable, + min_valley_bars=min_valley_bars, + ) + for day, val in zip(days, vals): + index_tuples.append((day, str(sym))) + values.append(val) + + if not index_tuples: + return empty_factor_series(name) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +def reversal_20( + closes: pd.DataFrame | pd.Series, + *, + days: int = VALLEY_QUANTILE_REVERSAL_DAYS, + close_col: str = "close", + name: str = "rev20", +) -> pd.Series: + """T-1-based ``days``-day reversal factor from FRONT-ADJUSTED daily closes. + + ``rev20(d) = -(close_{d-1} / close_{d-(days+1)} - 1)`` — PINNED §3 of the module + docstring. The report's naive form uses ``close_d``, which is 15:00 information and + would be a LOOKAHEAD at our 14:50 decision time; taking the whole ratio one day + earlier makes every input strictly visible when the factor is formed. + + The closes MUST be front-adjusted: the ratio spans ``days`` trading days and a raw + series would break across any split or dividend inside the window. + + Args: + closes: ``MultiIndex(date, symbol)`` daily panel (DataFrame with ``close_col``, + or a Series of closes). Typically ``panel[["close"]]`` from the runner — + NO new data source. + days: reversal span in trading days (definition, not a tuned knob). + close_col: the close column name when ``closes`` is a DataFrame. + name: the returned Series name. + + Returns: + ``MultiIndex(date, symbol)`` Series over the SAME rows as the input, NaN where + the T-1 window is incomplete or a close is non-positive / non-finite (honest + missing — never fabricated). Pure: never mutates ``closes``. + """ + if days < 1: + raise ValueError(f"days must be >= 1; got {days!r}.") + if isinstance(closes, pd.DataFrame): + if close_col not in closes.columns: + raise ValueError( + f"reversal_20 needs a '{close_col}' column; got {list(closes.columns)}." + ) + series = closes[close_col] + else: + series = closes + if series.empty: + return empty_factor_series(name) + + s = series.astype(float).sort_index() + # Non-positive / non-finite closes cannot form a ratio; blank them BEFORE shifting so + # a bad print poisons only the windows that actually touch it. + clean = s.where(np.isfinite(s.to_numpy(dtype=float)) & (s.to_numpy(dtype=float) > 0.0)) + by_symbol = clean.groupby(level=SYMBOL_LEVEL, sort=False) + prior = by_symbol.shift(1) # close_{d-1} + base = by_symbol.shift(days + 1) # close_{d-(days+1)} + rev = -(prior / base - 1.0) + return rev.rename(name) + + +def residualize_on_reversal( + qbar: pd.Series, + rev: pd.Series, + *, + min_cross_section: int = VALLEY_QUANTILE_MIN_CROSS_SECTION, + name: str = "valley_price_quantile", +) -> pd.Series: + """Per-date cross-sectional OLS residual of ``qbar`` on ``rev`` (step 6). + + The report neutralizes this factor against the 20-day reversal because the raw + quantile is positively exposed to it. For each panel date the cross-section is the + symbols carrying BOTH a finite ``qbar`` and a finite ``rev``; the residual is + ``qbar - (a + b*rev)`` from an intercept OLS fit on that cross-section. Mirrors PR-G's + per-date cross-sectional post-processing, and follows the project's neutralization + convention that an under-determined cross-section yields NaN rather than a fabricated + value. + + PINNED §7: a date with fewer than ``min_cross_section`` paired symbols is all-NaN; a + symbol missing ``rev`` is dropped from the fit and its residual is NaN (never a silent + zero fill); a degenerate cross-section (zero-variance ``rev``, which cannot identify a + slope) is likewise NaN rather than an unresidualized ``qbar`` passed off as + neutralized. + + Args: + qbar: ``MultiIndex(date, symbol)`` raw trailing-mean quantile panel. + rev: ``MultiIndex(date, symbol)`` reversal panel (from :func:`reversal_20`). + min_cross_section: minimum paired cross-section for a finite residual. + name: the returned Series name (the factor-panel column name). + + Returns: + A Series over EXACTLY ``qbar``'s rows, in ``qbar``'s order, holding the residual + (or NaN). Pure: never mutates its inputs. + """ + if min_cross_section < 2: + # Need >= 2 cross-section members before an intercept + slope fit is determined. + raise ValueError( + f"min_cross_section must be >= 2; got {min_cross_section!r}." + ) + if qbar is None or qbar.empty: + return empty_factor_series(name) + + y_all = qbar.to_numpy(dtype=float) + # Align the reversal onto qbar's rows WITHOUT reordering them (the output must line + # up with the caller's panel row-for-row). + x_all = rev.reindex(qbar.index).to_numpy(dtype=float) + out = np.full(y_all.shape, np.nan, dtype=float) + + dates = qbar.index.get_level_values(DATE_LEVEL).to_numpy() + paired = np.isfinite(y_all) & np.isfinite(x_all) + for date in pd.unique(dates): + on_date = dates == date + fit_rows = on_date & paired + n = int(fit_rows.sum()) + if n < min_cross_section: + continue # honest missing: the whole date stays NaN + x = x_all[fit_rows] + y = y_all[fit_rows] + x_mean = x.mean() + sxx = float(((x - x_mean) ** 2).sum()) + if not np.isfinite(sxx) or sxx <= 0.0: + continue # degenerate: no slope is identified -> NaN, not a raw qbar + y_mean = y.mean() + slope = float(((x - x_mean) * (y - y_mean)).sum()) / sxx + out[fit_rows] = y - (y_mean + slope * (x - x_mean)) + + return pd.Series(out, index=qbar.index, name=name) + + +def compute_valley_price_quantile( + bars: pd.DataFrame, + closes: pd.DataFrame | pd.Series, + *, + lookback_days: int = VALLEY_QUANTILE_LOOKBACK_DAYS, + baseline_days: int = VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, + sigma_k: float = VOLUME_PRV_SIGMA_K, + min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + min_valley_bars: int = VALLEY_QUANTILE_MIN_VALLEY_BARS, + min_cross_section: int = VALLEY_QUANTILE_MIN_CROSS_SECTION, + reversal_days: int = VALLEY_QUANTILE_REVERSAL_DAYS, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "valley_price_quantile", +) -> pd.Series: + """Single-call factor: raw quantile stats (1-5) then reversal neutralization (6). + + Convenience that chains :func:`compute_valley_price_quantile_stats`, + :func:`reversal_20` and :func:`residualize_on_reversal`. The runner does NOT use this + (it needs the per-symbol stats loop for memory-boundedness, and it reports the + before/after-neutralization diagnostics) but tests and any all-in-memory caller can. + + ``closes`` must be the FRONT-ADJUSTED daily close panel — see PINNED §3. Pure: never + mutates its inputs. + """ + stats = compute_valley_price_quantile_stats( + bars, + lookback_days=lookback_days, + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + min_valid_days=min_valid_days, + min_classifiable=min_classifiable, + min_valley_bars=min_valley_bars, + decision_time=decision_time, + ) + if stats.empty: + return empty_factor_series(name) + rev = reversal_20(closes, days=reversal_days) + return residualize_on_reversal( + stats, rev, min_cross_section=min_cross_section, name=name + ) + + +class ValleyPriceQuantileFactor(Factor): + """Valley weighted-price-quantile factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_valley_price_quantile_stats` and then + reversal-neutralized by :func:`residualize_on_reversal`); it does NO minute work of + its own, mirroring its siblings. + + Args: + lookback_days: trailing VALID trading-day window averaged; part of the factor + DEFINITION (reproduced from the report), not a tuned knob. It only names the + column so a non-default window cannot silently mislabel it. + """ + + name: str = f"valley_price_quantile_{VALLEY_QUANTILE_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = VALLEY_QUANTILE_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"valley-price-quantile lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"valley_price_quantile_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=+1 (report full-market RankIC +6.34%, RankICIR 4.32, long leg + 13.1%/yr, long-short 20.22%/yr, IR 3.29, max drawdown 10.18%, monthly win rate + 80.4%; CSI500 sub-domain long-short 11.71% / IR 1.76 — the closest comparable to + our eval cell). Semantics per the report: a HIGH position of the calm-minute + (valley) price within the day's range means the informed, unhurried part of the + day traded near the top of the range -> higher future return. The sign is fixed + BEFORE the run (a validated prototype must reproduce it). NOTE the report is a + MONTHLY, market-cap + industry neutral full-market series on Wind data while our + eval cell is CSI500 daily with industry + size neutral, so the report numbers are + a LOOSE reference only (disclosed, never mislabeled, never written in as an + expected value). is_intraday=False: minute INPUT but a DAILY signal traded + close-to-close. min_history_bars=0: the warm-up is DATA-dependent, not a fixed + leading count — the honest NaN rate is reported by data_coverage. + + The description spells out the two subtleties that make this factor harder than + its five siblings: the PREV_CLOSE-extended range read at OUR 14:50 visibility + rather than the report's true daily close, and the reversal neutralization taken + at T-1 so day d's 15:00 close is never an input to day d's value. + + D1 declarations (D0 pre-assignment table row 10 + note 2 — the + two-axes-differ exemplar): adjustment=returns_invariant — the rev20 + leg is a ratio of FRONT-ADJUSTED daily closes (anchor cancels; this + module's ``reversal_20`` PINNED §3) and the minute legs are raw but + fully same-day, so an ANCHOR perturbation does not move the value. + overnight_boundary=CROSSED_DISCLOSED — prev_close is a raw close on + the PREVIOUS day's basis entering day d's range, the pinned definition + keeps the ex-date values, and the deviation is MEASURED and disclosed + (0.73% true ex-dates / ~0.21% ex-date AND range-distorted; module + docstring PINNED §4, pinned in + tests/test_valley_price_quantile_factor.py). All three + CROSSED_DISCLOSED requirements are met — unlike PR-D, nothing is owed. + ``requires`` adds market_daily close beyond the minute fields: the T-1 + rev20 neutralization reads the qfq DAILY close panel (a real second + endpoint, declared truthfully rather than mirrored off input_fields). + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Valley weighted-price-quantile (Kaiyuan microstructure series #27, " + f"SIXTH factor 量谷加权价格分位点). SAME minute classification as PR-F " + f"volume_peak_count / PR-H peak_interval_kurtosis / PR-I " + f"valley_relative_vwap / PR-J valley_ridge_vwap_ratio / PR-K " + f"ridge_minute_return (SHARED taxonomy in factors.compute.minute." + f"primitives, not re-implemented): 1min bars PIT-truncated at 14:50, a " + f"minute is ERUPTIVE if vol > μ + {VOLUME_PRV_SIGMA_K:g}σ of its " + f"SAME-SLOT strictly-prior {VOLUME_PRV_BASELINE_DAYS}-day baseline, and " + f"a VALLEY (量谷) is a classifiable NON-eruptive minute. NEW STATISTIC " + f"vs PR-F..PR-K: a price POSITION rather than a count, a timing moment, " + f"a price RATIO or a return — each valid day scores WHERE the valley " + f"VWAP sits inside the day's price range, and the factor averages that " + f"over the trailing {self._lookback_days} VALID days before a " + f"cross-sectional reversal neutralization. PINNED choices: (1) the " + f"range is [min(visible low, prev_close), max(visible high, " + f"prev_close)] where prev_close is the LAST VISIBLE (<=14:50) RAW CLOSE " + f"of the PREVIOUS trading day — a DISCLOSED DEVIATION from the report, " + f"which uses the true previous daily close; the previous 15:00 close is " + f"not itself a lookahead at a 14:50 decision, so this is a " + f"SINGLE-VISIBILITY-DEFINITION choice (every price in the factor comes " + f"from one source under one cutoff), not a leakage fix; (2) the daily " + f"quantile (valley VWAP - lo)/(hi - lo) is NOT clipped — a VWAP outside " + f"the range means the range is wrong and must be visible rather than " + f"sanitized; (3) THE REVERSAL NEUTRALIZATION IS TAKEN AT T-1: rev20 = " + f"-(close_(d-1)/close_(d-{VALLEY_QUANTILE_REVERSAL_DAYS + 1}) " + f"- 1) on FRONT-ADJUSTED daily closes from the panel (NO new data source). " + f"The report's naive form uses close_d, which is 15:00 information and " + f"WOULD be a lookahead at our 14:50 decision — day d's close is therefore " + f"never an input to day d's factor value; front-adjusted closes are " + f"required because the ratio spans {VALLEY_QUANTILE_REVERSAL_DAYS} " + f"trading days; (4) RAW minute prices for the range and the VWAP, with a " + f"DISCLOSED imperfection: prev_close crosses the overnight boundary, so on " + f"an ex-dividend / split date the previous raw close sits on the OLD " + f"(higher) scale while the day's own bars sit on the new one. The bias is " + f"ONE-SIDED: prev_close pulls hi UP (measured 42 top-widening vs 4 " + f"bottom-widening among true ex-dates), inflating hi - lo while the valley " + f"VWAP and lo stay on the new scale, which compresses that day's quantile " + f"toward the LOW END of an artificially wide range — NOT toward the middle. " + f"Measured: 15.6% of valid days show any prev_close widening (overwhelmingly " + f"ordinary overnight gaps, which is INTENDED), only 0.73% are true ex-dates " + f"and only ~0.21% are both a true ex-date AND range-distorted, before the " + f"{self._lookback_days}-day mean dilutes it; disclosed rather than silently " + f"corrected; (5) a PRICE guard (finite, high >= low > 0) admits a bar to " + f"the range while a TRADE guard (finite positive volume AND amount) " + f"admits it to the VWAP, both applied at the summation step only so PR-F's " + f"baseline stays bit-identical; (6) a day is VALID iff it has >= " + f"{VOLUME_PRV_MIN_CLASSIFIABLE} classifiable bars AND >= " + f"{VALLEY_QUANTILE_MIN_VALLEY_BARS} TRADABLE valley bars AND positive " + f"valley volume AND an available prev_close AND hi > lo — so a symbol's " + f"FIRST visible day never produces a value; (7) the residualization is " + f"PER DATE on the covered cross-section, with < " + f"{VALLEY_QUANTILE_MIN_CROSS_SECTION} paired symbols, a missing rev20, or " + f"a degenerate (zero-variance) rev20 all yielding NaN rather than a " + f"zero-filled or unresidualized value passed off as neutralized; (8) " + f"DEVIATION FROM THE REPORT, disclosed: everything spans the PIT-VISIBLE " + f"window 09:31-14:50 only, not the full session. NaN below " + f"{VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived from 1min bars but a " + f"DAILY signal traded close-to-close." + ), + expected_ic_sign=1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from, plus the daily + # close the T-1 reversal neutralization regresses against. Declared for honest + # provenance disclosure (data_coverage lists them); the daily panel surfaces + # the pre-aggregated column itself. + input_fields=("volume", "amount", "high", "low", "close"), + requires=( + *_minute_requires("volume", "amount", "high", "low", "close"), + # The T-1 rev20 neutralization reads the front-adjusted DAILY + # close panel — a genuine market_daily requirement on top of + # the minute fields (see the docstring's D1 paragraph). + PanelField("close", source=MARKET_DAILY), + ), + adjustment="returns_invariant", + overnight_boundary="crossed_disclosed", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily valley-price-quantile column off ``panel``. + + The runner runs ``compute_valley_price_quantile_stats`` per symbol on the minute + cache and ``residualize_on_reversal`` once on the assembled panel upstream, then + joins the result as ``self.name``; here we only surface it, so this factor does no + temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"ValleyPriceQuantileFactor needs the pre-aggregated '{self.name}' " + f"column on the panel (produced upstream by " + f"compute_valley_price_quantile_stats + residualize_on_reversal and " + f"joined by the runner); panel has {list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "VALLEY_QUANTILE_LOOKBACK_DAYS", + "VALLEY_QUANTILE_MIN_CROSS_SECTION", + "VALLEY_QUANTILE_MIN_VALLEY_BARS", + "VALLEY_QUANTILE_REVERSAL_DAYS", + "ValleyPriceQuantileFactor", + "compute_valley_price_quantile", + "compute_valley_price_quantile_stats", + "residualize_on_reversal", + "reversal_20", + "valley_price_quantile_by_day", +] diff --git a/factors/compute/minute/valley_relative_vwap.py b/factors/compute/minute/valley_relative_vwap.py new file mode 100644 index 0000000..ab5a938 --- /dev/null +++ b/factors/compute/minute/valley_relative_vwap.py @@ -0,0 +1,434 @@ +"""VALLEY-RELATIVE VWAP factor (PR-I): math + surface (D2). + +Reproduces the THIRD factor of the Kaiyuan market-microstructure series #27 (开源证券 +《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, +§4): "参考聪明钱因子的构建方式，计算每日量谷的成交量加权价格，与当日总成交量加权价格做比， +并计算 20 日均值作为量谷相对加权价格因子". + +Same MACHINE as PR-F / PR-H, different FAMILY. The volume classification is the SHARED +taxonomy in :mod:`factors.compute.minute.primitives` (``prepare_visible_minute_bars`` + +``peak_mask_for_symbol``) — same-slot strictly-prior μ+kσ eruptive/mild test, same +classifiable rule, same valid-day floor. A VALLEY (量谷) is exactly that taxonomy's +``valley`` column: a classifiable, NON-eruptive minute. Nothing about the taxonomy is +re-implemented here, so the family can never drift apart. Where PR-F counts peaks +and PR-H measures the TIMING of peaks, this module measures the PRICE LEVEL at the +valleys. + +PINNED choices (deliberate and DISCLOSED, not tuned knobs; reproduced on the factor spec +so a reader sees exactly what was assumed): + + 1. VWAP VIA THE AGGREGATION IDENTITY. A bar's volume-weighted price is + ``p = amount / volume``, so a set's volume-weighted price + ``Σ(p_i·v_i) / Σv_i`` collapses EXACTLY to ``Σamount / Σvolume``. Both legs use + that identity: valley VWAP = Σamount(valley bars)/Σvolume(valley bars), day VWAP + = Σamount(all visible bars)/Σvolume(all visible bars). This is the day's REAL + VWAP, strictly better than approximating each bar by its close. + 2. POSITIVE-TRADE GUARD. A bar with non-finite or non-positive ``volume`` OR + ``amount`` carries no price information, so it is dropped from BOTH sums + (:func:`~factors.compute.minute.primitives.positive_trade_mask`). The guard runs + at the summation step only — never before classification — because the same-slot + μ/σ baseline is the shared taxonomy's and must stay bit-identical. + 3. RAW (UNADJUSTED) PRICES. The cached minute bars are unadjusted. Both legs are + sums over the SAME trading day, and a split/dividend adjustment factor is constant + within a day, so it cancels exactly in the ratio — no adjustment is needed (the + same reasoning PR-D's amplitude and I5b's price-limit checks rely on). + 4. THE DAY VWAP COVERS THE PIT-VISIBLE WINDOW ONLY (09:31–14:50), not the full + session. This is a NECESSARY DEVIATION from the report, which uses the whole day: + the standing 14:50 decision cutoff truncates history days and the signal day + identically, and reading the closing auction would be lookahead at our decision + time. Disclosed, not silently equated to the report's construction. + 5. VALLEY-BAR COUNT IS TAKEN AFTER THE GUARD. The ``min_valley_bars`` floor counts + the valley minutes that actually CONTRIBUTE to the numerator, so a day cannot + qualify on the strength of valley bars that traded nothing. + +Factor value: ``valley_relative_vwap_20`` = the MEAN of the daily ratio +``valley VWAP / day VWAP`` over the symbol's most recent ``lookback_days`` (=20) VALID +trading days INCLUDING ``d``. A day is VALID iff it clears three gates: at least +``min_classifiable`` (=100) classifiable bars (PR-F's gate, unchanged), at least +``min_valley_bars`` (=20) tradable valley bars, and strictly positive volume in BOTH +denominators. Fewer than ``min_valid_days`` (=10) valid days in the trailing window -> +NaN (honest missing; the runner discloses the coverage). Values are emitted only on +valid days. + +Pre-registered sign = +1. The report's full-market RankIC is +8.69% / RankICIR 4.44 +(long-short 25.35%/yr, IR 3.04, monthly win rate 79.7%) — the strongest factor in the +report; its CSI500 sub-domain long-short is 9.94% / IR 1.26, the closest comparable to +our eval cell. Semantics per the report: valley minutes are moments of subdued trading +sentiment where prices are unlikely to have over-reacted, so a HIGH relative valley +price means the calm, informed part of the day was bid up -> higher future return. NOTE +the report is a MONTHLY, market-cap + industry neutral full-market series on Wind data +while our eval cell is CSI500 daily with industry + size neutralization, so its numbers +are a LOOSE reference only (disclosed, never mislabeled, never written in as an expected +value). + +The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future +bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + VOLUME_PRV_BASELINE_DAYS, + VOLUME_PRV_BASELINE_MIN_OBS, + VOLUME_PRV_MIN_CLASSIFIABLE, + VOLUME_PRV_MIN_VALID_DAYS, + VOLUME_PRV_SIGMA_K, + empty_factor_series, + peak_mask_for_symbol, + positive_trade_mask, + prepare_visible_minute_bars, + rolling_valid_days, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). +# The classification constants live with the taxonomy in ``primitives``. +VALLEY_VWAP_LOOKBACK_DAYS = 20 # trailing VALID trading-day mean window, includes d +VALLEY_VWAP_MIN_VALLEY_BARS = 20 # min TRADABLE valley bars for a valid day + +# The extra 1min column this family needs on top of the taxonomy's (volume): the traded +# value, which turns the bar set into a volume-weighted price via Σamount/Σvolume. +_AMOUNT = "amount" + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def valley_vwap_ratio_by_day( + work: pd.DataFrame, + *, + min_valley_bars: int = VALLEY_VWAP_MIN_VALLEY_BARS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, +) -> pd.Series: + """Daily ``valley VWAP / day VWAP`` ratio for ONE symbol, on VALID days only. + + ``work`` is one symbol's frame as returned by + :func:`~factors.compute.minute.primitives.peak_mask_for_symbol`, which must have been + built from bars prepared with ``extra_columns=("amount",)`` so the traded value is + available alongside the ``valley`` / ``classifiable`` masks. + + Both VWAPs use the ``Σamount / Σvolume`` aggregation identity (PINNED §1 of the + module docstring). The POSITIVE-TRADE GUARD (§2) drops non-finite / non-positive + volume or amount bars from both sums; the day leg spans ALL visible bars that clear + the guard (eruptive minutes included — it is the WHOLE visible day's VWAP), while + the valley leg spans only the guarded valley bars. + + Returns: + Series indexed by ``trade_date`` (ascending) holding the ratio for the days that + clear all three validity gates — invalid days are ABSENT, not NaN, so they do + not occupy a slot in the caller's trailing window (the same rule PR-F/PR-H use). + """ + vol = work["volume"].to_numpy(dtype=float) + amt = work[_AMOUNT].to_numpy(dtype=float) + # Positive-trade guard: no price information in a bar that traded nothing (or whose + # fields are non-finite / negative). Applied HERE, at the summation step, never + # before classification — the shared same-slot baseline must stay bit-identical. + tradable = positive_trade_mask(vol, amt) + valley = work["valley"].to_numpy(dtype=bool) & tradable + + per_bar = pd.DataFrame( + { + "trade_date": work["trade_date"].to_numpy(), + "day_amt": np.where(tradable, amt, 0.0), + "day_vol": np.where(tradable, vol, 0.0), + "valley_amt": np.where(valley, amt, 0.0), + "valley_vol": np.where(valley, vol, 0.0), + "valley_bars": valley.astype(np.int64), + "classifiable_bars": work["classifiable"].to_numpy(dtype=bool).astype( + np.int64 + ), + } + ) + agg = per_bar.groupby("trade_date", sort=True).sum() + + # Three validity gates (§ module docstring / task card §1.4): PR-F's classifiable + # floor (unchanged), enough TRADABLE valley bars, and a positive denominator on both + # legs (a 0/0 ratio is never fabricated). + valid = ( + (agg["classifiable_bars"] >= min_classifiable) + & (agg["valley_bars"] >= min_valley_bars) + & (agg["day_vol"] > 0.0) + & (agg["valley_vol"] > 0.0) + ) + ok = agg.loc[valid] + if ok.empty: + return pd.Series( + [], index=pd.DatetimeIndex([], name="trade_date"), dtype=float + ) + valley_vwap = ok["valley_amt"] / ok["valley_vol"] + day_vwap = ok["day_amt"] / ok["day_vol"] + return (valley_vwap / day_vwap).astype(float) + + +def _valley_ratio_mean_for_symbol( + g: pd.DataFrame, + *, + baseline_days: int, + baseline_min_obs: int, + sigma_k: float, + lookback_days: int, + min_valid_days: int, + min_classifiable: int, + min_valley_bars: int, +) -> tuple[list[pd.Timestamp], list[float]]: + """Daily valley-relative-VWAP values for ONE symbol from its PIT-visible bars. + + Classifies the minutes with the SHARED :func:`peak_mask_for_symbol`, reduces each + valid day to its VWAP ratio, then takes the trailing-``lookback_days``-valid-day + mean. No cross-symbol leakage (``g`` is one symbol's slice) and no lookahead (the + baseline is strictly prior, the mean window is trailing). + """ + work = peak_mask_for_symbol( + g, + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + ) + ratio = valley_vwap_ratio_by_day( + work, min_valley_bars=min_valley_bars, min_classifiable=min_classifiable + ) + if ratio.empty: + return [], [] + + # Mean over the trailing lookback_days VALID days (including d); NaN until + # min_valid_days valid days have accumulated. Emitted only on valid days. + rolled = rolling_valid_days( + ratio, lookback_days=lookback_days, min_valid_days=min_valid_days + ).mean() + days = [pd.Timestamp(d).normalize() for d in rolled.index] + return days, list(rolled.to_numpy(dtype=float)) + + +def compute_valley_relative_vwap( + bars: pd.DataFrame, + *, + lookback_days: int = VALLEY_VWAP_LOOKBACK_DAYS, + baseline_days: int = VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, + sigma_k: float = VOLUME_PRV_SIGMA_K, + min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + min_valley_bars: int = VALLEY_VWAP_MIN_VALLEY_BARS, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "valley_relative_vwap", +) -> pd.Series: + """PIT-safe daily "valley-relative VWAP" factor from 1min ``bars``. + + Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, + classifies every visible minute with the SHARED PR-F taxonomy, computes each valid + day's ``valley VWAP / whole-visible-day VWAP`` ratio via the ``Σamount/Σvolume`` + identity, and returns the trailing-``lookback_days``-VALID-day mean of that ratio. + See the module docstring for the LOCKED definition and the five pinned choices. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is + strictly per symbol (no cross-symbol leakage). + lookback_days: trailing VALID trading-day window averaged (definition). + baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). + baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). + sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). + min_valid_days: minimum valid days in the trailing window for a finite value. + min_classifiable: a day needs at least this many classifiable bars (PR-F). + min_valley_bars: a day needs at least this many TRADABLE valley bars. + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + name: the returned Series name (the factor-panel column name). + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily + factor value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if baseline_days < 2: + # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is + # defined. + raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") + if baseline_min_obs < 2: + raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") + if sigma_k < 0.0: + raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") + if min_valid_days < 1: + raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") + if min_classifiable < 1: + raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") + if min_valley_bars < 1: + raise ValueError(f"min_valley_bars must be >= 1; got {min_valley_bars!r}.") + if len(bars) == 0: + return empty_factor_series(name) + + # extra_columns=("amount",) is the ONLY difference from the PR-F / PR-H entry + # points: the traded value rides along on the surviving rows, and the truncation / + # volume guard / slot assignment are untouched. + visible = prepare_visible_minute_bars( + bars, decision_time=decision_time, extra_columns=(_AMOUNT,) + ) + if visible.empty: + return empty_factor_series(name) + + index_tuples: list[tuple] = [] + values: list[float] = [] + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + days, vals = _valley_ratio_mean_for_symbol( + g.reset_index(drop=True), + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + lookback_days=lookback_days, + min_valid_days=min_valid_days, + min_classifiable=min_classifiable, + min_valley_bars=min_valley_bars, + ) + for day, val in zip(days, vals): + index_tuples.append((day, str(sym))) + values.append(val) + + if not index_tuples: + return empty_factor_series(name) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +class ValleyRelativeVwapFactor(Factor): + """Valley-relative VWAP factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_valley_relative_vwap`); it does NO minute work of its + own, mirroring its siblings and the value / financial factors that surface an + enriched column. + + Args: + lookback_days: trailing VALID trading-day window averaged; part of the factor + DEFINITION (reproduced from the report), not a tuned knob. It only names the + column so a non-default window cannot silently mislabel it. + """ + + name: str = f"valley_relative_vwap_{VALLEY_VWAP_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = VALLEY_VWAP_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"valley-relative-vwap lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"valley_relative_vwap_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=+1: the report's full-market RankIC is +8.69% (RankICIR 4.44, + long-short 25.35%/yr, IR 3.04, monthly win rate 79.7% — the strongest factor in + the report), and its CSI500 sub-domain long-short is 9.94% / IR 1.26, the closest + comparable to our eval cell. Semantics per the report: valley minutes are moments + of subdued sentiment where prices are unlikely to have over-reacted, so a HIGH + relative valley price predicts HIGHER forward returns. The sign is fixed BEFORE + the run (a validated prototype must reproduce it). NOTE the report is a MONTHLY, + market-cap + industry neutral full-market series on Wind data while our eval cell + is CSI500 daily with industry + size neutral, so the report numbers are a LOOSE + reference only (disclosed, never mislabeled, never written in as an expected + value). is_intraday=False by the module docstring's reasoning: minute INPUT but a + DAILY signal traded close-to-close. min_history_bars=0: the warm-up is + DATA-dependent (a value appears once enough VALID days accumulate), not a fixed + leading count — the honest NaN rate is reported by data_coverage. + + The description spells out the DIFFERENT FAMILY vs PR-F / PR-H (the same shared + classification, but a PRICE LEVEL at the VALLEYS rather than a count or timing of + the PEAKS) plus the pinned choices, including the one place we KNOWINGLY DEVIATE + from the report: our day VWAP covers the PIT-visible window only. + + D1 declarations (D0 pre-assignment table row 7): adjustment= + returns_invariant — both VWAP legs are sums over the SAME trading day + and "a split/dividend adjustment factor is constant within a day, so + it cancels exactly in the ratio" (module docstring PINNED §3). + Price information arrives via Σamount/Σvolume, which is why requires + lists no OHLC field — the anomaly is real and intended. + overnight_boundary=none — both legs are same-day; nothing crosses the + overnight boundary. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Valley-relative VWAP (Kaiyuan microstructure series #27, THIRD factor " + f"量谷相对加权价格). SAME minute classification as PR-F volume_peak_count " + f"and PR-H peak_interval_kurtosis (SHARED taxonomy in factors.compute." + f"minute.primitives, not re-implemented): 1min bars PIT-truncated at " + f"14:50, a minute is ERUPTIVE if vol > μ + {VOLUME_PRV_SIGMA_K:g}σ of " + f"its SAME-SLOT strictly-prior {VOLUME_PRV_BASELINE_DAYS}-day baseline, " + f"else it is a VALLEY (量谷). DIFFERENT FAMILY: instead of counting or " + f"timing the PEAKS this factor prices the VALLEYS — daily ratio = " + f"(valley VWAP) / (whole visible day VWAP), averaged over the trailing " + f"{self._lookback_days} VALID days. PINNED choices: (1) each VWAP uses " + f"the aggregation identity Σ(p·v)/Σv = Σamount/Σvolume, the day's REAL " + f"volume-weighted price rather than a close approximation; (2) bars with " + f"non-finite or non-positive volume or amount are dropped from BOTH " + f"sums (guard applied at summation only, so PR-F's baseline is " + f"untouched); (3) RAW unadjusted prices are correct here because the " + f"adjustment factor is constant within a day and cancels in the ratio; " + f"(4) DEVIATION FROM THE REPORT, disclosed: our day VWAP spans the " + f"PIT-VISIBLE window 09:31-14:50 only, not the full session — reading " + f"the close would be lookahead at our 14:50 decision time; (5) a day is " + f"VALID iff it has >= {VOLUME_PRV_MIN_CLASSIFIABLE} classifiable bars " + f"AND >= {VALLEY_VWAP_MIN_VALLEY_BARS} TRADABLE valley bars (counted " + f"after the guard) AND positive volume in both denominators; NaN below " + f"{VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived from 1min bars but a " + f"DAILY signal traded close-to-close." + ), + expected_ic_sign=1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from. Declared for + # honest provenance disclosure (data_coverage lists them); the daily panel + # surfaces the pre-aggregated column itself. + input_fields=("volume", "amount"), + requires=_minute_requires("volume", "amount"), + adjustment="returns_invariant", + overnight_boundary="none", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily valley-relative-VWAP column off ``panel``. + + The runner runs ``compute_valley_relative_vwap`` per symbol on the minute cache + upstream and joins the result as ``self.name``; here we only surface it, so this + factor does no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"ValleyRelativeVwapFactor needs the pre-aggregated '{self.name}' column " + f"on the panel (produced upstream by compute_valley_relative_vwap and " + f"joined by the runner); panel has {list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "VALLEY_VWAP_LOOKBACK_DAYS", + "VALLEY_VWAP_MIN_VALLEY_BARS", + "ValleyRelativeVwapFactor", + "compute_valley_relative_vwap", + "valley_vwap_ratio_by_day", +] diff --git a/factors/compute/minute/valley_ridge_vwap_ratio.py b/factors/compute/minute/valley_ridge_vwap_ratio.py new file mode 100644 index 0000000..f50681b --- /dev/null +++ b/factors/compute/minute/valley_ridge_vwap_ratio.py @@ -0,0 +1,520 @@ +"""VALLEY/RIDGE VWAP-RATIO factor (PR-J): math + surface (D2). + +Reproduces the FOURTH factor of the Kaiyuan market-microstructure series #27 (开源证券 +《高频成交量的峰、岭、谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417, +§7.1): "计算每日量谷成交量加权价格，与量岭成交量加权价格做比，并计算 20 日价格比均值作为 +谷岭加权价格比因子". + +Same MACHINE as PR-F / PR-H / PR-I and the same VWAP identity as PR-I — what changes is +the DENOMINATOR. PR-I divided the valley VWAP by the WHOLE VISIBLE DAY's VWAP; this +factor divides it by the RIDGE VWAP, so the two behavioural groups of the report's +peak/ridge/valley taxonomy are contrasted head-on rather than one being compared against +an aggregate that contains it. That makes PR-J the natural robustness test of PR-I: if +the price-level signal survives the swap, it is a property of the FAMILY; if it does not, +PR-I depended on its specific denominator. Either outcome is a legitimate result. + +The classification is the SHARED taxonomy in :mod:`factors.compute.minute.primitives` +(``prepare_visible_minute_bars`` + ``peak_mask_for_symbol``) — same same-slot +strictly-prior μ+kσ eruptive test, same classifiable rule, same valid-day floor. Nothing +about the taxonomy is re-implemented here, so the family can never drift apart. + +PINNED choices (deliberate and DISCLOSED, not tuned knobs; reproduced on the factor spec +so a reader sees exactly what was assumed): + + 1. THE RIDGE MASK IS ``eruptive & ~peak``. A RIDGE (量岭) is an eruptive minute that is + NOT an isolated peak. This is WIDER than the literal "eruptive next to an eruptive": + it also covers the session-boundary eruptions (a neighbour missing across the lunch + break / the cutoff / a data gap) and the eruptions whose neighbour is unclassifiable. + The rule is the exact COMPLEMENT of PR-F's deliberately conservative peak test, so + the taxonomy stays a clean partition — ``valley | peak | ridge == classifiable`` — + and no eruptive bar is silently dropped on either side. Disclosed rather than + silently equated to the report's wording. + 2. VWAP VIA THE AGGREGATION IDENTITY (same as PR-I). A bar's volume-weighted price is + ``p = amount / volume``, so a set's volume-weighted price ``Σ(p_i·v_i) / Σv_i`` + collapses EXACTLY to ``Σamount / Σvolume``. Both legs use that identity: valley VWAP + = Σamount(valley bars)/Σvolume(valley bars), ridge VWAP = Σamount(ridge + bars)/Σvolume(ridge bars). + 3. POSITIVE-TRADE GUARD. A bar with non-finite or non-positive ``volume`` OR ``amount`` + carries no price information, so it is dropped from BOTH sums. The guard runs at the + summation step only — never before classification — because the same-slot μ/σ + baseline is the shared taxonomy's and must stay bit-identical. A zero-volume minute + is therefore still classified (and still a valley) but contributes nothing to either + VWAP. + 4. RAW (UNADJUSTED) PRICES. The cached minute bars are unadjusted. Both legs are sums + over the SAME trading day, and a split/dividend adjustment factor is constant within + a day, so it cancels exactly in the ratio — no adjustment is needed (the same + reasoning PR-I, PR-D's amplitude and I5b's price-limit checks rely on). + 5. BOTH LEGS COVER THE PIT-VISIBLE WINDOW ONLY (09:31–14:50), not the full session. + A NECESSARY DEVIATION from the report, which uses the whole day: the standing 14:50 + decision cutoff truncates history days and the signal day identically, and reading + the closing auction would be lookahead at our decision time. + 6. AN ASYMMETRIC BAR FLOOR — ``min_valley_bars`` (=20) but ``min_ridge_bars`` (=10). + Ridge bars are STRUCTURALLY far scarcer than valley bars: a minute must erupt (a + minority event by construction, since the threshold is its own strictly-prior + same-slot μ+σ) AND fail the isolation test. Holding the ridge leg to the valley + leg's floor would throw away a large share of otherwise sound days and bias the + surviving sample towards unusually turbulent sessions. The floor is therefore + LOWERED, deliberately, and both the realized ridge-bar distribution and the + resulting day-validity rate are REPORTED (``with_diagnostics=True``) rather than + left implicit — if coverage comes out materially worse than PR-I's, that is a + finding to surface, not to hide. + 7. BOTH BAR COUNTS ARE TAKEN AFTER THE GUARD, so a day cannot qualify on the strength + of bars that traded nothing. + +Factor value: ``valley_ridge_vwap_ratio_20`` = the MEAN of the daily ratio +``valley VWAP / ridge VWAP`` over the symbol's most recent ``lookback_days`` (=20) VALID +trading days INCLUDING ``d``. A day is VALID iff it clears four gates: at least +``min_classifiable`` (=100) classifiable bars (PR-F's gate, unchanged), at least +``min_valley_bars`` (=20) tradable valley bars, at least ``min_ridge_bars`` (=10) +tradable ridge bars, and strictly positive volume in BOTH denominators. Fewer than +``min_valid_days`` (=10) valid days in the trailing window -> NaN (honest missing; the +runner discloses the coverage). Values are emitted only on valid days. + +Pre-registered sign = +1. The report's full-market RankIC is +6.98% / RankICIR 3.56 +(long-short 15.83%/yr, IR 1.83, monthly win rate 72.3%, only 2023 negative in 13 years); +its CSI500 sub-domain long-short is 10.49% / IR 1.34, the closest comparable to our eval +cell. Semantics per the report: a HIGH valley/ridge price ratio means retail +over-reaction pushed the eruptive minutes' price DOWN relative to the calm ones, so the +stock is depressed and performs better going forward. NOTE the report is a MONTHLY, +market-cap + industry neutral full-market series on Wind data while our eval cell is +CSI500 daily with industry + size neutralization, so its numbers are a LOOSE reference +only (disclosed, never mislabeled, never written in as an expected value). + +The value at ``d`` uses only bars at dates <= d, so a factor value never sees a future +bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + VOLUME_PRV_BASELINE_DAYS, + VOLUME_PRV_BASELINE_MIN_OBS, + VOLUME_PRV_MIN_CLASSIFIABLE, + VOLUME_PRV_MIN_VALID_DAYS, + VOLUME_PRV_SIGMA_K, + empty_factor_series, + peak_mask_for_symbol, + positive_trade_mask, + prepare_visible_minute_bars, + rolling_valid_days, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constants (pinned interpretations of the report; NOT tuned knobs). +# The classification constants live with the taxonomy in ``primitives``. +VALLEY_RIDGE_LOOKBACK_DAYS = 20 # trailing VALID trading-day mean window, includes d +VALLEY_RIDGE_MIN_VALLEY_BARS = 20 # min TRADABLE valley bars for a valid day +# PINNED LOWER than the valley floor (module docstring §6): ridge bars are structurally +# far scarcer, so holding both legs to 20 would bias the surviving sample. +VALLEY_RIDGE_MIN_RIDGE_BARS = 10 # min TRADABLE ridge bars for a valid day + +# The extra 1min column this family needs on top of the taxonomy's (volume): the traded +# value, which turns the bar set into a volume-weighted price via Σamount/Σvolume. +_AMOUNT = "amount" + +# Per-day diagnostic columns (the ridge-scarcity disclosure the task card requires). +DIAGNOSTIC_COLUMNS = ("classifiable_bars", "valley_bars", "ridge_bars", "valid") + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def valley_ridge_vwap_ratio_by_day( + work: pd.DataFrame, + *, + min_valley_bars: int = VALLEY_RIDGE_MIN_VALLEY_BARS, + min_ridge_bars: int = VALLEY_RIDGE_MIN_RIDGE_BARS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + with_diagnostics: bool = False, +) -> pd.Series | tuple[pd.Series, pd.DataFrame]: + """Daily ``valley VWAP / ridge VWAP`` ratio for ONE symbol, on VALID days only. + + ``work`` is one symbol's frame as returned by + :func:`~factors.compute.minute.primitives.peak_mask_for_symbol`, which must have been + built from bars prepared with ``extra_columns=("amount",)`` so the traded value is + available alongside the ``valley`` / ``ridge`` / ``classifiable`` masks. + + Both VWAPs use the ``Σamount / Σvolume`` aggregation identity (PINNED §2 of the + module docstring). The POSITIVE-TRADE GUARD (§3) drops non-finite / non-positive + volume or amount bars from both sums. Unlike PR-I, the denominator is NOT the whole + visible day but the RIDGE bars alone (§1: ``eruptive & ~peak``), so an isolated PEAK + contributes to NEITHER leg. + + Args: + work: one symbol's classified minute frame (see above). + min_valley_bars: minimum TRADABLE valley bars for a valid day. + min_ridge_bars: minimum TRADABLE ridge bars for a valid day (PINNED lower than + the valley floor — see §6). + min_classifiable: minimum classifiable bars for a valid day (PR-F's gate). + with_diagnostics: also return the per-day bar-count frame, so the caller can + REPORT the ridge-scarcity distribution instead of only gating on it. + + Returns: + Series indexed by ``trade_date`` (ascending) holding the ratio for the days that + clear all gates — invalid days are ABSENT, not NaN, so they do not occupy a slot + in the caller's trailing window (the same rule PR-F / PR-H / PR-I use). With + ``with_diagnostics=True``, a ``(ratio, diagnostics)`` pair where ``diagnostics`` + is indexed by EVERY day present in ``work`` and carries + :data:`DIAGNOSTIC_COLUMNS`. + """ + vol = work["volume"].to_numpy(dtype=float) + amt = work[_AMOUNT].to_numpy(dtype=float) + # Positive-trade guard: no price information in a bar that traded nothing (or whose + # fields are non-finite / negative). Applied HERE, at the summation step, never + # before classification — the shared same-slot baseline must stay bit-identical. + tradable = positive_trade_mask(vol, amt) + valley = work["valley"].to_numpy(dtype=bool) & tradable + ridge = work["ridge"].to_numpy(dtype=bool) & tradable + + per_bar = pd.DataFrame( + { + "trade_date": work["trade_date"].to_numpy(), + "valley_amt": np.where(valley, amt, 0.0), + "valley_vol": np.where(valley, vol, 0.0), + "ridge_amt": np.where(ridge, amt, 0.0), + "ridge_vol": np.where(ridge, vol, 0.0), + "valley_bars": valley.astype(np.int64), + "ridge_bars": ridge.astype(np.int64), + "classifiable_bars": work["classifiable"] + .to_numpy(dtype=bool) + .astype(np.int64), + } + ) + agg = per_bar.groupby("trade_date", sort=True).sum() + + # Four validity gates (§ module docstring / task card §1.4): PR-F's classifiable + # floor (unchanged), enough TRADABLE valley bars, enough TRADABLE ridge bars, and a + # positive denominator on both legs (a 0/0 ratio is never fabricated). + valid = ( + (agg["classifiable_bars"] >= min_classifiable) + & (agg["valley_bars"] >= min_valley_bars) + & (agg["ridge_bars"] >= min_ridge_bars) + & (agg["valley_vol"] > 0.0) + & (agg["ridge_vol"] > 0.0) + ) + ok = agg.loc[valid] + if ok.empty: + ratio = pd.Series([], index=pd.DatetimeIndex([], name="trade_date"), dtype=float) + else: + valley_vwap = ok["valley_amt"] / ok["valley_vol"] + ridge_vwap = ok["ridge_amt"] / ok["ridge_vol"] + ratio = (valley_vwap / ridge_vwap).astype(float) + + if not with_diagnostics: + return ratio + diagnostics = agg[["classifiable_bars", "valley_bars", "ridge_bars"]].copy() + diagnostics["valid"] = valid + return ratio, diagnostics + + +def _valley_ridge_ratio_mean_for_symbol( + g: pd.DataFrame, + *, + baseline_days: int, + baseline_min_obs: int, + sigma_k: float, + lookback_days: int, + min_valid_days: int, + min_classifiable: int, + min_valley_bars: int, + min_ridge_bars: int, + collect_diagnostics: bool = False, +) -> tuple[list[pd.Timestamp], list[float], pd.DataFrame | None]: + """Daily valley/ridge VWAP-ratio values for ONE symbol from its PIT-visible bars. + + Classifies the minutes with the SHARED :func:`peak_mask_for_symbol`, reduces each + valid day to its VWAP ratio, then takes the trailing-``lookback_days``-valid-day + mean. No cross-symbol leakage (``g`` is one symbol's slice) and no lookahead (the + baseline is strictly prior, the mean window is trailing). + """ + work = peak_mask_for_symbol( + g, + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + ) + result = valley_ridge_vwap_ratio_by_day( + work, + min_valley_bars=min_valley_bars, + min_ridge_bars=min_ridge_bars, + min_classifiable=min_classifiable, + with_diagnostics=collect_diagnostics, + ) + if collect_diagnostics: + ratio, diagnostics = result + else: + ratio, diagnostics = result, None + if ratio.empty: + return [], [], diagnostics + + # Mean over the trailing lookback_days VALID days (including d); NaN until + # min_valid_days valid days have accumulated. Emitted only on valid days. + rolled = rolling_valid_days( + ratio, lookback_days=lookback_days, min_valid_days=min_valid_days + ).mean() + days = [pd.Timestamp(d).normalize() for d in rolled.index] + return days, list(rolled.to_numpy(dtype=float)), diagnostics + + +def compute_valley_ridge_vwap_ratio( + bars: pd.DataFrame, + *, + lookback_days: int = VALLEY_RIDGE_LOOKBACK_DAYS, + baseline_days: int = VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, + sigma_k: float = VOLUME_PRV_SIGMA_K, + min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + min_valley_bars: int = VALLEY_RIDGE_MIN_VALLEY_BARS, + min_ridge_bars: int = VALLEY_RIDGE_MIN_RIDGE_BARS, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "valley_ridge_vwap_ratio", + diagnostics_out: list | None = None, +) -> pd.Series: + """PIT-safe daily "valley/ridge VWAP ratio" factor from 1min ``bars``. + + Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, + classifies every visible minute with the SHARED PR-F taxonomy, computes each valid + day's ``valley VWAP / ridge VWAP`` ratio via the ``Σamount/Σvolume`` identity, and + returns the trailing-``lookback_days``-VALID-day mean of that ratio. See the module + docstring for the LOCKED definition and the seven pinned choices. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping is + strictly per symbol (no cross-symbol leakage). + lookback_days: trailing VALID trading-day window averaged (definition). + baseline_days: strictly-prior same-slot baseline window in trading days (PR-F). + baseline_min_obs: minimum same-slot observations for a classifiable bar (PR-F). + sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ`` (PR-F). + min_valid_days: minimum valid days in the trailing window for a finite value. + min_classifiable: a day needs at least this many classifiable bars (PR-F). + min_valley_bars: a day needs at least this many TRADABLE valley bars. + min_ridge_bars: a day needs at least this many TRADABLE ridge bars (PINNED lower + than the valley floor — see the module docstring §6). + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + name: the returned Series name (the factor-panel column name). + diagnostics_out: optional list the per-symbol day-level bar-count frames are + APPENDED to (each carries a ``symbol`` column), so a caller can report the + ridge-scarcity distribution. Purely observational — supplying it does not + change the returned factor. + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily + factor value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if baseline_days < 2: + # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is + # defined. + raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") + if baseline_min_obs < 2: + raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") + if sigma_k < 0.0: + raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") + if min_valid_days < 1: + raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") + if min_classifiable < 1: + raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") + if min_valley_bars < 1: + raise ValueError(f"min_valley_bars must be >= 1; got {min_valley_bars!r}.") + if min_ridge_bars < 1: + raise ValueError(f"min_ridge_bars must be >= 1; got {min_ridge_bars!r}.") + if len(bars) == 0: + return empty_factor_series(name) + + # extra_columns=("amount",) is the ONLY difference from the PR-F / PR-H entry + # points: the traded value rides along on the surviving rows, and the truncation / + # volume guard / slot assignment are untouched. + visible = prepare_visible_minute_bars( + bars, decision_time=decision_time, extra_columns=(_AMOUNT,) + ) + if visible.empty: + return empty_factor_series(name) + + collect = diagnostics_out is not None + index_tuples: list[tuple] = [] + values: list[float] = [] + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + days, vals, diagnostics = _valley_ridge_ratio_mean_for_symbol( + g.reset_index(drop=True), + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + lookback_days=lookback_days, + min_valid_days=min_valid_days, + min_classifiable=min_classifiable, + min_valley_bars=min_valley_bars, + min_ridge_bars=min_ridge_bars, + collect_diagnostics=collect, + ) + if collect and diagnostics is not None and not diagnostics.empty: + frame = diagnostics.copy() + frame[SYMBOL_LEVEL] = str(sym) + diagnostics_out.append(frame) + for day, val in zip(days, vals): + index_tuples.append((day, str(sym))) + values.append(val) + + if not index_tuples: + return empty_factor_series(name) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +class ValleyRidgeVwapRatioFactor(Factor): + """Valley/ridge VWAP-ratio factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_valley_ridge_vwap_ratio`); it does NO minute work of its + own, mirroring its siblings and the value / financial factors that surface an + enriched column. + + Args: + lookback_days: trailing VALID trading-day window averaged; part of the factor + DEFINITION (reproduced from the report), not a tuned knob. It only names the + column so a non-default window cannot silently mislabel it. + """ + + name: str = f"valley_ridge_vwap_ratio_{VALLEY_RIDGE_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = VALLEY_RIDGE_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"valley-ridge-vwap-ratio lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"valley_ridge_vwap_ratio_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=+1: the report's full-market RankIC is +6.98% (RankICIR 3.56, + long-short 15.83%/yr, IR 1.83, monthly win rate 72.3%, only 2023 negative across + 13 years), and its CSI500 sub-domain long-short is 10.49% / IR 1.34, the closest + comparable to our eval cell. Semantics per the report: a HIGH valley/ridge price + ratio means retail over-reaction pushed the eruptive minutes' price DOWN relative + to the calm ones, so the stock is depressed and performs better going forward. + The sign is fixed BEFORE the run (a validated prototype must reproduce it). NOTE + the report is a MONTHLY, market-cap + industry neutral full-market series on Wind + data while our eval cell is CSI500 daily with industry + size neutral, so the + report numbers are a LOOSE reference only (disclosed, never mislabeled, never + written in as an expected value). is_intraday=False by the module docstring's + reasoning: minute INPUT but a DAILY signal traded close-to-close. + min_history_bars=0: the warm-up is DATA-dependent (a value appears once enough + VALID days accumulate), not a fixed leading count — the honest NaN rate is + reported by data_coverage. + + The description spells out the RELATION TO PR-I (same shared classification, same + VWAP identity, DENOMINATOR swapped from the whole visible day to the ridge bars) + plus the pinned choices — above all the ASYMMETRIC bar floor, which exists + because ridge bars are structurally far scarcer than valley bars. + + D1 declarations (D0 pre-assignment table row 8): adjustment= + returns_invariant — the same same-day two-leg cancellation argument as + PR-I, with the denominator swapped to the ridge VWAP (module docstring + PINNED §4); price arrives via Σamount/Σvolume, so requires lists no + OHLC field. overnight_boundary=none — both legs are same-day. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Valley/ridge VWAP ratio (Kaiyuan microstructure series #27, FOURTH " + f"factor 谷岭加权价格比). SAME minute classification as PR-F " + f"volume_peak_count / PR-H peak_interval_kurtosis / PR-I " + f"valley_relative_vwap (SHARED taxonomy in factors.compute.minute." + f"primitives, not re-implemented): 1min bars PIT-truncated at 14:50, a " + f"minute is ERUPTIVE if vol > μ + {VOLUME_PRV_SIGMA_K:g}σ of its " + f"SAME-SLOT strictly-prior {VOLUME_PRV_BASELINE_DAYS}-day baseline, " + f"else it is a VALLEY (量谷). DENOMINATOR SWAPPED vs PR-I: instead of " + f"the whole visible day's VWAP the divisor is the RIDGE (量岭) VWAP, so " + f"the two behavioural groups are contrasted head-on — daily ratio = " + f"(valley VWAP) / (ridge VWAP), averaged over the trailing " + f"{self._lookback_days} VALID days. PINNED choices: (1) the RIDGE mask " + f"is 'eruptive AND NOT an isolated peak', which is WIDER than 'eruptive " + f"next to an eruptive' — it also covers session-boundary eruptions and " + f"eruptions with an unclassifiable neighbour, keeping valley|peak|ridge " + f"an exact partition of the classifiable bars; an isolated PEAK " + f"contributes to NEITHER leg; (2) each VWAP uses the aggregation " + f"identity Σ(p·v)/Σv = Σamount/Σvolume; (3) bars with non-finite or " + f"non-positive volume or amount are dropped from BOTH sums (guard " + f"applied at summation only, so PR-F's baseline is untouched); (4) RAW " + f"unadjusted prices are correct here because the adjustment factor is " + f"constant within a day and cancels in the ratio; (5) DEVIATION FROM " + f"THE REPORT, disclosed: both legs span the PIT-VISIBLE window " + f"09:31-14:50 only, not the full session — reading the close would be " + f"lookahead at our 14:50 decision time; (6) a day is VALID iff it has " + f">= {VOLUME_PRV_MIN_CLASSIFIABLE} classifiable bars AND >= " + f"{VALLEY_RIDGE_MIN_VALLEY_BARS} TRADABLE valley bars AND >= " + f"{VALLEY_RIDGE_MIN_RIDGE_BARS} TRADABLE ridge bars (both counted AFTER " + f"the guard) AND positive volume in both denominators — the ridge floor " + f"is deliberately LOWER because a ridge bar must erupt AND fail the " + f"isolation test, making ridges structurally far scarcer than valleys, " + f"and the realized ridge-bar distribution plus day-validity rate are " + f"REPORTED by the runner rather than left implicit; NaN below " + f"{VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived from 1min bars but a " + f"DAILY signal traded close-to-close." + ), + expected_ic_sign=1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar fields the upstream aggregation is derived from. Declared for + # honest provenance disclosure (data_coverage lists them); the daily panel + # surfaces the pre-aggregated column itself. + input_fields=("volume", "amount"), + requires=_minute_requires("volume", "amount"), + adjustment="returns_invariant", + overnight_boundary="none", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily valley/ridge VWAP-ratio column off ``panel``. + + The runner runs ``compute_valley_ridge_vwap_ratio`` per symbol on the minute + cache upstream and joins the result as ``self.name``; here we only surface it, so + this factor does no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"ValleyRidgeVwapRatioFactor needs the pre-aggregated '{self.name}' " + f"column on the panel (produced upstream by " + f"compute_valley_ridge_vwap_ratio and joined by the runner); panel has " + f"{list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "DIAGNOSTIC_COLUMNS", + "VALLEY_RIDGE_LOOKBACK_DAYS", + "VALLEY_RIDGE_MIN_RIDGE_BARS", + "VALLEY_RIDGE_MIN_VALLEY_BARS", + "ValleyRidgeVwapRatioFactor", + "compute_valley_ridge_vwap_ratio", + "valley_ridge_vwap_ratio_by_day", +] diff --git a/factors/compute/minute/volume_peak_count.py b/factors/compute/minute/volume_peak_count.py new file mode 100644 index 0000000..07da815 --- /dev/null +++ b/factors/compute/minute/volume_peak_count.py @@ -0,0 +1,318 @@ +"""Volume-peak-count factor (PR-F): math + surface (D2 one-factor-one-file). + +Reproduces the Kaiyuan market-microstructure series #27 (开源证券《高频成交量的峰、岭、 +谷信息——市场微观结构研究系列（27）》, 2025-07-20, reportId 4957417) flagship +"volume-peak-minute-count" factor (§2) as a daily PIT-safe column derived DIRECTLY +from the 1min cache (no coarser resampling — the peak/ridge/valley taxonomy lives at +the 1-minute grain). The taxonomy itself (``prepare_visible_minute_bars`` + +``peak_mask_for_symbol``) lives in :mod:`factors.compute.minute.primitives`; this +file reduces it to the peak COUNT. + +The report is under-specified about the PIT boundary; the interpretations below are +deliberate, DISCLOSED choices PINNED in the task card (task_card_pr_f_*.md §1), not +tuned knobs. They are reproduced on the factor spec so a reader can see exactly what +was assumed: + + 1. PIT truncation (standing authorization): each day keeps only the 1min bars with + ``available_time <= (that bar's trade_date + decision_time)`` (default 14:50) — + history days AND the signal day are truncated identically, so the same-slot + cross-day baseline is measured on a consistent window (≈ the morning session plus + the afternoon up to the cutoff). + 2. Same-slot baseline (PINNED as STRICTLY PRIOR — the report does not say whether + the current day is included, and strictly-prior is the more PIT-stable reading): + for minute slot ``s`` and day ``t`` the baseline ``μ_s`` / ``σ_s`` (ddof=1) is + the symbol's SAME-SLOT volume over the trailing ``baseline_days`` (=20) trading + days STRICTLY BEFORE ``t``; fewer than ``baseline_min_obs`` (=10) same-slot + observations in that window -> the ``(t, s)`` bar is NOT classifiable. + 3. classify: ``vol > μ_s + k*σ_s`` (k = 1) -> ERUPTIVE, else MILD (a "valley"). + 4. a slot is a PEAK iff it is eruptive AND both its 1-minute neighbours in the SAME + continuous session exist and are MILD (see ``peak_mask_for_symbol``). + +Factor value: ``volume_peak_count_20`` = the total count of peak minutes over the +symbol's most recent ``lookback_days`` (=20) VALID trading days INCLUDING ``d`` (a +simple count — after truncation the per-day slot count is uniform, so counts are +cross-sectionally comparable without normalization). A day is VALID iff it has at +least ``min_classifiable`` (=100) classifiable bars; fewer than ``min_valid_days`` +(=10) valid days in the trailing window -> NaN (honest missing; the runner discloses +the coverage). Values are emitted only on valid days. + +Pre-registered sign = +1 (more volume peaks = more informed-trading participation = +higher future returns; the report's full-market RankIC is +10.62% / RankICIR 4.36 and +its CSI500 sub-domain long-short is +14.96%/yr — the CSI500 line is the direct anchor +for our eval cell). NOTE the report is a monthly, market-cap + industry neutral series +on Wind data; our eval cell is CSI500 daily with industry + size neutral, so the +report numbers are a LOOSE reference only (disclosed, never mislabeled). Raw minute +volume (cached as-is) has magnitude jumps across split days that pollute the 20-day σ; +the report (Wind) does not adjust for this either, so we disclose it and do NOT correct +it. The value at ``(d, s)`` uses only bars at dates <= d, so a factor value never sees +a future bar (invariant #1); it is a DAILY signal traded close-to-close from d+1. + +The factor math is pure and never fetches; the ``VolumePeakCountFactor`` surface +class only SELECTS the pre-aggregated column off the enriched panel. +""" + +from __future__ import annotations + +import pandas as pd + +from data.availability_policy import STK_MINS_1MIN +from data.clean.intraday_schema import ( + DAILY_INDEX_NAMES, + DEFAULT_DECISION_TIME, + SYMBOL_LEVEL, + validate_intraday_bars, +) +from factors.base import Factor +from factors.compute.minute.primitives import ( + VOLUME_PRV_BASELINE_DAYS, + VOLUME_PRV_BASELINE_MIN_OBS, + VOLUME_PRV_MIN_CLASSIFIABLE, + VOLUME_PRV_MIN_VALID_DAYS, + VOLUME_PRV_SIGMA_K, + empty_factor_series, + peak_mask_for_symbol, + prepare_visible_minute_bars, +) +from factors.spec import FactorSpec, PanelField + +# Factor DEFINITION constant (pinned interpretation of the report; NOT a tuned knob). +# The classification constants live with the taxonomy in ``primitives``. +VOLUME_PRV_LOOKBACK_DAYS = 20 # trailing VALID trading-day count window (N), includes d + + +def _minute_requires(*fields: str) -> tuple[PanelField, ...]: + """The stk_mins_1min requires tuple of a minute-derived factor (D1).""" + return tuple(PanelField(f, source=STK_MINS_1MIN) for f in fields) + + +def _peak_count_for_symbol( + g: pd.DataFrame, + *, + baseline_days: int, + baseline_min_obs: int, + sigma_k: float, + lookback_days: int, + min_valid_days: int, + min_classifiable: int, +) -> tuple[list[pd.Timestamp], list[float]]: + """Daily volume-peak-count values for ONE symbol from its PIT-visible bars. + + Identifies the peaks with the shared :func:`peak_mask_for_symbol` and reduces them to + the trailing-``lookback_days``-VALID-day count. + """ + work = peak_mask_for_symbol( + g, + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + ) + + peak_count = work.groupby("trade_date")["peak"].sum() + classifiable_count = work.groupby("trade_date")["classifiable"].sum() + valid_days = classifiable_count.index[classifiable_count >= min_classifiable] + if len(valid_days) == 0: + return [], [] + + # Count over the trailing lookback_days VALID days (including d); NaN until + # min_valid_days valid days have accumulated. Values are emitted only on valid days. + pc_valid = peak_count.loc[valid_days].astype(float).sort_index() + factor_valid = pc_valid.rolling(lookback_days, min_periods=min_valid_days).sum() + days = [pd.Timestamp(d).normalize() for d in factor_valid.index] + return days, list(factor_valid.to_numpy(dtype=float)) + + +def compute_volume_peak_count( + bars: pd.DataFrame, + *, + lookback_days: int = VOLUME_PRV_LOOKBACK_DAYS, + baseline_days: int = VOLUME_PRV_BASELINE_DAYS, + baseline_min_obs: int = VOLUME_PRV_BASELINE_MIN_OBS, + sigma_k: float = VOLUME_PRV_SIGMA_K, + min_valid_days: int = VOLUME_PRV_MIN_VALID_DAYS, + min_classifiable: int = VOLUME_PRV_MIN_CLASSIFIABLE, + decision_time: str = DEFAULT_DECISION_TIME, + name: str = "volume_peak_count", +) -> pd.Series: + """PIT-safe daily "volume-peak-minute-count" factor from 1min ``bars``. + + Takes normalized 1min ``bars``, PIT-truncates each day at ``decision_time``, + classifies every visible minute against its SAME-SLOT strictly-prior baseline + (eruptive vs mild), marks the eruptive minutes whose both 1-minute same-session + neighbours are mild as PEAKS, and returns the trailing-``lookback_days``-VALID-day + peak count. See the module docstring for the LOCKED definition. + + Args: + bars: normalized 1min bars (:mod:`data.clean.intraday_schema`), + ``MultiIndex(time, symbol)``. May carry one or many symbols; the grouping + is strictly per symbol (no cross-symbol leakage). + lookback_days: trailing VALID trading-day window for the count (definition). + baseline_days: strictly-prior same-slot baseline window in trading days. + baseline_min_obs: minimum same-slot observations for a classifiable bar. + sigma_k: eruptive threshold multiplier ``k`` in ``vol > μ + k*σ``. + min_valid_days: minimum valid days in the trailing window for a finite value. + min_classifiable: a day is VALID iff it has at least this many classifiable bars. + decision_time: per-bar PIT cutoff time-of-day (default 14:50:00). + name: the returned Series name (the factor-panel column name). + + Returns: + ``MultiIndex(date, symbol)`` Series (midnight-normalized dates) of the daily + factor value, sorted, named ``name``. Pure: never mutates ``bars``. + """ + validate_intraday_bars(bars) + if lookback_days < 1: + raise ValueError(f"lookback_days must be >= 1; got {lookback_days!r}.") + if baseline_days < 2: + # Need >= 2 baseline observations so a ddof=1 std of the same-slot volume is + # defined. + raise ValueError(f"baseline_days must be >= 2; got {baseline_days!r}.") + if baseline_min_obs < 2: + raise ValueError(f"baseline_min_obs must be >= 2; got {baseline_min_obs!r}.") + if sigma_k < 0.0: + raise ValueError(f"sigma_k must be >= 0; got {sigma_k!r}.") + if min_valid_days < 1: + raise ValueError(f"min_valid_days must be >= 1; got {min_valid_days!r}.") + if min_classifiable < 1: + raise ValueError(f"min_classifiable must be >= 1; got {min_classifiable!r}.") + if len(bars) == 0: + return empty_factor_series(name) + + visible = prepare_visible_minute_bars(bars, decision_time=decision_time) + if visible.empty: + return empty_factor_series(name) + + index_tuples: list[tuple] = [] + values: list[float] = [] + for sym, g in visible.groupby(SYMBOL_LEVEL, sort=True): + days, vals = _peak_count_for_symbol( + g.reset_index(drop=True), + baseline_days=baseline_days, + baseline_min_obs=baseline_min_obs, + sigma_k=sigma_k, + lookback_days=lookback_days, + min_valid_days=min_valid_days, + min_classifiable=min_classifiable, + ) + for day, val in zip(days, vals): + index_tuples.append((day, str(sym))) + values.append(val) + + if not index_tuples: + return empty_factor_series(name) + index = pd.MultiIndex.from_tuples(index_tuples, names=DAILY_INDEX_NAMES) + return pd.Series(values, index=index, name=name).sort_index() + + +class VolumePeakCountFactor(Factor): + """Volume-peak-minute-count factor (daily signal, minute-derived). + + ``compute`` reads the pre-aggregated daily column the runner placed on the panel + (produced by :func:`compute_volume_peak_count`); it does NO minute work of its + own, mirroring the value / financial factors that surface an enriched column. + + Args: + lookback_days: trailing VALID trading-day count window; part of the factor + DEFINITION (a pinned interpretation of the report), not a tuned knob. It + only names the column so a non-default window cannot silently mislabel it. + """ + + name: str = f"volume_peak_count_{VOLUME_PRV_LOOKBACK_DAYS}" + + def __init__(self, lookback_days: int = VOLUME_PRV_LOOKBACK_DAYS) -> None: + if not isinstance(lookback_days, int) or lookback_days < 1: + raise ValueError( + f"volume-peak-count lookback_days must be a positive integer; got " + f"{lookback_days!r}." + ) + self._lookback_days = lookback_days + self.name = f"volume_peak_count_{lookback_days}" + + @property + def lookback_days(self) -> int: + return self._lookback_days + + @property + def spec(self) -> FactorSpec: + """Evaluation contract; a property so ``factor_id`` tracks the window. + + expected_ic_sign=+1: the report's IC is POSITIVE (full-market RankIC +10.62% / + RankICIR 4.36; CSI500 sub-domain long-short +14.96%/yr) — more volume peaks + (informed-trading participation) predicts HIGHER forward returns. The sign is + fixed BEFORE the run (a validated prototype must reproduce it). NOTE the report + is a MONTHLY, market-cap + industry neutral series on Wind data while our eval + cell is CSI500 daily with industry + size neutral, so the report numbers are a + LOOSE reference only (disclosed, never mislabeled). is_intraday=False by the + module docstring's reasoning: minute INPUT but a DAILY signal traded + close-to-close. min_history_bars=0: the warm-up is DATA-dependent (a value + appears once >= ``VOLUME_PRV_MIN_VALID_DAYS`` valid days accumulate in the + trailing window, and a day is valid only once its same-slot baselines fill in), + not a fixed leading count — the honest NaN rate is reported by data_coverage. + + The description spells out the pinned interpretations of the under-specified + report (PIT truncation, strictly-prior same-slot baseline, μ+σ eruptive + threshold, mild-neighbour peak rule, valid-day gate) so a reader sees exactly + what was assumed. + + D1 declarations (D0 pre-assignment table row 4 + note 3): adjustment= + none — the factor reads the pure volume channel, never a price + (this module's docstring; taxonomy in factors/compute/minute/ + primitives.py). overnight_boundary=none — no raw-price comparison + exists. The KNOWN raw-volume magnitude jump across split days is + disclosed at the definition site and deliberately NOT corrected + (report alignment); per D0 note 3 it belongs to NEITHER taxonomy + axis — do not "fix" it via these declarations. + """ + return FactorSpec( + factor_id=self.name, + version="1.0", + description=( + f"Volume-peak-minute count (Kaiyuan microstructure series #27). PINNED " + f"interpretations of an under-specified report: (1) 1min bars " + f"PIT-truncated at 14:50 per bar; (2) same-slot baseline = μ/σ (ddof=1) " + f"of the STRICTLY-PRIOR {VOLUME_PRV_BASELINE_DAYS} trading days' " + f"same-slot volume, needing >= {VOLUME_PRV_BASELINE_MIN_OBS} obs else " + f"unclassifiable; (3) a minute is ERUPTIVE if vol > μ + " + f"{VOLUME_PRV_SIGMA_K:g}σ else MILD; (4) a PEAK is an eruptive minute " + f"whose both 1-minute same-session neighbours exist and are mild " + f"(ridge / session-boundary / unclassifiable-neighbour minutes are not " + f"peaks); (5) factor = peak-minute count over the trailing " + f"{self._lookback_days} VALID days (>= {VOLUME_PRV_MIN_CLASSIFIABLE} " + f"classifiable bars) including d, NaN below " + f"{VOLUME_PRV_MIN_VALID_DAYS} valid days. Derived from 1min bars but a " + f"DAILY signal traded close-to-close." + ), + expected_ic_sign=1, + is_intraday=False, + forward_return_horizon=1, + return_basis="close_to_close", + # The 1min bar field the upstream aggregation is derived from. Declared for + # honest provenance disclosure (data_coverage lists it); the daily panel + # surfaces the pre-aggregated column itself. + input_fields=("volume",), + requires=_minute_requires("volume"), + adjustment="none", + overnight_boundary="none", + family="microstructure", + min_history_bars=0, + ) + + def compute(self, panel: pd.DataFrame) -> pd.Series: + """Select the pre-aggregated daily volume-peak-count column off ``panel``. + + The runner runs ``compute_volume_peak_count`` on the minute cache upstream and + joins the result as ``self.name``; here we only surface it, so this factor does + no temporal logic and cannot introduce lookahead. + """ + if self.name not in panel.columns: + raise ValueError( + f"VolumePeakCountFactor needs the pre-aggregated '{self.name}' column " + f"on the panel (produced upstream by compute_volume_peak_count and " + f"joined by the runner); panel has {list(panel.columns)}." + ) + return panel[self.name].rename(self.name) + + +__all__ = [ + "VOLUME_PRV_LOOKBACK_DAYS", + "VolumePeakCountFactor", + "compute_volume_peak_count", +] diff --git a/factors/registry/builtin.py b/factors/registry/builtin.py index 593bc08..f372853 100644 --- a/factors/registry/builtin.py +++ b/factors/registry/builtin.py @@ -25,17 +25,6 @@ from collections.abc import Mapping -from data.clean.intraday_aggregate import JUMP_LOOKBACK_DAYS -from data.clean.intraday_amount_ratio import PEAK_RIDGE_LOOKBACK_DAYS -from data.clean.intraday_amp_anomaly import AMP_ANOMALY_LOOKBACK_DAYS -from data.clean.intraday_amp_cut import AMP_CUT_LOOKBACK_DAYS -from data.clean.intraday_amplitude import IDEAL_AMP_LOOKBACK_DAYS -from data.clean.intraday_peak_interval import PEAK_INTERVAL_LOOKBACK_DAYS -from data.clean.intraday_ridge_return import RIDGE_RETURN_LOOKBACK_DAYS -from data.clean.intraday_valley_quantile import VALLEY_QUANTILE_LOOKBACK_DAYS -from data.clean.intraday_valley_ridge_vwap import VALLEY_RIDGE_LOOKBACK_DAYS -from data.clean.intraday_valley_vwap import VALLEY_VWAP_LOOKBACK_DAYS -from data.clean.intraday_volume_prv import VOLUME_PRV_LOOKBACK_DAYS from factors.compute.candidates import ( VALUE_FIELDS, LiquidityFactor, @@ -45,17 +34,48 @@ VolatilityFactor, ) from factors.compute.financial import SUPPORTED_FIELDS, FinancialFactor -from factors.compute.intraday_derived import ( +from factors.compute.minute.amp_marginal_anomaly_vol import ( + AMP_ANOMALY_LOOKBACK_DAYS, AmpMarginalAnomalyVolFactor, +) +from factors.compute.minute.intraday_amp_cut import ( + AMP_CUT_LOOKBACK_DAYS, IntradayAmpCutFactor, +) +from factors.compute.minute.jump_amount_corr import ( + JUMP_LOOKBACK_DAYS, JumpAmountCorrFactor, +) +from factors.compute.minute.minute_ideal_amplitude import ( + IDEAL_AMP_LOOKBACK_DAYS, MinuteIdealAmplitudeFactor, +) +from factors.compute.minute.peak_interval_kurtosis import ( + PEAK_INTERVAL_LOOKBACK_DAYS, PeakIntervalKurtosisFactor, +) +from factors.compute.minute.peak_ridge_amount_ratio import ( + PEAK_RIDGE_LOOKBACK_DAYS, PeakRidgeAmountRatioFactor, +) +from factors.compute.minute.ridge_minute_return import ( + RIDGE_RETURN_LOOKBACK_DAYS, RidgeMinuteReturnFactor, +) +from factors.compute.minute.valley_price_quantile import ( + VALLEY_QUANTILE_LOOKBACK_DAYS, ValleyPriceQuantileFactor, +) +from factors.compute.minute.valley_relative_vwap import ( + VALLEY_VWAP_LOOKBACK_DAYS, ValleyRelativeVwapFactor, +) +from factors.compute.minute.valley_ridge_vwap_ratio import ( + VALLEY_RIDGE_LOOKBACK_DAYS, ValleyRidgeVwapRatioFactor, +) +from factors.compute.minute.volume_peak_count import ( + VOLUME_PRV_LOOKBACK_DAYS, VolumePeakCountFactor, ) from factors.compute.momentum import MomentumFactor diff --git a/tests/test_amp_marginal_anomaly_vol_factor.py b/tests/test_amp_marginal_anomaly_vol_factor.py index b478f7a..d293789 100644 --- a/tests/test_amp_marginal_anomaly_vol_factor.py +++ b/tests/test_amp_marginal_anomaly_vol_factor.py @@ -19,16 +19,16 @@ import pytest from data.clean.intraday_aggregate import resample_intraday_bars -from data.clean.intraday_amp_anomaly import ( +from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars +from factors.compute.minute.amp_marginal_anomaly_vol import ( AMP_ANOMALY_FREQ, AMP_ANOMALY_LOOKBACK_DAYS, AMP_ANOMALY_MIN_POOL, AMP_ANOMALY_MIN_SELECTED, AMP_ANOMALY_SIGMA_K, + AmpMarginalAnomalyVolFactor, compute_amp_marginal_anomaly_vol, ) -from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from factors.compute.intraday_derived import AmpMarginalAnomalyVolFactor from factors.spec import FactorSpec _SYM = "000001.SZ" diff --git a/tests/test_intraday_amp_cut_factor.py b/tests/test_intraday_amp_cut_factor.py index 101ed8c..497dd95 100644 --- a/tests/test_intraday_amp_cut_factor.py +++ b/tests/test_intraday_amp_cut_factor.py @@ -21,20 +21,20 @@ import pandas as pd import pytest -from data.clean.intraday_amp_cut import ( +from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars +from factors.compute.minute.intraday_amp_cut import ( AMP_CUT_LAMBDA, AMP_CUT_LOOKBACK_DAYS, AMP_CUT_MIN_CROSS_SECTION, AMP_CUT_MIN_DAY_MINUTES, AMP_CUT_MIN_VALID_DAYS, + IntradayAmpCutFactor, V_MEAN_COL, V_STD_COL, combine_amp_cut_cross_section, compute_amp_cut_stats, compute_intraday_amp_cut, ) -from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from factors.compute.intraday_derived import IntradayAmpCutFactor from factors.spec import FactorSpec _SYM = "000001.SZ" diff --git a/tests/test_jump_amount_corr_factor.py b/tests/test_jump_amount_corr_factor.py index f231344..5bd8987 100644 --- a/tests/test_jump_amount_corr_factor.py +++ b/tests/test_jump_amount_corr_factor.py @@ -6,9 +6,11 @@ import pandas as pd import pytest -from data.clean.intraday_aggregate import compute_jump_amount_corr from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from factors.compute.intraday_derived import JumpAmountCorrFactor +from factors.compute.minute.jump_amount_corr import ( + JumpAmountCorrFactor, + compute_jump_amount_corr, +) from factors.spec import FactorSpec _SYM = "000001.SZ" diff --git a/tests/test_minute_ideal_amplitude_factor.py b/tests/test_minute_ideal_amplitude_factor.py index 7d57d9b..c30143e 100644 --- a/tests/test_minute_ideal_amplitude_factor.py +++ b/tests/test_minute_ideal_amplitude_factor.py @@ -13,14 +13,14 @@ import pandas as pd import pytest -from data.clean.intraday_amplitude import ( +from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars +from factors.compute.minute.minute_ideal_amplitude import ( IDEAL_AMP_LAMBDA, IDEAL_AMP_LOOKBACK_DAYS, IDEAL_AMP_MIN_MINUTES, + MinuteIdealAmplitudeFactor, compute_minute_ideal_amplitude, ) -from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from factors.compute.intraday_derived import MinuteIdealAmplitudeFactor from factors.spec import FactorSpec _SYM = "000001.SZ" diff --git a/tests/test_minute_shims.py b/tests/test_minute_shims.py new file mode 100644 index 0000000..01a1815 --- /dev/null +++ b/tests/test_minute_shims.py @@ -0,0 +1,163 @@ +"""D2 shim discipline tests (design v3.2 §6.4 / R14). + +Two independent locks: + +1. **Purity** — each FULLY-migrated module (the 10 pure ``data/clean`` + intraday factor modules + ``factors/compute/intraday_derived.py``) must + contain ONLY a module docstring, ``from ... import ...`` statements and an + ``__all__`` assignment. Any function/class/constant defined in a shim would + be a resurrected second definition point — exactly the drift §6.4 forbids. + R14 scope note: ``intraday_schema`` and the ``intraday_aggregate`` generic + core keep REAL code and are deliberately NOT purity-tested (mixed modules); + testing them would force the implementer to quietly relax the check. + +2. **Re-export identity** — every public name a shim re-exports must be the + SAME OBJECT as the one defined in ``factors.compute.minute`` (``is``, not + ``==``). This is what makes "the pre-existing property tests exercise the + NEW engine" a proven fact rather than an assumption: the shim path and the + new path cannot diverge because there is only one object. + +Mutation evidence (run for this commit, recorded in the acceptance report): +adding a stray ``def _local(): pass`` to ``data/clean/intraday_amplitude.py`` +makes ``test_fully_migrated_shims_are_import_only`` fail (rc=1); reverting it +passes (rc=0). Rebinding a shim name to a wrapper function makes +``test_shim_reexports_are_identical_objects`` fail. +""" + +from __future__ import annotations + +import ast +import importlib +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parent.parent + +# The FULLY-migrated modules (R14: the only ones the purity test may cover). +_PURE_SHIMS = [ + "data/clean/intraday_amount_ratio.py", + "data/clean/intraday_amp_anomaly.py", + "data/clean/intraday_amp_cut.py", + "data/clean/intraday_amplitude.py", + "data/clean/intraday_peak_interval.py", + "data/clean/intraday_ridge_return.py", + "data/clean/intraday_valley_quantile.py", + "data/clean/intraday_valley_ridge_vwap.py", + "data/clean/intraday_valley_vwap.py", + "data/clean/intraday_volume_prv.py", + "factors/compute/intraday_derived.py", +] + +# shim module -> the factors.compute.minute module(s) its names must resolve to. +_SHIM_SOURCES = { + "data.clean.intraday_amount_ratio": ["factors.compute.minute.peak_ridge_amount_ratio"], + "data.clean.intraday_amp_anomaly": ["factors.compute.minute.amp_marginal_anomaly_vol"], + "data.clean.intraday_amp_cut": ["factors.compute.minute.intraday_amp_cut"], + "data.clean.intraday_amplitude": ["factors.compute.minute.minute_ideal_amplitude"], + "data.clean.intraday_peak_interval": ["factors.compute.minute.peak_interval_kurtosis"], + "data.clean.intraday_ridge_return": ["factors.compute.minute.ridge_minute_return"], + "data.clean.intraday_valley_quantile": ["factors.compute.minute.valley_price_quantile"], + "data.clean.intraday_valley_ridge_vwap": [ + "factors.compute.minute.valley_ridge_vwap_ratio" + ], + "data.clean.intraday_valley_vwap": ["factors.compute.minute.valley_relative_vwap"], + "data.clean.intraday_volume_prv": [ + "factors.compute.minute.volume_peak_count", + "factors.compute.minute.primitives", + ], + "factors.compute.intraday_derived": [ + "factors.compute.minute.amp_marginal_anomaly_vol", + "factors.compute.minute.intraday_amp_cut", + "factors.compute.minute.jump_amount_corr", + "factors.compute.minute.minute_ideal_amplitude", + "factors.compute.minute.peak_interval_kurtosis", + "factors.compute.minute.peak_ridge_amount_ratio", + "factors.compute.minute.ridge_minute_return", + "factors.compute.minute.valley_price_quantile", + "factors.compute.minute.valley_relative_vwap", + "factors.compute.minute.valley_ridge_vwap_ratio", + "factors.compute.minute.volume_peak_count", + ], +} + +# Factor math re-exported by the MIXED aggregate module (kept real code; its +# re-exports must still be identity-equal to the migrated definitions). +_AGGREGATE_REEXPORTS = { + "compute_jump_amount_corr": "factors.compute.minute.jump_amount_corr", + "JUMP_LOOKBACK_DAYS": "factors.compute.minute.jump_amount_corr", + "JUMP_MIN_PAIRS": "factors.compute.minute.jump_amount_corr", + "JUMP_Z": "factors.compute.minute.jump_amount_corr", + "compute_minute_mmp": "factors.compute.minute.mmp", + "mmp_valid_minute_counts": "factors.compute.minute.mmp", + "MMP_LOOKBACK": "factors.compute.minute.mmp", + "DEFAULT_EPSILON": "factors.compute.minute.mmp", +} + + +def _is_docstring(node: ast.stmt) -> bool: + return isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance( + node.value.value, str + ) + + +def _is_all_assignment(node: ast.stmt) -> bool: + return ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == "__all__" + ) + + +@pytest.mark.parametrize("rel_path", _PURE_SHIMS) +def test_fully_migrated_shims_are_import_only(rel_path): + """Every fully-migrated module is docstring + imports + ``__all__`` — nothing else.""" + tree = ast.parse((_REPO / rel_path).read_text(encoding="utf-8")) + for i, node in enumerate(tree.body): + if i == 0 and _is_docstring(node): + continue + if isinstance(node, ast.ImportFrom): + continue + if _is_all_assignment(node): + continue + raise AssertionError( + f"{rel_path} is a D2 re-export shim but contains a " + f"{type(node).__name__} at line {node.lineno} — shims may only " + f"import and declare __all__ (design v3.2 §6.4)." + ) + + +@pytest.mark.parametrize("shim_name", sorted(_SHIM_SOURCES)) +def test_shim_reexports_are_identical_objects(shim_name): + """Each shim ``__all__`` name IS the object defined in factors.compute.minute.""" + shim = importlib.import_module(shim_name) + sources = [importlib.import_module(m) for m in _SHIM_SOURCES[shim_name]] + assert shim.__all__, f"{shim_name} must declare a non-empty __all__" + for name in shim.__all__: + obj = getattr(shim, name) + homes = [src for src in sources if getattr(src, name, None) is obj] + assert homes, ( + f"{shim_name}.{name} is not identical to any declared source object — " + f"the shim has drifted from the single definition point." + ) + + +def test_aggregate_reexports_are_identical_objects(): + """The MIXED aggregate module's factor-math re-exports point at the new homes.""" + agg = importlib.import_module("data.clean.intraday_aggregate") + for name, src_name in _AGGREGATE_REEXPORTS.items(): + src = importlib.import_module(src_name) + assert getattr(agg, name) is getattr(src, name), ( + f"data.clean.intraday_aggregate.{name} is not the object defined in " + f"{src_name} — the re-export has drifted." + ) + + +def test_purity_scope_is_exactly_the_fully_migratable_set(): + """R14: the purity list covers the 10 pure data/clean factor modules + the + surface shim, and deliberately EXCLUDES the mixed/kept-real modules.""" + covered = set(_PURE_SHIMS) + assert "data/clean/intraday_schema.py" not in covered + assert "data/clean/intraday_aggregate.py" not in covered + assert len(covered) == 11 diff --git a/tests/test_peak_interval_kurtosis_factor.py b/tests/test_peak_interval_kurtosis_factor.py index 7d04c50..df1d2cd 100644 --- a/tests/test_peak_interval_kurtosis_factor.py +++ b/tests/test_peak_interval_kurtosis_factor.py @@ -19,23 +19,23 @@ import pandas as pd import pytest -from data.clean.intraday_peak_interval import ( +from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars +from factors.compute.minute.peak_interval_kurtosis import ( PEAK_INTERVAL_LOOKBACK_DAYS, PEAK_INTERVAL_MIN_INTERVALS, + PeakIntervalKurtosisFactor, compute_peak_interval_kurtosis, excess_kurtosis, peak_intervals_by_day, ) -from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from data.clean.intraday_volume_prv import ( +from factors.compute.minute.primitives import ( VOLUME_PRV_BASELINE_DAYS, VOLUME_PRV_BASELINE_MIN_OBS, VOLUME_PRV_SIGMA_K, - compute_volume_peak_count, peak_mask_for_symbol, prepare_visible_minute_bars, ) -from factors.compute.intraday_derived import PeakIntervalKurtosisFactor +from factors.compute.minute.volume_peak_count import compute_volume_peak_count from factors.spec import FactorSpec _SYM = "000001.SZ" diff --git a/tests/test_peak_ridge_amount_ratio_factor.py b/tests/test_peak_ridge_amount_ratio_factor.py index a3ec4d5..b105bcc 100644 --- a/tests/test_peak_ridge_amount_ratio_factor.py +++ b/tests/test_peak_ridge_amount_ratio_factor.py @@ -36,16 +36,17 @@ import pandas as pd import pytest -import data.clean.intraday_amount_ratio as amount_ratio_mod -from data.clean.intraday_amount_ratio import ( +import factors.compute.minute.peak_ridge_amount_ratio as amount_ratio_mod +from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars +from factors.compute.minute.peak_ridge_amount_ratio import ( PEAK_RIDGE_LOOKBACK_DAYS, PEAK_RIDGE_MIN_PEAK_BARS, PEAK_RIDGE_MIN_RIDGE_BARS, + PeakRidgeAmountRatioFactor, compute_peak_ridge_amount_ratio, peak_ridge_amount_by_day, ) -from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from data.clean.intraday_volume_prv import ( +from factors.compute.minute.primitives import ( VOLUME_PRV_BASELINE_DAYS, VOLUME_PRV_BASELINE_MIN_OBS, VOLUME_PRV_MIN_CLASSIFIABLE, @@ -54,7 +55,6 @@ peak_mask_for_symbol, prepare_visible_minute_bars, ) -from factors.compute.intraday_derived import PeakRidgeAmountRatioFactor from factors.spec import FactorSpec _SYM = "000001.SZ" diff --git a/tests/test_ridge_minute_return_factor.py b/tests/test_ridge_minute_return_factor.py index f063ddf..d34cea2 100644 --- a/tests/test_ridge_minute_return_factor.py +++ b/tests/test_ridge_minute_return_factor.py @@ -34,14 +34,8 @@ import pandas as pd import pytest -from data.clean.intraday_ridge_return import ( - RIDGE_RETURN_LOOKBACK_DAYS, - RIDGE_RETURN_MIN_RIDGE_BARS, - compute_ridge_minute_return, - ridge_minute_return_by_day, -) from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from data.clean.intraday_volume_prv import ( +from factors.compute.minute.primitives import ( VOLUME_PRV_BASELINE_DAYS, VOLUME_PRV_BASELINE_MIN_OBS, VOLUME_PRV_MIN_VALID_DAYS, @@ -49,7 +43,13 @@ peak_mask_for_symbol, prepare_visible_minute_bars, ) -from factors.compute.intraday_derived import RidgeMinuteReturnFactor +from factors.compute.minute.ridge_minute_return import ( + RIDGE_RETURN_LOOKBACK_DAYS, + RIDGE_RETURN_MIN_RIDGE_BARS, + RidgeMinuteReturnFactor, + compute_ridge_minute_return, + ridge_minute_return_by_day, +) from factors.spec import FactorSpec _SYM = "000001.SZ" @@ -717,7 +717,7 @@ def _prv_kw(**over): def test_volume_peak_count_unchanged_by_pr_k(): """PR-F's factor value is bit-identical (the PR-J lock's hand case, verbatim).""" - from data.clean.intraday_volume_prv import compute_volume_peak_count + from factors.compute.minute.volume_peak_count import compute_volume_peak_count closes, vols = _case_b_day() rows = _background(_BG_DAYS, _CASE_B_N) + _session("2021-07-11", closes, vols) @@ -729,7 +729,7 @@ def test_volume_peak_count_unchanged_by_pr_k(): def test_peak_interval_kurtosis_unchanged_by_pr_k(): """PR-H's factor value is bit-identical (the PR-H hand case, verbatim).""" - from data.clean.intraday_peak_interval import compute_peak_interval_kurtosis + from factors.compute.minute.peak_interval_kurtosis import compute_peak_interval_kurtosis n = 25 vols = [100.0] * n @@ -746,7 +746,7 @@ def test_valley_relative_vwap_unchanged_by_pr_k(): This one needs AMOUNTS set independently of the closes, so it builds its own frame rather than going through ``_bars`` (whose amount is close * volume). """ - from data.clean.intraday_valley_vwap import compute_valley_relative_vwap + from factors.compute.minute.valley_relative_vwap import compute_valley_relative_vwap n = 12 erupt = (3, 7) @@ -765,7 +765,7 @@ def test_valley_relative_vwap_unchanged_by_pr_k(): def test_valley_ridge_vwap_ratio_unchanged_by_pr_k(): """PR-J's factor value is bit-identical (the PR-J case-A hand value, verbatim).""" - from data.clean.intraday_valley_ridge_vwap import compute_valley_ridge_vwap_ratio + from factors.compute.minute.valley_ridge_vwap_ratio import compute_valley_ridge_vwap_ratio n = 16 ridges, peak = (3, 4, 8, 9), 12 diff --git a/tests/test_valley_price_quantile_factor.py b/tests/test_valley_price_quantile_factor.py index 61997b1..48e4311 100644 --- a/tests/test_valley_price_quantile_factor.py +++ b/tests/test_valley_price_quantile_factor.py @@ -43,27 +43,27 @@ import pytest from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from data.clean.intraday_valley_quantile import ( +from factors.compute.minute.primitives import ( + VOLUME_PRV_BASELINE_DAYS, + VOLUME_PRV_BASELINE_MIN_OBS, + VOLUME_PRV_MIN_CLASSIFIABLE, + VOLUME_PRV_MIN_VALID_DAYS, + VOLUME_PRV_SIGMA_K, + peak_mask_for_symbol, + prepare_visible_minute_bars, +) +from factors.compute.minute.valley_price_quantile import ( VALLEY_QUANTILE_LOOKBACK_DAYS, VALLEY_QUANTILE_MIN_CROSS_SECTION, VALLEY_QUANTILE_MIN_VALLEY_BARS, VALLEY_QUANTILE_REVERSAL_DAYS, + ValleyPriceQuantileFactor, compute_valley_price_quantile, compute_valley_price_quantile_stats, residualize_on_reversal, reversal_20, valley_price_quantile_by_day, ) -from data.clean.intraday_volume_prv import ( - VOLUME_PRV_BASELINE_DAYS, - VOLUME_PRV_BASELINE_MIN_OBS, - VOLUME_PRV_MIN_CLASSIFIABLE, - VOLUME_PRV_MIN_VALID_DAYS, - VOLUME_PRV_SIGMA_K, - peak_mask_for_symbol, - prepare_visible_minute_bars, -) -from factors.compute.intraday_derived import ValleyPriceQuantileFactor from factors.spec import FactorSpec _SYM = "000001.SZ" @@ -861,9 +861,9 @@ def test_prior_factor_modules_are_untouched_by_this_pr(): behaviour on a trivial input, so an accidental edit to the shared module shows up here as well as in the five factors' own test files. """ - from data.clean.intraday_ridge_return import compute_ridge_minute_return - from data.clean.intraday_valley_ridge_vwap import compute_valley_ridge_vwap_ratio - from data.clean.intraday_valley_vwap import compute_valley_relative_vwap + from factors.compute.minute.ridge_minute_return import compute_ridge_minute_return + from factors.compute.minute.valley_ridge_vwap_ratio import compute_valley_ridge_vwap_ratio + from factors.compute.minute.valley_relative_vwap import compute_valley_relative_vwap empty = empty_intraday_bars() for fn in ( diff --git a/tests/test_valley_relative_vwap_factor.py b/tests/test_valley_relative_vwap_factor.py index 1066cba..7edcfc6 100644 --- a/tests/test_valley_relative_vwap_factor.py +++ b/tests/test_valley_relative_vwap_factor.py @@ -21,13 +21,7 @@ import pytest from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from data.clean.intraday_valley_vwap import ( - VALLEY_VWAP_LOOKBACK_DAYS, - VALLEY_VWAP_MIN_VALLEY_BARS, - compute_valley_relative_vwap, - valley_vwap_ratio_by_day, -) -from data.clean.intraday_volume_prv import ( +from factors.compute.minute.primitives import ( VOLUME_PRV_BASELINE_DAYS, VOLUME_PRV_BASELINE_MIN_OBS, VOLUME_PRV_MIN_VALID_DAYS, @@ -35,7 +29,13 @@ peak_mask_for_symbol, prepare_visible_minute_bars, ) -from factors.compute.intraday_derived import ValleyRelativeVwapFactor +from factors.compute.minute.valley_relative_vwap import ( + VALLEY_VWAP_LOOKBACK_DAYS, + VALLEY_VWAP_MIN_VALLEY_BARS, + ValleyRelativeVwapFactor, + compute_valley_relative_vwap, + valley_vwap_ratio_by_day, +) from factors.spec import FactorSpec _SYM = "000001.SZ" @@ -538,7 +538,7 @@ def test_valley_is_exactly_classifiable_and_not_eruptive(): def test_volume_peak_count_unchanged_by_the_exposure_refactor(): """PR-F's factor value is bit-identical whether or not amount is carried.""" - from data.clean.intraday_volume_prv import compute_volume_peak_count + from factors.compute.minute.volume_peak_count import compute_volume_peak_count vols, amts = _case_b_day() rows = _background(_BG_DAYS, _CASE_B_N) + _session("2021-07-11", vols, amts) @@ -557,7 +557,7 @@ def test_volume_peak_count_unchanged_by_the_exposure_refactor(): def test_peak_interval_kurtosis_unchanged_by_the_exposure_refactor(): """PR-H's factor value is bit-identical whether or not amount is carried.""" - from data.clean.intraday_peak_interval import compute_peak_interval_kurtosis + from factors.compute.minute.peak_interval_kurtosis import compute_peak_interval_kurtosis # the PR-H hand case: peaks at 1, 3, 6, 10, 15, 21, 23 -> intervals [2,3,4,5,6,2] n = 25 diff --git a/tests/test_valley_ridge_vwap_ratio_factor.py b/tests/test_valley_ridge_vwap_ratio_factor.py index b6f9349..d50db47 100644 --- a/tests/test_valley_ridge_vwap_ratio_factor.py +++ b/tests/test_valley_ridge_vwap_ratio_factor.py @@ -23,14 +23,7 @@ import pytest from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from data.clean.intraday_valley_ridge_vwap import ( - VALLEY_RIDGE_LOOKBACK_DAYS, - VALLEY_RIDGE_MIN_RIDGE_BARS, - VALLEY_RIDGE_MIN_VALLEY_BARS, - compute_valley_ridge_vwap_ratio, - valley_ridge_vwap_ratio_by_day, -) -from data.clean.intraday_volume_prv import ( +from factors.compute.minute.primitives import ( VOLUME_PRV_BASELINE_DAYS, VOLUME_PRV_BASELINE_MIN_OBS, VOLUME_PRV_MIN_VALID_DAYS, @@ -38,7 +31,14 @@ peak_mask_for_symbol, prepare_visible_minute_bars, ) -from factors.compute.intraday_derived import ValleyRidgeVwapRatioFactor +from factors.compute.minute.valley_ridge_vwap_ratio import ( + VALLEY_RIDGE_LOOKBACK_DAYS, + VALLEY_RIDGE_MIN_RIDGE_BARS, + VALLEY_RIDGE_MIN_VALLEY_BARS, + ValleyRidgeVwapRatioFactor, + compute_valley_ridge_vwap_ratio, + valley_ridge_vwap_ratio_by_day, +) from factors.spec import FactorSpec _SYM = "000001.SZ" @@ -718,7 +718,7 @@ def test_valley_and_peak_and_classifiable_unchanged_by_the_ridge_exposure(): def test_volume_peak_count_unchanged_by_the_ridge_exposure(): """PR-F's factor value is bit-identical (same hand case as the PR-I lock).""" - from data.clean.intraday_volume_prv import compute_volume_peak_count + from factors.compute.minute.volume_peak_count import compute_volume_peak_count vols, amts = _case_b_day() rows = _background(_BG_DAYS, _CASE_B_N) + _session("2021-07-11", vols, amts) @@ -738,7 +738,7 @@ def test_volume_peak_count_unchanged_by_the_ridge_exposure(): def test_peak_interval_kurtosis_unchanged_by_the_ridge_exposure(): """PR-H's factor value is bit-identical (the PR-H hand case, verbatim).""" - from data.clean.intraday_peak_interval import compute_peak_interval_kurtosis + from factors.compute.minute.peak_interval_kurtosis import compute_peak_interval_kurtosis n = 25 vols = [100.0] * n @@ -760,7 +760,7 @@ def test_peak_interval_kurtosis_unchanged_by_the_ridge_exposure(): def test_valley_relative_vwap_unchanged_by_the_ridge_exposure(): """PR-I's factor value is bit-identical (the PR-I case-A hand value, verbatim).""" - from data.clean.intraday_valley_vwap import compute_valley_relative_vwap + from factors.compute.minute.valley_relative_vwap import compute_valley_relative_vwap n = 12 erupt = (3, 7) diff --git a/tests/test_volume_peak_count_factor.py b/tests/test_volume_peak_count_factor.py index cdf0430..ce44442 100644 --- a/tests/test_volume_peak_count_factor.py +++ b/tests/test_volume_peak_count_factor.py @@ -19,17 +19,19 @@ import pandas as pd import pytest -from data.clean.intraday_volume_prv import ( +from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars +from factors.compute.minute.primitives import ( VOLUME_PRV_BASELINE_DAYS, VOLUME_PRV_BASELINE_MIN_OBS, - VOLUME_PRV_LOOKBACK_DAYS, VOLUME_PRV_MIN_CLASSIFIABLE, VOLUME_PRV_MIN_VALID_DAYS, VOLUME_PRV_SIGMA_K, +) +from factors.compute.minute.volume_peak_count import ( + VOLUME_PRV_LOOKBACK_DAYS, + VolumePeakCountFactor, compute_volume_peak_count, ) -from data.clean.intraday_schema import empty_intraday_bars, normalize_intraday_bars -from factors.compute.intraday_derived import VolumePeakCountFactor from factors.spec import FactorSpec _SYM = "000001.SZ" From deba6bcbe3977160ec7a44d852a358eb4fafdf04 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 02:52:53 -0700 Subject: [PATCH 3/8] test(factors): pin the overnight_boundary declarations + settle the PR-D ex-date measurement debt D2 commit C - the NET-NEW P10 property the D0 catalogue stamped for this stage. tests/test_overnight_boundary_declarations.py implements D0 s1.2's reading of the declaration probe: a UNIFORM basis rescale of the symbol's entire < d price history (lambda=0.5, a power of two, so 'none' checks assert bit-equality). Declared none -> day-d value must not move (ridge_minute_return / intraday_amp_cut / jump_amount_corr minute-side, momentum through the real front_adjust chain daily-side); declared crossed_disclosed -> the value MUST move (valley_price_quantile + minute_ideal_amplitude positive controls, the s3.5 pair-migration rule). Two calibration notes recorded in the module docstring: (1) a first draft also rescaled 'amount' and the jump factor's cross-day pooled correlation rightly moved - that simulated a rescale no real ex-date performs (amount is traded RMB value); the D0 wording, prices only, is the faithful probe; (2) the first ridge data set had no ridge at the day's first bar, so the cross-day-lag mutation could not fire - the data now places a ridge run at slots (0,1) and the mutation evidence was RE-RUN and hits (cross-day lag: rc=1; restored: rc=0). minute_ideal_amplitude's D0-note-1 measurement debt is PAID: measured against the D1 frozen CSI500 panel + adj_factor cache, 4.58% of finite values (52,876 / 1,155,283) carry a true ex-date strictly inside their 10-day pooling window; on a seeded 120-value sample (each pinned value re-derived from raw 1min cache and matched 120/120), the ranking-key re-basing intervention moves the factor in 70% of windows (|delta| median 1.07e-05 / p90 1.79e-04 / max 2.91e-03 vs panel std 6.0e-04; top/bottom-k membership shifts median 2.96% / p90 45.8% of pooled bars). All three CROSSED_DISCLOSED requirements now met; the numbers live in the spec docstring, and the D1 debt-tracker test (test_minute_ideal_amplitude_docstring_tracks_the_d2_measurement_debt) now pins the SETTLED disclosure - the docstring must carry the measured numbers and must no longer claim the obligation open (an authorized tracker update, not a weakened test: the tracked debt was paid). --- .../compute/minute/minute_ideal_amplitude.py | 22 +- tests/test_factor_requires_spec_v1.py | 15 +- tests/test_overnight_boundary_declarations.py | 315 ++++++++++++++++++ 3 files changed, 341 insertions(+), 11 deletions(-) create mode 100644 tests/test_overnight_boundary_declarations.py diff --git a/factors/compute/minute/minute_ideal_amplitude.py b/factors/compute/minute/minute_ideal_amplitude.py index 95c8a96..280c258 100644 --- a/factors/compute/minute/minute_ideal_amplitude.py +++ b/factors/compute/minute/minute_ideal_amplitude.py @@ -231,13 +231,21 @@ def spec(self) -> FactorSpec: window, so when the window contains an ex-date the pooled ordering interleaves bars on two different price bases; the definition is pinned to the report (Wind ranks on the raw price) and deliberately - kept. OPEN OBLIGATION, tracked here per D0 note 1: of the three - CROSSED_DISCLOSED requirements (crossing is real / values kept by - definition / deviation MEASURED and disclosed) the third — a - PR-L-style ex-date deviation measurement (share of pooling windows - containing a true ex-date + realized rank-perturbation magnitude) — - is still MISSING and is owed in D2 alongside that stage's - overnight-boundary property tests. + kept. The D0-note-1 deviation MEASUREMENT (owed in D2) is DONE — + measured against the D1 frozen CSI500 panel (2021-07..2026-06, 995 + symbols, 1,155,283 finite values; ex-dates = af(t) != af(t_prev) on + the daily adj_factor cache): 4.58% of finite values (52,876) have + >= 1 true ex-date STRICTLY INSIDE their 10-day pooling window. On a + seeded 120-value stratified sample of those (each pinned value first + re-derived from the raw 1min cache and matched 120/120), re-basing + the pre-ex-date bars onto day d's price scale (the ranking-key-only + intervention) moves the factor in 70% of windows: |delta| median + 1.07e-05 / p90 1.79e-04 / max 2.91e-03 (vs panel std 6.0e-04); + |delta|/|value| median 2.9%, p90 181% (values near zero flip hard); + the pooled top/bottom-k membership shifts in 70% of windows, median + 2.96% / p90 45.8% of pooled bars. The crossing is real, the values + are kept by the pinned definition, and the deviation is now measured + and disclosed — all three CROSSED_DISCLOSED requirements are met. """ return FactorSpec( factor_id=self.name, diff --git a/tests/test_factor_requires_spec_v1.py b/tests/test_factor_requires_spec_v1.py index 64be00e..b7b22e3 100644 --- a/tests/test_factor_requires_spec_v1.py +++ b/tests/test_factor_requires_spec_v1.py @@ -306,11 +306,18 @@ def test_crossed_disclosed_members_are_exactly_the_two_declared(): def test_minute_ideal_amplitude_docstring_tracks_the_d2_measurement_debt(): - # D0 §2 note 1: the CROSSED_DISCLOSED pre-assignment is a CANDIDATE whose - # third requirement (the measured deviation) is still missing; D1 pins - # the obligation in the spec docstring so it cannot silently evaporate. + # D0 §2 note 1: the CROSSED_DISCLOSED pre-assignment was a CANDIDATE whose + # third requirement (the measured deviation) was still missing at D1; D1 + # pinned the obligation in the spec docstring so it could not silently + # evaporate. D2 PAID the debt (measured vs the D1 frozen CSI500 panel: + # 4.58% affected windows; ranking-key re-basing moves 70% of sampled + # values) — this tracker now pins the SETTLED disclosure instead: the + # docstring must carry the measurement, and must no longer claim the + # obligation is open. doc = type(MinuteIdealAmplitudeFactor()).spec.fget.__doc__ - assert "MISSING" in doc and "D2" in doc + assert "MEASUREMENT" in doc and "D2" in doc + assert "4.58%" in doc and "52,876" in doc # the full-grid affected share + assert "MISSING" not in doc # the debt must not still be claimed open def test_spec_variant_replace_carries_the_new_declarations(): diff --git a/tests/test_overnight_boundary_declarations.py b/tests/test_overnight_boundary_declarations.py new file mode 100644 index 0000000..fbef173 --- /dev/null +++ b/tests/test_overnight_boundary_declarations.py @@ -0,0 +1,315 @@ +"""NET-NEW P10: ``overnight_boundary`` declarations are TESTABLE (D2, D0 §1.2). + +The D0 catalogue stamped this property NET-NEW and pinned it to D2: every +factor's ``overnight_boundary`` declaration must be checkable by the UNIFORM +BASIS RESCALE of D0 §1.2's reading (JC2) — multiply the symbol's entire +``< d`` history by a constant λ, simulating ``d`` being an ex-date: + +* minute factors: scale the raw PRICE channels (open/high/low/close) of every + bar strictly before day ``d`` — and ONLY those. ``amount`` is traded VALUE + in RMB, which no split/dividend rescales (PR-M's documented fact), and + ``volume`` is deliberately untouched — the volume-channel split pollution is + OUTSIDE both taxonomy axes (D0 note 3) and rescaling it would disturb the + peak classification this axis does not judge. (A first draft of this file + scaled ``amount`` too and the jump factor's cross-day pooled correlation + rightly moved — that draft was simulating a rescale no real ex-date + performs; the D0 §1.2 wording, prices only, is the faithful probe.) +* daily factors: scale the RAW prices of every row before ``d`` by λ and the + ``adj_factor`` by 1/λ (that is exactly what an ex-date does to the raw + series), then run the REAL ``front_adjust`` -> compute chain. + +Declared ``none`` -> the day-``d`` value must NOT move (no raw-price +comparison crosses the boundary). Declared ``crossed_disclosed`` -> the value +MUST move detectably (the positive control that keeps the invariance tests +honest — §3.5's pair-migration rule). λ = 0.5 is a power of two, so every +within-day ratio rescales EXACTLY in IEEE arithmetic and the ``none`` checks +can assert bit-equality, not approx. + +Why NOT a single-point perturbation: any multi-day-lookback factor depends on +``d-1`` data, so perturbing one number fails everything and the axis loses all +discriminating power (D0 §1.2's explicit caution). The uniform rescale is the +one probe that isolates "does the BASIS BREAK enter the value". + +Mutation evidence (run for this commit, recorded in the acceptance report): +making ridge_minute_return's lag CROSS the day boundary (dropping the +``trade_date`` grouping in its ``prev_close``) fails +``test_ridge_minute_return_is_invariant_to_a_pre_d_basis_rescale`` (rc=1) — +the rescale probe detects exactly the boundary-straddling return the ``none`` +declaration forbids; restoring the within-day lag passes (rc=0). +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from data.clean.adjust import front_adjust +from data.clean.intraday_schema import normalize_intraday_bars +from factors.compute.minute.intraday_amp_cut import ( + IntradayAmpCutFactor, + compute_amp_cut_stats, +) +from factors.compute.minute.jump_amount_corr import ( + JumpAmountCorrFactor, + compute_jump_amount_corr, +) +from factors.compute.minute.minute_ideal_amplitude import ( + MinuteIdealAmplitudeFactor, + compute_minute_ideal_amplitude, +) +from factors.compute.minute.ridge_minute_return import ( + RidgeMinuteReturnFactor, + compute_ridge_minute_return, +) +from factors.compute.minute.valley_price_quantile import ( + ValleyPriceQuantileFactor, + compute_valley_price_quantile_stats, +) +from factors.compute.momentum import MomentumFactor + +_SYM = "000001.SZ" +_LAMBDA = 0.5 # power of two -> within-day ratios rescale EXACTLY + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # +def _minute_bars(rows): + """rows = [(time, symbol, open, high, low, close, volume, amount)] -> bars.""" + df = pd.DataFrame( + { + "time": pd.to_datetime([r[0] for r in rows]), + "symbol": [r[1] for r in rows], + "open": [float(r[2]) for r in rows], + "high": [float(r[3]) for r in rows], + "low": [float(r[4]) for r in rows], + "close": [float(r[5]) for r in rows], + "volume": [float(r[6]) for r in rows], + "amount": [float(r[7]) for r in rows], + } + ) + return normalize_intraday_bars(df, freq="1min") + + +def _rescale_before(bars: pd.DataFrame, day: str, lam: float = _LAMBDA) -> pd.DataFrame: + """The uniform basis rescale: scale the PRICE channels of every bar < ``day``. + + ``amount`` and ``volume`` untouched on purpose (module docstring; D0 §1.2 + wording + note 3 — a real ex-date rescales prices, not traded RMB value). + """ + out = bars.copy() + before = out["bar_end"].dt.normalize() < pd.Timestamp(day) + for col in ("open", "high", "low", "close"): + out.loc[before, col] = out.loc[before, col] * lam + return out + + +def _session(day, specs, sym=_SYM, start="09:31:00"): + """specs = [(open, high, low, close, volume, amount), ...] -> row tuples.""" + base = pd.Timestamp(day) + pd.Timedelta(start) + return [ + (base + pd.Timedelta(minutes=i), sym, *spec) for i, spec in enumerate(specs) + ] + + +def _flat_day(day, n, price=100.0, volume=100.0, sym=_SYM): + """A flat background day: constant price, constant volume (baseline builder).""" + return _session( + day, [(price, price, price, price, volume, price * volume)] * n, sym=sym + ) + + +# --------------------------------------------------------------------------- # +# NONE declarations: the day-d value must NOT move (bit-equal under λ=0.5) +# --------------------------------------------------------------------------- # +def test_ridge_minute_return_is_invariant_to_a_pre_d_basis_rescale(): + """PR-K declares none: within-day lags never straddle the simulated ex-date.""" + assert RidgeMinuteReturnFactor().spec.overnight_boundary == "none" + n = 12 + kw = dict(min_valid_days=1, min_classifiable=1, min_ridge_bars=1) + rows = [] + for i in range(10): + day = (pd.Timestamp("2021-07-01") + pd.Timedelta(days=i)).strftime("%Y-%m-%d") + rows += _flat_day(day, n) + # Two engineered days. Ridge runs at slots (0,1) AND (3,4): the run at the + # DAY'S FIRST BAR is what gives this probe teeth — under the pinned + # within-day lag that bar carries no return, but a defective CROSS-DAY lag + # would price it against the previous (rescaled) day's close and the + # invariance below would break. Moving closes keep both days valid, so day + # d's trailing window really contains rescaled (< d) data — the invariance + # is not vacuously true. + for day in ("2021-07-11", "2021-07-12"): + specs = [] + closes = [100.0, 100.0, 100.0, 150.0, 75.0] + [75.0] * (n - 5) + for i in range(n): + vol = 200.0 if i in (0, 1, 3, 4) else 100.0 + c = closes[i] + specs.append((c, c, c, c, vol, c * vol)) + rows += _session(day, specs) + + bars = _minute_bars(rows) + d = pd.Timestamp("2021-07-12") + base = compute_ridge_minute_return(bars, **kw) + rescaled = compute_ridge_minute_return(_rescale_before(bars, "2021-07-12"), **kw) + assert base.loc[(d, _SYM)] == rescaled.loc[(d, _SYM)] + # sanity: the window really contains a < d valid day (sum over TWO days: + # per day r1 = 0, r3 = +0.5, r4 = -0.5 -> 0; slot 0 carries no return) + assert base.loc[(d, _SYM)] == pytest.approx(0.0) + + +def test_intraday_amp_cut_stats_are_invariant_to_a_pre_d_basis_rescale(): + """PR-G declares none: amp and the 1-min return are within-day ratios.""" + assert IntradayAmpCutFactor().spec.overnight_boundary == "none" + n = 8 + kw = dict(min_day_minutes=2, min_valid_days=2, lam=0.25) + rows = [] + rng = np.random.RandomState(3) + for day_i, day in enumerate(("2021-07-01", "2021-07-02", "2021-07-05")): + specs = [] + for i in range(n): + low = 100.0 + i + day_i + high = low * (1.0 + 0.01 * (1 + ((i + day_i) % 4))) + close = low * (1.0 + 0.005 * ((i + 2 * day_i) % 5)) + specs.append((low, high, low, close, 100.0, close * 100.0)) + rows += _session(day, specs) + bars = _minute_bars(rows) + del rng + d = pd.Timestamp("2021-07-05") + base = compute_amp_cut_stats(bars, **kw) + rescaled = compute_amp_cut_stats(_rescale_before(bars, "2021-07-05"), **kw) + assert base.loc[(d, _SYM), "v_mean"] == rescaled.loc[(d, _SYM), "v_mean"] + assert base.loc[(d, _SYM), "v_std"] == rescaled.loc[(d, _SYM), "v_std"] + # sanity: the trailing stats really pooled < d days (min_valid_days=2) + assert np.isfinite(base.loc[(d, _SYM), "v_std"]) + + +def test_jump_amount_corr_is_invariant_to_a_pre_d_basis_rescale(): + """PR-C declares none: the amplitude z-score is a same-day ratio (exactly + invariant under the rescale) and the correlated quantity is ``amount``, + which a real ex-date does not rescale — so the jump set and every pooled + pair are bit-identical.""" + assert JumpAmountCorrFactor().spec.overnight_boundary == "none" + n = 10 + kw = dict(min_pairs=2) + rows = [] + for day_i, day in enumerate(("2021-07-01", "2021-07-02", "2021-07-05")): + specs = [] + for i in range(n): + open_ = 100.0 + # two clear amplitude jumps per day at slots 3 and 6 + width = 8.0 if i in (3, 6) else 1.0 + high, low = open_ + width, open_ - width + amount = 1000.0 + 137.0 * ((i * (day_i + 2)) % 7) + specs.append((open_, high, low, open_, 100.0, amount)) + rows += _session(day, specs) + bars = _minute_bars(rows) + d = pd.Timestamp("2021-07-05") + base = compute_jump_amount_corr(bars, **kw) + rescaled = compute_jump_amount_corr(_rescale_before(bars, "2021-07-05"), **kw) + assert base.loc[(d, _SYM)] == rescaled.loc[(d, _SYM)] + assert np.isfinite(base.loc[(d, _SYM)]) + + +def test_momentum_is_invariant_to_a_simulated_ex_date_through_front_adjust(): + """momentum_20 declares none: both legs sit on one continuous qfq basis. + + The daily-side probe runs the REAL composed chain: raw prices + adj_factor + -> front_adjust -> compute. Simulating an ex-date at ``d`` multiplies the + raw prices BEFORE d by λ and their adj_factor by 1/λ — front_adjust must + absorb the break so the day-d momentum is bit-identical. + """ + factor = MomentumFactor(window=5) + assert factor.spec.overnight_boundary == "none" + dates = pd.bdate_range("2024-01-02", periods=10) + closes = 100.0 + np.arange(10, dtype=float) * 2.0 + index = pd.MultiIndex.from_product([dates, [_SYM]], names=["date", "symbol"]) + panel = pd.DataFrame( + { + "open": closes, + "high": closes + 1.0, + "low": closes - 1.0, + "close": closes, + "volume": 100.0, + "amount": closes * 100.0, + "adj_factor": 1.0, + }, + index=index, + ) + d = dates[-1] + base = factor.compute(front_adjust(panel)) + + exdate = panel.copy() + before = exdate.index.get_level_values("date") < d + for col in ("open", "high", "low", "close"): + exdate.loc[before, col] = exdate.loc[before, col] * _LAMBDA + exdate.loc[before, "adj_factor"] = exdate.loc[before, "adj_factor"] / _LAMBDA + rescaled = factor.compute(front_adjust(exdate)) + assert base.loc[(d, _SYM)] == rescaled.loc[(d, _SYM)] + assert np.isfinite(base.loc[(d, _SYM)]) + + +# --------------------------------------------------------------------------- # +# CROSSED_DISCLOSED declarations: the value MUST move (positive controls) +# --------------------------------------------------------------------------- # +def test_valley_price_quantile_moves_under_a_pre_d_basis_rescale(): + """PR-L declares crossed_disclosed: prev_close enters day d's range on the + OLD basis, so the simulated ex-date must move the day-d value detectably.""" + assert ValleyPriceQuantileFactor().spec.overnight_boundary == "crossed_disclosed" + n = 8 + kw = dict(min_valid_days=1, min_classifiable=1, min_valley_bars=1) + rows = [] + for i in range(10): + day = (pd.Timestamp("2021-07-01") + pd.Timedelta(days=i)).strftime("%Y-%m-%d") + rows += _flat_day(day, n, price=105.0) + # test day: prices 100..110, prev_close (105) sits INSIDE the day's range, + # so before the rescale it does not widen the range; after λ=0.5 it drops + # to 52.5 and stretches the low end -> q_day must change. + specs = [] + for i in range(n): + low = 100.0 + i + high = low + 2.0 + specs.append((low, high, low, low + 1.0, 100.0, (low + 1.0) * 100.0)) + rows += _session("2021-07-12", specs) + bars = _minute_bars(rows) + d = pd.Timestamp("2021-07-12") + base = compute_valley_price_quantile_stats(bars, **kw) + rescaled = compute_valley_price_quantile_stats( + _rescale_before(bars, "2021-07-12"), **kw + ) + assert np.isfinite(base.loc[(d, _SYM)]) and np.isfinite(rescaled.loc[(d, _SYM)]) + assert base.loc[(d, _SYM)] != rescaled.loc[(d, _SYM)] + + +def test_minute_ideal_amplitude_moves_under_a_pre_d_basis_rescale(): + """PR-D declares crossed_disclosed: the pooled ranking key is the RAW close + across the multi-day window, so rescaling the < d days re-interleaves the + pool and the top/bottom cut must move (the D0 note-1 candidate confirmed).""" + assert MinuteIdealAmplitudeFactor().spec.overnight_boundary == "crossed_disclosed" + n = 6 + kw = dict(min_minutes=4, lam=0.25) + rows = [] + # day A: HIGH closes (~200) with SMALL amps; day B (= d): closes ~100 with + # LARGE amps. Before the rescale the pooled top-k by close comes from day A + # (small amps); after halving day A's prices its closes sit at ~100 and the + # top-k re-interleaves -> V_high - V_low must change. + specs_a = [] + for i in range(n): + low = 200.0 + 2.0 * i + high = low * 1.001 + specs_a.append((low, high, low, low + 1.0, 100.0, (low + 1.0) * 100.0)) + rows += _session("2021-07-09", specs_a) + specs_b = [] + for i in range(n): + low = 100.0 + 2.0 * i + high = low * (1.0 + 0.05 * (i + 1)) + specs_b.append((low, high, low, low + 1.5, 100.0, (low + 1.5) * 100.0)) + rows += _session("2021-07-12", specs_b) + bars = _minute_bars(rows) + d = pd.Timestamp("2021-07-12") + base = compute_minute_ideal_amplitude(bars, **kw) + rescaled = compute_minute_ideal_amplitude( + _rescale_before(bars, "2021-07-12"), **kw + ) + assert np.isfinite(base.loc[(d, _SYM)]) and np.isfinite(rescaled.loc[(d, _SYM)]) + assert base.loc[(d, _SYM)] != rescaled.loc[(d, _SYM)] From c23a195e46b82d8c36bdf092de21b4960884a166 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 07:45:30 -0700 Subject: [PATCH 4/8] feat(qt): D2 frozen-baseline reconciliation + stratified hand-anchor tooling D2 commits F/G tooling (design v3.2 s5 legs 2+4, R11/R12). qt/panel_reconcile.py rebuilds all 14 closing-factor RAW panels on the current tree (the panel_freeze recipes route through the D2 shims, so they exercise the migrated factors/compute/minute math automatically), writes them to the SEPARATE artifacts/refactor_baseline/panels_d2/ directory (the frozen D1 baseline is never overwritten), and compares cell by cell: identical index, identical NaN sets, relative diff <= 1e-12 on jointly-finite cells. Anti-empty -reconciliation provenance: the frozen side is hash-verified against the D1 manifest BEFORE any comparison is trusted, and both sides are read from disk through independent file reads. tests/test_panel_reconcile_d2.py feeds the comparator engineered defects (value drift beyond budget, NaN-set change in both directions, index mismatch, near-zero fabrication) and asserts each one CONVICTS - a comparator that cannot fail is the compare_postmerge.py failure mode. qt/hand_anchors_d2.py + qt/hand_anchor_rows.py hand-recompute stratified anchor rows (R12: warm-up end / guard boundary / ex-date window / seeded interior) from the raw cache parquets in plain numpy+pandas arithmetic. The module NEVER imports factors.* or data.clean.* (runtime guard raises if the engine sneaks into sys.modules); engine values come from the panels_d2 files (disk reads). qt/hand_anchors_engine_values.py is the one companion allowed to import the engine - it fills in the engine side for the four ops-rewritten daily factors that have no frozen panel (momentum/reversal/liquidity/ overnight_mom). Two engine-semantics facts the hand check surfaced and now encodes (the pre-flight spot-check convicted the first drafts, which is the point of an independent reimplementation): (1) PR-C jump_amount_corr computes on FULL-day bars - its engine has no 14:50 cutoff (day-level PIT only; PR-C predates the cutoff convention); (2) PR-E resamples the FULL day to 5min FIRST and then PIT-filters the DERIVED bars by their own available_time = max(source)+1min, so a bucket with any post-14:49 constituent is dropped WHOLE. With both encoded, the 24-row pre-flight spot-check against the frozen panels passes 0-failure (20 exact, worst rel 5.8e-15, all within the 1e-12 float budget). tests/test_provisional_anchor_d2.py freezes the PROVISIONAL full-precision phase0 demo anchor (R10 timeline; promoted in D6): ic_mean 0.9600438863169586 / annual_return 0.8407986461861146 / sharpe 10.304260918159038 / plus turnover/cost/vol/maxDD, exact-equality contract, with the coarse 0.9600/0.8408 reading locked as a corollary. --- qt/hand_anchor_rows.py | 429 +++++++++++++++++ qt/hand_anchors_d2.py | 695 ++++++++++++++++++++++++++++ qt/hand_anchors_engine_values.py | 99 ++++ qt/panel_reconcile.py | 331 +++++++++++++ tests/test_panel_reconcile_d2.py | 80 ++++ tests/test_provisional_anchor_d2.py | 77 +++ 6 files changed, 1711 insertions(+) create mode 100644 qt/hand_anchor_rows.py create mode 100644 qt/hand_anchors_d2.py create mode 100644 qt/hand_anchors_engine_values.py create mode 100644 qt/panel_reconcile.py create mode 100644 tests/test_panel_reconcile_d2.py create mode 100644 tests/test_provisional_anchor_d2.py diff --git a/qt/hand_anchor_rows.py b/qt/hand_anchor_rows.py new file mode 100644 index 0000000..9898def --- /dev/null +++ b/qt/hand_anchor_rows.py @@ -0,0 +1,429 @@ +"""D2 hand-anchor row selection + orchestration (engine-free; see hand_anchors_d2). + +Selects the stratified rows (R12), calls the plain-arithmetic hand computations +of :mod:`qt.hand_anchors_d2`, compares each against the NEW-ENGINE value read +from the ``panels_d2`` parquet files (file reads — never engine imports), and +writes the selection + results JSON. The four ops-rewritten daily factors that +have no frozen panel are SELECTED and hand-computed here too; their engine-side +values are filled in by ``qt.hand_anchors_engine_values`` (the only module of +the trio allowed to import the engine). + +Selection sources (all engine-free): the ``panels_d2`` files give the finite / +NaN pattern (warm-up ends, interior rows); the ``adj_factor`` cache gives the +ex-date rows; guard-boundary rows are found by HAND-scanning seeded symbols' +day statistics and taking a day whose gated count sits exactly at its floor — +when the floor is not attained in the scan budget the NEAREST attainable day is +used and the miss is DISCLOSED in the output (never silently substituted). +""" + +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +from qt import hand_anchors_d2 as H + +TOL = H.TOL +SEED = H.SEED +D2 = H.D2_PANELS +SCAN_SYMBOLS = 30 # guard-boundary scan budget (seeded symbols per factor) + + +def _panel(factor_id: str) -> pd.Series: + frame = pd.read_parquet(D2 / f"{factor_id}.parquet") + return frame.set_index(["date", "symbol"])[factor_id].sort_index() + + +def _symbols(series: pd.Series) -> list[str]: + return sorted(set(series.index.get_level_values("symbol"))) + + +def _finite(series: pd.Series) -> pd.Series: + return series[np.isfinite(series.to_numpy(float))] + + +def _pick_warmup(series: pd.Series, rng: np.random.RandomState) -> tuple: + """A symbol's FIRST finite row (its warm-up end on the factor's own axis).""" + for sym in rng.permutation(_symbols(series)): + sub = series.xs(sym, level="symbol").sort_index() + fin = np.isfinite(sub.to_numpy(float)) + if fin.any() and not fin[0]: # a real warm-up ramp exists + j = int(np.argmax(fin)) + return (sub.index[j], str(sym)) + raise RuntimeError("no warm-up row found") + + +def _pick_random(series: pd.Series, rng: np.random.RandomState, k: int = 2) -> list[tuple]: + fin = _finite(series) + idx = rng.choice(len(fin), size=k, replace=False) + return [(fin.index[int(i)][0], str(fin.index[int(i)][1])) for i in idx] + + +def _pick_exdate(series: pd.Series, rng: np.random.RandomState, window: int) -> tuple: + """A finite row whose trailing ``window`` panel days span a true ex-date.""" + for sym in rng.permutation(_symbols(series)): + af = H.af_series(str(sym)) + if af.empty: + continue + changes = af.index[af.ne(af.shift(1)) & af.shift(1).notna()] + if len(changes) == 0: + continue + sub = series.xs(sym, level="symbol").sort_index() + dates = list(sub.index) + fin = np.isfinite(sub.to_numpy(float)) + for ch in changes: + after = [j for j, dt in enumerate(dates) if dt >= ch] + for j in after[: window - 1]: + lo = max(0, j - window + 1) + if fin[j] and any(dates[i] >= ch for i in range(lo + 1, j + 1)): + return (dates[j], str(sym)) + raise RuntimeError("no ex-date row found") + + +def _guard_scan( + series: pd.Series, + rng: np.random.RandomState, + count_of_day, + floor: int, +) -> tuple[tuple, int]: + """Find a finite row whose OWN day count sits exactly at ``floor``. + + ``count_of_day(symbol) -> pd.Series(trade_date -> gated count)`` is a HAND + computation. Returns ((date, symbol), realized_count); if the exact floor is + not attained within the scan budget the nearest attainable is returned + (caller discloses). + """ + best: tuple | None = None + best_count = None + syms = list(rng.permutation(_symbols(series)))[:SCAN_SYMBOLS] + for sym in syms: + counts = count_of_day(str(sym)) + if counts is None or counts.empty: + continue + sub = series.xs(str(sym), level="symbol") + fin_dates = set(sub.index[np.isfinite(sub.to_numpy(float))]) + counts = counts[counts.index.isin(fin_dates)] + if counts.empty: + continue + at = counts[counts == floor] + if len(at): + return ((at.index[0], str(sym)), floor) + cand = counts[counts >= floor] + if len(cand): + c = int(cand.min()) + if best_count is None or c < best_count: + best_count = c + best = (cand.idxmin(), str(sym)) + if best is None: + raise RuntimeError("guard scan found no candidate row") + return (best, int(best_count)) + + +# ---- per-factor day-count providers for the guard scan (hand computations) -- +def _peak_family_counts(column: str): + def provider(sym: str) -> pd.Series | None: + bars = H.read_minutes(sym, H.WINDOW_LO, H.WINDOW_LO + pd.Timedelta(days=240)) + if bars.empty: + return None + st = H.day_stats(H.classify(bars)) + return st[column].astype(int) + + return provider + + +def run_hand_anchors(out_path: Path) -> int: + started = time.monotonic() + rng = np.random.RandomState(SEED) + results: list[dict] = [] + + def compare(factor_id, cls, key, hand, note=""): + d, sym = key + engine = float("nan") + if (D2 / f"{factor_id}.parquet").exists(): + series = _panel(factor_id) + if (d, sym) in series.index: + engine = float(series.loc[(d, sym)]) + rel = ( + abs(hand - engine) / max(abs(hand), abs(engine)) + if np.isfinite(hand) and np.isfinite(engine) and max(abs(hand), abs(engine)) > 0 + else (0.0 if hand == engine or (np.isnan(hand) and np.isnan(engine)) else float("inf")) + ) + ok = rel <= TOL + results.append( + { + "factor_id": factor_id, "class": cls, "date": str(pd.Timestamp(d).date()), + "symbol": sym, "hand": hand, "engine": engine, + "rel_diff": rel, "ok": bool(ok), "note": note, + } + ) + print(f"{'OK ' if ok else 'FAIL'} {factor_id:28s} {cls:12s} " + f"{pd.Timestamp(d).date()} {sym} hand={hand!r} engine={engine!r} rel={rel:.2e} {note}") + + # ---------------- minute factors, peak family --------------------------- + peak_specs = [ + ("volume_peak_count_20", H.hand_volume_peak_count, "classifiable", + H.MIN_CLASSIFIABLE, H.VPC_LOOKBACK), + ("peak_interval_kurtosis_20", H.hand_peak_interval_kurtosis, "peak_any", + 2, H.PIK_LOOKBACK), # >=2 peaks -> >=1 interval on the day + ("valley_relative_vwap_20", H.hand_valley_relative_vwap, "valley_n", + H.VVW_MIN_VALLEY, H.VVW_LOOKBACK), + ("valley_ridge_vwap_ratio_20", H.hand_valley_ridge_vwap_ratio, "ridge_n", + H.VRV_MIN_RIDGE, H.VRV_LOOKBACK), + ("ridge_minute_return_20", H.hand_ridge_minute_return, "ridge_n", + H.RMR_MIN_RIDGE, H.RMR_LOOKBACK), + ("peak_ridge_amount_ratio_20", H.hand_peak_ridge_amount_ratio, "peak_n", + H.PRA_MIN_PEAK, H.PRA_LOOKBACK), + ] + for factor_id, hand_fn, guard_col, guard_floor, window in peak_specs: + series = _panel(factor_id) + w = _pick_warmup(series, rng) + compare(factor_id, "warmup_end", w, hand_fn(w[1], w[0])) + (g, realized) = _guard_scan(series, rng, _peak_family_counts(guard_col), guard_floor) + note = "" if realized == guard_floor else f"nearest>{guard_floor}: {realized} (floor not attained in scan)" + compare(factor_id, "guard_boundary", g, hand_fn(g[1], g[0]), note) + e = _pick_exdate(series, rng, window) + compare(factor_id, "ex_date_window", e, hand_fn(e[1], e[0])) + for r in _pick_random(series, rng): + compare(factor_id, "random", r, hand_fn(r[1], r[0])) + + # ---------------- jump --------------------------------------------------- + series = _panel("jump_amount_corr_20") + w = _pick_warmup(series, rng) + compare("jump_amount_corr_20", "warmup_end", w, H.hand_jump_amount_corr(w[1], w[0])) + e = _pick_exdate(series, rng, H.JUMP_LOOKBACK) + compare("jump_amount_corr_20", "ex_date_window", e, H.hand_jump_amount_corr(e[1], e[0])) + for r in _pick_random(series, rng, 3): + compare("jump_amount_corr_20", "random", r, H.hand_jump_amount_corr(r[1], r[0])) + + # ---------------- minute ideal amplitude -------------------------------- + series = _panel("minute_ideal_amp_10") + w = _pick_warmup(series, rng) + compare("minute_ideal_amp_10", "warmup_end", w, H.hand_minute_ideal_amp(w[1], w[0])) + e = _pick_exdate(series, rng, H.IDEAL_LOOKBACK) + compare("minute_ideal_amp_10", "ex_date_window", e, H.hand_minute_ideal_amp(e[1], e[0])) + for r in _pick_random(series, rng, 3): + compare("minute_ideal_amp_10", "random", r, H.hand_minute_ideal_amp(r[1], r[0])) + + # ---------------- amp anomaly (5min derived) ---------------------------- + series = _panel("amp_marginal_anomaly_vol_20") + w = _pick_warmup(series, rng) + compare("amp_marginal_anomaly_vol_20", "warmup_end", w, H.hand_amp_anomaly(w[1], w[0])) + e = _pick_exdate(series, rng, H.ANOM_LOOKBACK) + compare("amp_marginal_anomaly_vol_20", "ex_date_window", e, H.hand_amp_anomaly(e[1], e[0])) + for r in _pick_random(series, rng, 3): + compare("amp_marginal_anomaly_vol_20", "random", r, H.hand_amp_anomaly(r[1], r[0])) + + # ---------------- amp cut (full cross-section; amortized on 2 dates) ---- + series = _panel("intraday_amp_cut_10") + all_syms = _symbols(series) + w = _pick_warmup(series, rng) + r1, r2 = _pick_random(series, rng, 2) + e = _pick_exdate(series, rng, H.CUT_LOOKBACK) + picks = [("warmup_end", w), ("ex_date_window", e), ("random", r1), ("random", r2)] + # add a second symbol on r1's date (amortizes the heavy cross-section) + same_date = _finite(series).xs(r1[0], level="date", drop_level=False) + extra_sym = str(same_date.index[rng.randint(len(same_date))][1]) + picks.append(("random", (r1[0], extra_sym))) + xs_cache: dict = {} + for cls, (d, sym) in picks: + if d not in xs_cache: + xs_cache[d] = { + s: H._amp_cut_stats_one(s, d) for s in all_syms + } + stats = xs_cache[d] + vm = np.array([m for m, s in stats.values()]) + vs = np.array([s for m, s in stats.values()]) + names = list(stats.keys()) + fin = np.isfinite(vm) & np.isfinite(vs) + vm_f, vs_f = vm[fin], vs[fin] + names_f = [n for n, f in zip(names, fin) if f] + if sym not in names_f or len(names_f) < H.CUT_MIN_XS: + hand = float("nan") + else: + sm, ss = vm_f.std(ddof=1), vs_f.std(ddof=1) + if not (np.isfinite(sm) and np.isfinite(ss)) or sm == 0 or ss == 0: + hand = float("nan") + else: + zm = (vm_f - vm_f.mean()) / sm + zs = (vs_f - vs_f.mean()) / ss + hand = float(((zm + zs) / 2.0)[names_f.index(sym)]) + compare("intraday_amp_cut_10", cls, (d, sym), hand, + note=f"cross_section={int(fin.sum())}") + + # ---------------- valley price quantile (cross-section; 2 dates) -------- + series = _panel("valley_price_quantile_20") + w = _pick_warmup(series, rng) + r1, r2 = _pick_random(series, rng, 2) + e = _pick_exdate(series, rng, H.VPQ_LOOKBACK) + picks = [("warmup_end", w), ("ex_date_window", e), ("random", r1), ("random", r2)] + same_date = _finite(series).xs(r1[0], level="date", drop_level=False) + extra_sym = str(same_date.index[rng.randint(len(same_date))][1]) + picks.append(("random", (r1[0], extra_sym))) + vpq_cache: dict = {} + for cls, (d, sym) in picks: + if d not in vpq_cache: + qb = {s: H._vpq_qbar_one(s, d) for s in all_syms} + vpq_cache[d] = qb + qb = vpq_cache[d] + qbars, revs, names = [], [], [] + for s, val in qb.items(): + if not np.isfinite(val): + continue + qfq = H._qfq_close(s) + qfq = qfq[np.isfinite(qfq.to_numpy(float)) & (qfq.to_numpy(float) > 0)] + if d not in qfq.index: + continue + j = qfq.index.get_loc(d) + if j < H.VPQ_REV_DAYS + 1: + continue + rev = -(qfq.iloc[j - 1] / qfq.iloc[j - (H.VPQ_REV_DAYS + 1)] - 1.0) + if not np.isfinite(rev): + continue + qbars.append(val) + revs.append(float(rev)) + names.append(s) + if sym not in names or len(names) < H.VPQ_MIN_XS: + hand = float("nan") + else: + x, y = np.array(revs), np.array(qbars) + xm = x.mean() + sxx = float(((x - xm) ** 2).sum()) + ym = y.mean() + slope = float(((x - xm) * (y - ym)).sum()) / sxx + hand = float((y - (ym + slope * (x - xm)))[names.index(sym)]) + compare("valley_price_quantile_20", cls, (d, sym), hand, + note=f"cross_section={len(names)}") + + # ---------------- book factors ------------------------------------------ + for factor_id, field in (("value_ep", "pe"), ("value_bp", "pb")): + series = _panel(factor_id) + # guard boundary: a NaN row caused by a non-positive published ratio + found = None + for sym in rng.permutation(_symbols(series))[:SCAN_SYMBOLS]: + db = H.read_daily(str(sym), "daily_basic") + if db.empty: + continue + neg = db[db[field] <= 0] + if len(neg): + found = (neg.iloc[0]["date"], str(sym)) + break + if found is not None: + compare(factor_id, "guard_boundary", + found, H.hand_value_ratio(found[1], found[0], field), + note=f"{field}<=0 -> NaN") + first = series.xs(_symbols(series)[0], level="symbol").index[0] + k0 = (first, _symbols(series)[0]) + compare(factor_id, "first_row", k0, H.hand_value_ratio(k0[1], k0[0], field), + note="warm-up not applicable: same-day published ratio (disclosed)") + for r in _pick_random(series, rng, 3): + compare(factor_id, "random", r, H.hand_value_ratio(r[1], r[0], field)) + + series = _panel("volatility_20") + w = _pick_warmup(series, rng) + compare("volatility_20", "warmup_end", w, H.hand_volatility_20(w[1], w[0]), + note="warmup == min_periods guard boundary (same floor; disclosed)") + e = _pick_exdate(series, rng, 21) + compare("volatility_20", "ex_date_window", e, H.hand_volatility_20(e[1], e[0])) + for r in _pick_random(series, rng, 3): + compare("volatility_20", "random", r, H.hand_volatility_20(r[1], r[0])) + + # ---------------- daily ops-rewritten factors (engine side filled later) - + daily_rows: list[dict] = [] + md_syms = _symbols(series) # volatility panel symbols == universe symbols + for factor_id, window, kind in ( + ("momentum_20", 20, "momentum"), + ("reversal_20", 20, "reversal"), + ("liquidity_20", 20, "liquidity"), + ("overnight_mom_20", 20, "overnight"), + ): + rows: list[tuple[str, tuple, str]] = [] + sym = str(md_syms[int(rng.randint(len(md_syms)))]) + md = H.read_daily(sym, "market_daily") + dates = list(md["date"]) + need = window + 1 + if len(dates) <= need + 2: + raise RuntimeError(f"{factor_id}: symbol {sym} history too short") + rows.append(("warmup_end", (dates[need - 1] if kind != "liquidity" else dates[window - 1], sym), "")) + # ex-date row (price factors only; liquidity reads the amount channel) + if kind != "liquidity": + af = H.af_series(sym) + ch = af.index[af.ne(af.shift(1)) & af.shift(1).notna()] + ex_row = None + for c in ch: + later = [dt for dt in dates if dt >= c] + if later: + j = dates.index(later[0]) + if j + 3 < len(dates): + ex_row = dates[j + 3] + break + if ex_row is not None: + rows.append(("ex_date_window", (ex_row, sym), "")) + for _ in range(2 if kind != "liquidity" else 3): + j = int(rng.randint(need + 1, len(dates))) + rows.append(("random", (dates[j], sym), "")) + for cls, (d, s), note in rows: + if kind in ("momentum", "reversal"): + qfq = H._qfq_close(s) + hand = float("nan") + if d in qfq.index: + j = qfq.index.get_loc(d) + if j >= window: + hand = float(qfq.iloc[j] / qfq.iloc[j - window] - 1.0) + if kind == "reversal": + hand = -hand + elif kind == "liquidity": + amt = H.read_daily(s, "market_daily").set_index("date")["amount"] + hand = float("nan") + if d in amt.index: + j = amt.index.get_loc(d) + if j >= window - 1: + m = float(amt.iloc[j - window + 1 : j + 1].mean()) + hand = float(np.log(m)) if m > 0 else float("nan") + else: # overnight + md_f = H.read_daily(s, "market_daily").set_index("date") + af = H.af_series(s).reindex(md_f.index) + anchor = af.dropna().iloc[-1] + open_q = md_f["open"] * af / anchor + close_q = md_f["close"] * af / anchor + hand = float("nan") + if d in md_f.index: + j = md_f.index.get_loc(d) + if j >= window: + terms = [] + okall = True + for t in range(j - window + 1, j + 1): + o, c = float(open_q.iloc[t]), float(close_q.iloc[t - 1]) + if not (o > 0 and c > 0): + okall = False + break + terms.append(np.log(o / c)) + hand = float(np.sum(terms)) if okall else float("nan") + daily_rows.append( + {"factor_id": factor_id, "class": cls, "date": str(pd.Timestamp(d).date()), + "symbol": s, "hand": hand, "note": note} + ) + print(f"HAND {factor_id:28s} {cls:12s} {pd.Timestamp(d).date()} {s} hand={hand!r}") + + H.assert_no_engine_imports() + payload = { + "seed": SEED, "tolerance": TOL, + "elapsed_seconds": round(time.monotonic() - started, 1), + "frozen14": results, + "daily_pending_engine": daily_rows, + "all_ok_frozen14": all(r["ok"] for r in results), + } + out_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + n_bad = sum(1 for r in results if not r["ok"]) + print(f"hand anchors (frozen 14): {len(results)} rows, {n_bad} mismatches -> {out_path}") + print(f"daily rows pending engine comparison: {len(daily_rows)} " + f"(run python -m qt.hand_anchors_engine_values)") + return 0 if n_bad == 0 else 1 + + +if __name__ == "__main__": # pragma: no cover - thin CLI shim + sys.exit(run_hand_anchors(H.OUT_JSON)) diff --git a/qt/hand_anchors_d2.py b/qt/hand_anchors_d2.py new file mode 100644 index 0000000..4f3c677 --- /dev/null +++ b/qt/hand_anchors_d2.py @@ -0,0 +1,695 @@ +"""D2 stratified hand-computed anchors (R12): plain arithmetic vs the new engine. + +Every migrated factor gets >= 5 anchor rows recomputed BY HAND from the raw +cache parquets — stdlib + numpy + pandas arithmetic only. THIS MODULE MUST +NEVER IMPORT ``factors.*`` OR ``data.clean.*`` (the #79 rule: a hand check +that imports the engine inherits the engine's bugs); a runtime guard at the +bottom of the module raises if any such module is loaded. The engine side of +each comparison is read FROM DISK (the ``panels_d2`` parquet files the D2 +reconciliation wrote) for the 14 frozen factors; the four ops-rewritten daily +factors without a frozen panel (momentum/reversal/liquidity/overnight_mom) are +compared by the companion ``qt.hand_anchors_engine_values`` (which may import +the engine — the INDEPENDENCE requirement binds the hand computation, not the +comparer). + +Stratified sampling (R12): per factor, one row from each APPLICABLE boundary +class — (a) the warm-up END (the symbol's first finite value; verified by hand +to sit exactly at the accumulation floor), (b) a GUARD boundary row (a day +whose gated count sits exactly at its ``min_*`` floor where the data attains +it; otherwise the nearest attainable is used and DISCLOSED), (c) an EX-DATE +row for price-dependent factors (the trailing window spans a true +``af(t) != af(t_prev)`` event) — plus >= 2 seeded interior rows. Uniform +sampling would hit a boundary row with ~0.4% probability; boundaries are where +migration bugs live. + +Hand reimplementation notes (all plain arithmetic, no engine helpers): +* PIT visibility: the runners normalize with ``data_lag='1min'``, so a bar is + visible iff ``bar_end + 1min <= trade_date + 14:50``. +* peak taxonomy: same-slot strictly-prior baselines via EXPLICIT position + windows (numpy mean/std ddof=1 over the trailing 20 day-rows, >= 10 obs, + excluding the day itself); eruptive = vol > mu + sigma; a peak needs both + 60s-adjacent neighbours mild; ridge = eruptive & ~peak; valley = mild. +* qfq for the daily factors: raw close x af / af(symbol's last panel day) — + the ``front_adjust`` anchor convention, re-derived here from its definition. + +Run AFTER ``python -m qt.panel_reconcile`` produced ``panels_d2``: + + python -m qt.hand_anchors_d2 # select + hand-compute + compare (14) + python -m qt.hand_anchors_engine_values # compare the 4 daily factors +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +REPO = Path(__file__).resolve().parent.parent +CACHE = REPO / "artifacts/cache/tushare/v1" +D2_PANELS = REPO / "artifacts/refactor_baseline/panels_d2" +OUT_JSON = REPO / "artifacts/refactor_baseline/hand_anchors_d2.json" +WINDOW_LO = pd.Timestamp("2021-07-01") +WINDOW_HI = pd.Timestamp("2026-06-30") +TOL = 1e-12 +SEED = 20260724 + +# ---- pinned definition constants, restated as PLAIN NUMBERS (the point of a +# hand check is to restate the definition, not to import it) ----------------- +BASELINE_DAYS, BASELINE_MIN_OBS, SIGMA_K = 20, 10, 1.0 +MIN_VALID_DAYS, MIN_CLASSIFIABLE = 10, 100 +JUMP_LOOKBACK, JUMP_MIN_PAIRS, JUMP_Z = 20, 10, 1.0 +IDEAL_LOOKBACK, IDEAL_LAM, IDEAL_MIN_MINUTES = 10, 0.25, 1150 +ANOM_LOOKBACK, ANOM_MIN_POOL, ANOM_MIN_SELECTED, ANOM_K = 20, 460, 20, 1.0 +CUT_LOOKBACK, CUT_LAM, CUT_MIN_DAY, CUT_MIN_VALID, CUT_MIN_XS = 10, 0.20, 100, 6, 10 +VPC_LOOKBACK = 20 +PIK_LOOKBACK, PIK_MIN_INTERVALS = 20, 20 +VVW_LOOKBACK, VVW_MIN_VALLEY = 20, 20 +VRV_LOOKBACK, VRV_MIN_VALLEY, VRV_MIN_RIDGE = 20, 20, 10 +RMR_LOOKBACK, RMR_MIN_RIDGE = 20, 10 +VPQ_LOOKBACK, VPQ_MIN_VALLEY, VPQ_REV_DAYS, VPQ_MIN_XS = 20, 20, 20, 10 +PRA_LOOKBACK, PRA_MIN_PEAK, PRA_MIN_RIDGE = 20, 5, 10 + + +# --------------------------------------------------------------------------- # +# raw cache readers (plain parquet reads) +# --------------------------------------------------------------------------- # +def read_minutes( + symbol: str, lo: pd.Timestamp, hi: pd.Timestamp, *, pit: bool = True +) -> pd.DataFrame: + """1min bars for one symbol over [lo, hi] trade dates. + + ``pit=True`` keeps only the 14:50-visible bars (``bar_end + 1min <= + trade_date + 14:50`` — the runners' ``data_lag='1min'``). ``pit=False`` + returns the FULL day: two factors need it — PR-C (jump) predates the + cutoff convention and its engine computes on full-day bars (day-level PIT + only), and PR-E (amp anomaly) resamples the FULL day to 5min FIRST and + PIT-filters the DERIVED bars by their own available_time. + """ + months = pd.period_range(lo, hi, freq="M") + parts = [] + base = ( + CACHE / "stk_mins_1min" / "freq=1min" + / f"symbol_prefix={symbol[:3]}" / f"symbol={symbol}" + ) + for p in months: + f = base / f"year={p.year}" / f"month={p.month:02d}.parquet" + if f.exists(): + parts.append(pd.read_parquet(f)) + if not parts: + return pd.DataFrame() + df = pd.concat(parts, ignore_index=True) + df = df.drop_duplicates(subset=["symbol", "bar_end"], keep="last") + df["trade_date"] = df["bar_end"].dt.normalize() + df = df[(df["trade_date"] >= lo) & (df["trade_date"] <= hi)] + if pit: + visible = (df["bar_end"] + pd.Timedelta("1min")) <= ( + df["trade_date"] + pd.Timedelta("14:50:00") + ) + df = df[visible] + return df.sort_values("bar_end").reset_index(drop=True) + + +def read_daily(symbol: str, endpoint: str) -> pd.DataFrame: + f = CACHE / endpoint / f"symbol_prefix={symbol[:3]}" / f"{symbol}.parquet" + if not f.exists(): + return pd.DataFrame() + df = pd.read_parquet(f) + df = df.drop_duplicates(subset=["date", "symbol"], keep="last") + return df[(df["date"] >= WINDOW_LO) & (df["date"] <= WINDOW_HI)].sort_values("date") + + +def af_series(symbol: str) -> pd.Series: + df = read_daily(symbol, "adj_factor") + return df.set_index("date")["adj_factor"] if len(df) else pd.Series(dtype=float) + + +# --------------------------------------------------------------------------- # +# the peak taxonomy BY HAND (explicit windows; no engine import) +# --------------------------------------------------------------------------- # +def classify(bars: pd.DataFrame) -> pd.DataFrame: + """Per-bar classifiable/valley/peak/ridge for ONE symbol's visible bars.""" + work = bars.copy() + vol = work["volume"].to_numpy(float) + work = work[np.isfinite(vol) & (vol >= 0.0)].copy() + work["slot"] = ( + (work["bar_end"] - work["trade_date"]) // pd.Timedelta(minutes=1) + ).astype(int) + days = sorted(work["trade_date"].unique()) + day_ix = {d: i for i, d in enumerate(days)} + slots = sorted(work["slot"].unique()) + slot_ix = {s: i for i, s in enumerate(slots)} + mat = np.full((len(days), len(slots)), np.nan) + di = work["trade_date"].map(day_ix).to_numpy() + si = work["slot"].map(slot_ix).to_numpy() + mat[di, si] = work["volume"].to_numpy(float) + + thr = np.full_like(mat, np.nan) + for i in range(len(days)): + lo = max(0, i - BASELINE_DAYS) + win = mat[lo:i, :] # strictly prior positions + if win.shape[0] == 0: + continue + n_obs = np.sum(~np.isnan(win), axis=0) + with np.errstate(invalid="ignore"): + mu = np.nanmean(win, axis=0) + sd = np.nanstd(win, axis=0, ddof=1) + ok = n_obs >= BASELINE_MIN_OBS + thr[i, ok] = mu[ok] + SIGMA_K * sd[ok] + + t = thr[di, si] + v = work["volume"].to_numpy(float) + classifiable = np.isfinite(t) + eruptive = classifiable & (v > t) + mild = classifiable & ~eruptive + work["classifiable"] = classifiable + work["valley"] = mild + work = work.sort_values(["trade_date", "bar_end"], kind="mergesort").reset_index( + drop=True + ) + # neighbour test per day: exactly 60s away and mild on both sides + peak = np.zeros(len(work), dtype=bool) + er = work["classifiable"].to_numpy() & ( + work["volume"].to_numpy(float) + > thr[work["trade_date"].map(day_ix).to_numpy(), + work["slot"].map(slot_ix).to_numpy()] + ) + ml = work["valley"].to_numpy(bool) + be = work["bar_end"].to_numpy("datetime64[ns]").astype("int64") + td = work["trade_date"].to_numpy() + for i in range(len(work)): + if not er[i]: + continue + prev_ok = ( + i > 0 and td[i - 1] == td[i] and be[i] - be[i - 1] == 60_000_000_000 + and ml[i - 1] + ) + next_ok = ( + i + 1 < len(work) and td[i + 1] == td[i] + and be[i + 1] - be[i] == 60_000_000_000 and ml[i + 1] + ) + peak[i] = prev_ok and next_ok + work["eruptive"] = er + work["peak"] = peak + work["ridge"] = er & ~peak + return work + + +def day_stats(work: pd.DataFrame) -> pd.DataFrame: + """Per-day taxonomy counts + guarded sums used by the peak family.""" + vol = work["volume"].to_numpy(float) + amt = work["amount"].to_numpy(float) + tradable = np.isfinite(vol) & (vol > 0) & np.isfinite(amt) & (amt > 0) + amt_ok = np.isfinite(amt) & (amt > 0) + g = pd.DataFrame( + { + "trade_date": work["trade_date"].to_numpy(), + "classifiable": work["classifiable"].to_numpy(bool).astype(int), + "peak_n": (work["peak"].to_numpy(bool) & amt_ok).astype(int), + "ridge_n_amt": (work["ridge"].to_numpy(bool) & amt_ok).astype(int), + "valley_n": (work["valley"].to_numpy(bool) & tradable).astype(int), + "ridge_n": (work["ridge"].to_numpy(bool) & tradable).astype(int), + "peak_any": work["peak"].to_numpy(bool).astype(int), + "valley_amt": np.where(work["valley"].to_numpy(bool) & tradable, amt, 0.0), + "valley_vol": np.where(work["valley"].to_numpy(bool) & tradable, vol, 0.0), + "ridge_amt": np.where(work["ridge"].to_numpy(bool) & tradable, amt, 0.0), + "ridge_vol": np.where(work["ridge"].to_numpy(bool) & tradable, vol, 0.0), + "day_amt": np.where(tradable, amt, 0.0), + "day_vol": np.where(tradable, vol, 0.0), + "peak_amt": np.where(work["peak"].to_numpy(bool) & amt_ok, amt, 0.0), + "ridge_amt2": np.where(work["ridge"].to_numpy(bool) & amt_ok, amt, 0.0), + } + ) + return g.groupby("trade_date", sort=True).sum() + + +# --------------------------------------------------------------------------- # +# hand factor values (one (d, symbol) at a time; each restates the LOCKED +# definition in plain arithmetic) +# --------------------------------------------------------------------------- # +def _lookback_slice(days: list, d: pd.Timestamp, n: int) -> list: + j = days.index(d) + return days[max(0, j - n + 1) : j + 1] + + +def hand_volume_peak_count(symbol: str, d: pd.Timestamp) -> float: + bars = read_minutes(symbol, d - pd.Timedelta(days=200), d) + work = classify(bars) + st = day_stats(work) + valid = st.index[st["classifiable"] >= MIN_CLASSIFIABLE].tolist() + if d not in valid: + return float("nan") + win = _lookback_slice(valid, d, VPC_LOOKBACK) + if len(win) < MIN_VALID_DAYS: + return float("nan") + return float(st.loc[win, "peak_any"].sum()) + + +def hand_peak_interval_kurtosis(symbol: str, d: pd.Timestamp) -> float: + bars = read_minutes(symbol, d - pd.Timedelta(days=200), d) + work = classify(bars) + st = day_stats(work) + valid = st.index[st["classifiable"] >= MIN_CLASSIFIABLE].tolist() + if d not in valid: + return float("nan") + win = _lookback_slice(valid, d, PIK_LOOKBACK) + if len(win) < MIN_VALID_DAYS: + return float("nan") + pool = [] + for day in win: + sub = work[work["trade_date"] == day].reset_index(drop=True) + pos = np.arange(len(sub)) + pk = pos[sub["peak"].to_numpy(bool)] + pool.extend(np.diff(pk).tolist()) + x = np.array(pool, dtype=float) + if x.size < PIK_MIN_INTERVALS or x.size < 4: + return float("nan") + dctr = x - x.mean() + m2 = float(np.dot(dctr, dctr)) + if not m2 > 0: + return float("nan") + m4 = float(np.dot(dctr * dctr, dctr * dctr)) + n = x.size + return n * (n + 1.0) * (n - 1.0) * m4 / ((n - 2.0) * (n - 3.0) * m2 * m2) - 3.0 * ( + n - 1.0 + ) ** 2 / ((n - 2.0) * (n - 3.0)) + + +def hand_valley_relative_vwap(symbol: str, d: pd.Timestamp) -> float: + bars = read_minutes(symbol, d - pd.Timedelta(days=200), d) + st = day_stats(classify(bars)) + valid_mask = ( + (st["classifiable"] >= MIN_CLASSIFIABLE) + & (st["valley_n"] >= VVW_MIN_VALLEY) + & (st["day_vol"] > 0) + & (st["valley_vol"] > 0) + ) + valid = st.index[valid_mask].tolist() + if d not in valid: + return float("nan") + win = _lookback_slice(valid, d, VVW_LOOKBACK) + if len(win) < MIN_VALID_DAYS: + return float("nan") + ratios = [ + (st.loc[t, "valley_amt"] / st.loc[t, "valley_vol"]) + / (st.loc[t, "day_amt"] / st.loc[t, "day_vol"]) + for t in win + ] + return float(np.mean(ratios)) + + +def hand_valley_ridge_vwap_ratio(symbol: str, d: pd.Timestamp) -> float: + bars = read_minutes(symbol, d - pd.Timedelta(days=200), d) + st = day_stats(classify(bars)) + valid_mask = ( + (st["classifiable"] >= MIN_CLASSIFIABLE) + & (st["valley_n"] >= VRV_MIN_VALLEY) + & (st["ridge_n"] >= VRV_MIN_RIDGE) + & (st["valley_vol"] > 0) + & (st["ridge_vol"] > 0) + ) + valid = st.index[valid_mask].tolist() + if d not in valid: + return float("nan") + win = _lookback_slice(valid, d, VRV_LOOKBACK) + if len(win) < MIN_VALID_DAYS: + return float("nan") + ratios = [ + (st.loc[t, "valley_amt"] / st.loc[t, "valley_vol"]) + / (st.loc[t, "ridge_amt"] / st.loc[t, "ridge_vol"]) + for t in win + ] + return float(np.mean(ratios)) + + +def hand_ridge_minute_return(symbol: str, d: pd.Timestamp) -> float: + bars = read_minutes(symbol, d - pd.Timedelta(days=200), d) + work = classify(bars) + close = work["close"].to_numpy(float) + td = work["trade_date"].to_numpy() + prev = np.full(len(work), np.nan) + prev[1:] = np.where(td[1:] == td[:-1], close[:-1], np.nan) + has_ret = np.isfinite(close) & (close > 0) & np.isfinite(prev) & (prev > 0) + ret = np.where(has_ret, close / np.where(has_ret, prev, 1.0) - 1.0, 0.0) + ridge_ret = work["ridge"].to_numpy(bool) & has_ret + per = pd.DataFrame( + { + "trade_date": td, + "s": np.where(ridge_ret, ret, 0.0), + "n": ridge_ret.astype(int), + "c": work["classifiable"].to_numpy(bool).astype(int), + } + ).groupby("trade_date", sort=True).sum() + valid = per.index[(per["c"] >= MIN_CLASSIFIABLE) & (per["n"] >= RMR_MIN_RIDGE)].tolist() + if d not in valid: + return float("nan") + win = _lookback_slice(valid, d, RMR_LOOKBACK) + if len(win) < MIN_VALID_DAYS: + return float("nan") + return float(per.loc[win, "s"].sum()) + + +def hand_peak_ridge_amount_ratio(symbol: str, d: pd.Timestamp) -> float: + bars = read_minutes(symbol, d - pd.Timedelta(days=200), d) + st = day_stats(classify(bars)) + valid_mask = ( + (st["classifiable"] >= MIN_CLASSIFIABLE) + & (st["peak_n"] >= PRA_MIN_PEAK) + & (st["ridge_n_amt"] >= PRA_MIN_RIDGE) + & (st["peak_amt"] > 0) + & (st["ridge_amt2"] > 0) + ) + valid = st.index[valid_mask].tolist() + if d not in valid: + return float("nan") + win = _lookback_slice(valid, d, PRA_LOOKBACK) + if len(win) < MIN_VALID_DAYS: + return float("nan") + return float(st.loc[win, "peak_amt"].sum() / st.loc[win, "ridge_amt2"].sum()) + + +def hand_jump_amount_corr(symbol: str, d: pd.Timestamp) -> float: + # FULL-day bars: the engine's compute has NO 14:50 cutoff (PR-C predates + # the cutoff convention; its PIT contract is day-level, dates <= d only). + bars = read_minutes(symbol, d - pd.Timedelta(days=90), d, pit=False) + ok = (bars["open"].to_numpy(float) > 0) & np.isfinite(bars["amount"].to_numpy(float)) + work = bars[ok].sort_values("bar_end").reset_index(drop=True) + days = sorted(work["trade_date"].unique()) + if d not in days: + return float("nan") + win = set(_lookback_slice(days, d, JUMP_LOOKBACK)) + xs, ys = [], [] + for day, sub in work.groupby("trade_date", sort=True): + if day not in win: + continue + amp = (sub["high"].to_numpy(float) - sub["low"].to_numpy(float)) / sub[ + "open" + ].to_numpy(float) + mu, sd = amp.mean(), amp.std(ddof=1) + z = (amp - mu) / sd if sd > 0 else np.full_like(amp, np.nan) + be = sub["bar_end"].to_numpy("datetime64[ns]").astype("int64") + amt = sub["amount"].to_numpy(float) + for i in range(len(sub) - 1): + if np.isfinite(z[i]) and z[i] > JUMP_Z and be[i + 1] - be[i] == 60_000_000_000: + xs.append(amt[i]) + ys.append(amt[i + 1]) + n = len(xs) + if n < JUMP_MIN_PAIRS: + return float("nan") + x, y = np.array(xs), np.array(ys) + cov = n * float((x * y).sum()) - float(x.sum()) * float(y.sum()) + vx = n * float((x * x).sum()) - float(x.sum()) ** 2 + vy = n * float((y * y).sum()) - float(y.sum()) ** 2 + den = np.sqrt(vx * vy) + if not den > 0: + return float("nan") + return float(np.clip(cov / den, -1.0, 1.0)) + + +def hand_minute_ideal_amp(symbol: str, d: pd.Timestamp) -> float: + bars = read_minutes(symbol, d - pd.Timedelta(days=60), d) + low = bars["low"].to_numpy(float) + high = bars["high"].to_numpy(float) + work = bars[(low > 0) & (high >= low)].reset_index(drop=True) + days = sorted(work["trade_date"].unique()) + if d not in days: + return float("nan") + win = set(_lookback_slice(days, d, IDEAL_LOOKBACK)) + pool = work[work["trade_date"].isin(win)] + closes = pool["close"].to_numpy(float) + amps = pool["high"].to_numpy(float) / pool["low"].to_numpy(float) - 1.0 + bar_ns = pool["bar_end"].to_numpy("datetime64[ns]").astype("int64") + n = closes.size + if n < IDEAL_MIN_MINUTES: + return float("nan") + k = int(np.floor(IDEAL_LAM * n)) + if k < 1: + return float("nan") + order = np.lexsort((bar_ns, closes)) + a = amps[order] + return float(a[-k:].mean() - a[:k].mean()) + + +def hand_amp_anomaly(symbol: str, d: pd.Timestamp) -> float: + """5min resample by hand (ceil bar_end to 5min; OHLC first/max/min/last). + + ORDER MATTERS and mirrors the engine's definition: the FULL day is + resampled to 5min FIRST, then the DERIVED bar is PIT-filtered by its own + ``available_time = max(source 1min bar_end) + 1min <= trade_date + 14:50`` + — so a bucket containing any post-14:49 constituent is dropped WHOLE (a + partial bucket must never masquerade as a visible 5min bar). + """ + bars = read_minutes(symbol, d - pd.Timedelta(days=90), d, pit=False) + if bars.empty: + return float("nan") + b = bars.sort_values("bar_end").copy() + b["bucket"] = b["bar_end"].dt.ceil("5min") + coarse = b.groupby("bucket", sort=True).agg( + high=("high", "max"), low=("low", "min"), close=("close", "last"), + bar_end=("bar_end", "max"), + ).reset_index(drop=True) + coarse["trade_date"] = coarse["bar_end"].dt.normalize() + # PIT on the DERIVED bar: available = max(source bar_end) + 1min. + visible = (coarse["bar_end"] + pd.Timedelta("1min")) <= ( + coarse["trade_date"] + pd.Timedelta("14:50:00") + ) + coarse = coarse[visible].reset_index(drop=True) + low = coarse["low"].to_numpy(float) + high = coarse["high"].to_numpy(float) + coarse = coarse[(low > 0) & (high >= low)].reset_index(drop=True) + days = sorted(coarse["trade_date"].unique()) + if d not in days: + return float("nan") + win = set(_lookback_slice(days, d, ANOM_LOOKBACK)) + dabs_all, ret_all = [], [] + for day, sub in coarse.groupby("trade_date", sort=True): + if day not in win: + continue + amp = sub["high"].to_numpy(float) / sub["low"].to_numpy(float) - 1.0 + close = sub["close"].to_numpy(float) + dab = np.abs(np.diff(amp)) + ret = close[1:] / close[:-1] - 1.0 + keep = np.isfinite(dab) & np.isfinite(ret) + dabs_all.extend(dab[keep].tolist()) + ret_all.extend(ret[keep].tolist()) + dabs = np.array(dabs_all) + rets = np.array(ret_all) + if dabs.size < ANOM_MIN_POOL: + return float("nan") + thr = dabs.mean() + ANOM_K * dabs.std(ddof=1) + sel = dabs > thr + if int(sel.sum()) < ANOM_MIN_SELECTED: + return float("nan") + return float(rets[sel].std(ddof=1)) + + +def _amp_cut_stats_one(symbol: str, d: pd.Timestamp) -> tuple[float, float]: + bars = read_minutes(symbol, d - pd.Timedelta(days=60), d) + if bars.empty: + return float("nan"), float("nan") + low = bars["low"].to_numpy(float) + high = bars["high"].to_numpy(float) + work = bars[(low > 0) & (high >= low)].sort_values("bar_end").reset_index(drop=True) + vdays, vvals = [], [] + for day, sub in work.groupby("trade_date", sort=True): + amp = sub["high"].to_numpy(float) / sub["low"].to_numpy(float) - 1.0 + close = sub["close"].to_numpy(float) + ret = np.full(len(sub), np.nan) + ret[1:] = close[1:] / close[:-1] - 1.0 + be = sub["bar_end"].to_numpy("datetime64[ns]").astype("int64") + valid = np.isfinite(amp) & np.isfinite(ret) + a, r, b_ns = amp[valid], ret[valid], be[valid] + if a.size < CUT_MIN_DAY: + continue + k = int(np.floor(CUT_LAM * a.size)) + if k < 1: + continue + order = np.lexsort((b_ns, r)) + sa = a[order] + vdays.append(day) + vvals.append(float(sa[-k:].mean() - sa[:k].mean())) + if d not in vdays: + return float("nan"), float("nan") + win_vals = [v for t, v in zip(vdays, vvals) if t in set(_lookback_slice(vdays, d, CUT_LOOKBACK))] + if len(win_vals) < CUT_MIN_VALID: + return float("nan"), float("nan") + arr = np.array(win_vals) + return float(arr.mean()), float(arr.std(ddof=1)) + + +def hand_intraday_amp_cut(symbols: list[str], target: str, d: pd.Timestamp) -> float: + """Full cross-sectional z-combine by hand: needs EVERY covered symbol's stats.""" + vm, vs, names = [], [], [] + for s in symbols: + m, sd = _amp_cut_stats_one(s, d) + if np.isfinite(m) and np.isfinite(sd): + vm.append(m) + vs.append(sd) + names.append(s) + if target not in names or len(names) < CUT_MIN_XS: + return float("nan") + vm_a, vs_a = np.array(vm), np.array(vs) + sm, ss = vm_a.std(ddof=1), vs_a.std(ddof=1) + if not (np.isfinite(sm) and np.isfinite(ss)) or sm == 0 or ss == 0: + return float("nan") + zm = (vm_a - vm_a.mean()) / sm + zs = (vs_a - vs_a.mean()) / ss + return float(((zm + zs) / 2.0)[names.index(target)]) + + +def _vpq_qbar_one(symbol: str, d: pd.Timestamp) -> float: + bars = read_minutes(symbol, d - pd.Timedelta(days=200), d) + if bars.empty: + return float("nan") + work = classify(bars) + vol = work["volume"].to_numpy(float) + amt = work["amount"].to_numpy(float) + high = work["high"].to_numpy(float) + low = work["low"].to_numpy(float) + close = work["close"].to_numpy(float) + tradable = np.isfinite(vol) & (vol > 0) & np.isfinite(amt) & (amt > 0) + priced = np.isfinite(high) & np.isfinite(low) & (low > 0) & (high >= low) + valley = work["valley"].to_numpy(bool) & tradable + per = pd.DataFrame( + { + "trade_date": work["trade_date"].to_numpy(), + "v_amt": np.where(valley, amt, 0.0), + "v_vol": np.where(valley, vol, 0.0), + "v_n": valley.astype(int), + "c_n": work["classifiable"].to_numpy(bool).astype(int), + "hi": np.where(priced, high, -np.inf), + "lo": np.where(priced, low, np.inf), + } + ) + g = per.groupby("trade_date", sort=True) + agg = g[["v_amt", "v_vol", "v_n", "c_n"]].sum() + agg["hi"] = g["hi"].max() + agg["lo"] = g["lo"].min() + usable = np.isfinite(close) & (close > 0) + lc = pd.DataFrame( + {"trade_date": work["trade_date"].to_numpy()[usable], "close": close[usable]} + ).groupby("trade_date", sort=True)["close"].last() + pc = lc.reindex(agg.index).shift(1) + hi = np.maximum(agg["hi"].to_numpy(float), pc.to_numpy(float)) + lo = np.minimum(agg["lo"].to_numpy(float), pc.to_numpy(float)) + valid = ( + (agg["c_n"].to_numpy() >= MIN_CLASSIFIABLE) + & (agg["v_n"].to_numpy() >= VPQ_MIN_VALLEY) + & (agg["v_vol"].to_numpy(float) > 0) + & np.isfinite(pc.to_numpy(float)) + & np.isfinite(hi) & np.isfinite(lo) & (hi > lo) + ) + vdays = list(agg.index[valid]) + if d not in vdays: + return float("nan") + q = ( + agg["v_amt"].to_numpy(float)[valid] / agg["v_vol"].to_numpy(float)[valid] - lo[valid] + ) / (hi[valid] - lo[valid]) + qs = pd.Series(q, index=agg.index[valid]) + win = _lookback_slice(vdays, d, VPQ_LOOKBACK) + if len(win) < MIN_VALID_DAYS: + return float("nan") + return float(qs.loc[win].mean()) + + +def _qfq_close(symbol: str) -> pd.Series: + md = read_daily(symbol, "market_daily") + if md.empty: + return pd.Series(dtype=float) + af = af_series(symbol) + close = md.set_index("date")["close"] + afx = af.reindex(close.index) + anchor = afx.dropna().iloc[-1] if afx.notna().any() else np.nan + return close * afx / anchor + + +def hand_valley_price_quantile(symbols: list[str], target: str, d: pd.Timestamp) -> float: + """Cross-sectional OLS residual of qbar on rev20 (T-1, qfq) BY HAND.""" + qbars, revs, names = [], [], [] + for s in symbols: + qb = _vpq_qbar_one(s, d) + if not np.isfinite(qb): + continue + qfq = _qfq_close(s) + qfq = qfq[np.isfinite(qfq.to_numpy(float)) & (qfq.to_numpy(float) > 0)] + if d not in qfq.index: + continue + j = qfq.index.get_loc(d) + if j < VPQ_REV_DAYS + 1: + continue + rev = -(qfq.iloc[j - 1] / qfq.iloc[j - (VPQ_REV_DAYS + 1)] - 1.0) + if not np.isfinite(rev): + continue + qbars.append(qb) + revs.append(float(rev)) + names.append(s) + if target not in names or len(names) < VPQ_MIN_XS: + return float("nan") + x, y = np.array(revs), np.array(qbars) + xm = x.mean() + sxx = float(((x - xm) ** 2).sum()) + if not sxx > 0: + return float("nan") + ym = y.mean() + slope = float(((x - xm) * (y - ym)).sum()) / sxx + resid = y - (ym + slope * (x - xm)) + return float(resid[names.index(target)]) + + +def hand_value_ratio(symbol: str, d: pd.Timestamp, field: str) -> float: + db = read_daily(symbol, "daily_basic") + if db.empty: + return float("nan") + row = db[db["date"] == d] + if row.empty: + return float("nan") + v = float(row.iloc[0][field]) + return 1.0 / v if np.isfinite(v) and v > 0 else float("nan") + + +def hand_volatility_20(symbol: str, d: pd.Timestamp) -> float: + qfq = _qfq_close(symbol) + if d not in qfq.index: + return float("nan") + j = qfq.index.get_loc(d) + if j < 20: + return float("nan") + window_prices = qfq.iloc[j - 20 : j + 1].to_numpy(float) + rets = window_prices[1:] / window_prices[:-1] - 1.0 + if np.isnan(rets).any(): + return float("nan") + return float(np.std(rets, ddof=1)) + + +# NOTE on the qfq anchor: front_adjust anchors on the symbol's LAST adj_factor +# row in the loaded window; _qfq_close reproduces that from the raw caches. + +# --------------------------------------------------------------------------- # +# runtime import guard: the hand computation must not have loaded the engine +# --------------------------------------------------------------------------- # +def assert_no_engine_imports() -> None: + loaded = [ + m for m in sys.modules + if m.startswith("factors.") or m.startswith("data.clean") + or m in ("factors", ) + ] + if loaded: + raise RuntimeError( + f"hand-anchor purity violated: engine modules imported: {loaded}" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--selection", default=str(OUT_JSON)) + args = parser.parse_args(argv) + assert_no_engine_imports() + from qt.hand_anchor_rows import run_hand_anchors # heavy driver, engine-free + + return run_hand_anchors(Path(args.selection)) + + +if __name__ == "__main__": # pragma: no cover - thin CLI shim + raise SystemExit(main()) diff --git a/qt/hand_anchors_engine_values.py b/qt/hand_anchors_engine_values.py new file mode 100644 index 0000000..8f27af5 --- /dev/null +++ b/qt/hand_anchors_engine_values.py @@ -0,0 +1,99 @@ +"""Engine-side comparison for the four daily hand anchors (D2, R12 companion). + +``qt.hand_anchors_d2`` / ``qt.hand_anchor_rows`` hand-compute momentum_20 / +reversal_20 / liquidity_20 / overnight_mom_20 anchor rows WITHOUT importing the +engine and leave them in ``hand_anchors_d2.json`` under +``daily_pending_engine``. This companion is the one place ALLOWED to import +the engine: it rebuilds the freeze data plane (same config, cache-only), runs +the ops-rewritten factor classes, and compares each pending row at the same +<= 1e-12 relative tolerance. The independence requirement binds the HAND side, +not the comparer (module docstring of hand_anchors_d2). + +Run AFTER ``python -m qt.hand_anchor_rows``: + + python -m qt.hand_anchors_engine_values +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import numpy as np + +from qt.hand_anchors_d2 import OUT_JSON, TOL + + +def main(argv: list[str] | None = None) -> int: + import pandas as pd + + from factors.compute.candidates import ( + LiquidityFactor, + OvernightMomentumFactor, + ReversalFactor, + ) + from factors.compute.momentum import MomentumFactor + from qt.config import load_config + from qt.eval_jump_amount_corr import _check_preconditions + from qt.panel_freeze import DEFAULT_CONFIG, PANEL_STORE_NAME + from qt.pipeline import _build_cache, _build_universe, _load_panel, _make_logger + + payload = json.loads(Path(OUT_JSON).read_text(encoding="utf-8")) + pending = payload.get("daily_pending_engine", []) + if not pending: + print("no pending daily rows; run qt.hand_anchor_rows first") + return 1 + + cfg = load_config(DEFAULT_CONFIG) + _check_preconditions(cfg) + cfg = cfg.model_copy( + update={"data": cfg.data.model_copy(update={"output_name": PANEL_STORE_NAME})} + ) + logger = _make_logger( + Path(cfg.output.log_dir) / "hand_anchors_engine.log", + name="qt.hand_anchors_engine", + ) + cache = _build_cache(cfg) + _, symbols = _build_universe(cfg, logger, cache) + panel = _load_panel(cfg, symbols, logger, cache) + + factories = { + "momentum_20": MomentumFactor(window=20), + "reversal_20": ReversalFactor(window=20), + "liquidity_20": LiquidityFactor(window=20), + "overnight_mom_20": OvernightMomentumFactor(window=20), + } + values = {name: f.compute(panel) for name, f in factories.items()} + + n_bad = 0 + out_rows = [] + for row in pending: + series = values[row["factor_id"]] + key = (pd.Timestamp(row["date"]), row["symbol"]) + engine = float(series.loc[key]) if key in series.index else float("nan") + hand = row["hand"] + if np.isfinite(hand) and np.isfinite(engine): + denom = max(abs(hand), abs(engine)) + rel = abs(hand - engine) / denom if denom > 0 else 0.0 + elif np.isnan(hand) and np.isnan(engine): + rel = 0.0 + else: + rel = float("inf") + ok = rel <= TOL + n_bad += 0 if ok else 1 + out_rows.append({**row, "engine": engine, "rel_diff": rel, "ok": bool(ok)}) + print( + f"{'OK ' if ok else 'FAIL'} {row['factor_id']:18s} {row['class']:12s} " + f"{row['date']} {row['symbol']} hand={hand!r} engine={engine!r} rel={rel:.2e}" + ) + + payload["daily_engine_compared"] = out_rows + payload["all_ok_daily"] = n_bad == 0 + Path(OUT_JSON).write_text(json.dumps(payload, indent=2), encoding="utf-8") + print(f"daily anchors: {len(out_rows)} rows, {n_bad} mismatches -> {OUT_JSON}") + return 0 if n_bad == 0 else 1 + + +if __name__ == "__main__": # pragma: no cover - thin CLI shim + sys.exit(main()) diff --git a/qt/panel_reconcile.py b/qt/panel_reconcile.py new file mode 100644 index 0000000..768a672 --- /dev/null +++ b/qt/panel_reconcile.py @@ -0,0 +1,331 @@ +"""D2 cell-by-cell reconciliation of the migrated engine vs the D1 frozen baseline. + +Design v3.2 §五 leg 4 (R11), pulled forward into D2 by the task card: rebuild +all 14 closing factors' RAW panels on the CURRENT tree (the ``qt.panel_freeze`` +recipes route through the ``data.clean`` shims, so they exercise the migrated +``factors.compute.minute`` math automatically), write them to a SEPARATE +directory, and compare each panel CELL BY CELL against the frozen D1 baseline: + +* the (date, symbol) index sets must be IDENTICAL; +* the NaN sets must be IDENTICAL (a NaN-set change would be an undeclared + NaN-policy change — D2 declares none); +* on the jointly-finite cells the relative difference + ``|new - old| / max(|old|, |new|)`` must be <= 1e-12 (the float-reordering + budget; any excess means STOP AND FIX THE ENGINE, never widen the budget). + +Anti-empty-reconciliation provenance (the ``compare_postmerge.py`` lesson): + +* the frozen side is NEVER regenerated here — it is read from + ``artifacts/refactor_baseline/panels`` exactly as D1 froze it, and each + frozen file's canonical content hash is first checked against the D1 + ``manifest.json`` (a stale/clobbered baseline fails loudly before any + comparison is trusted); +* the comparison reads BOTH sides from disk through two independent file + reads — the freshly built panel is written to + ``artifacts/refactor_baseline/panels_d2`` first and read back, so no shared + in-memory object can make the equality vacuous. + +Run: ``python -m qt.panel_reconcile`` (deliberately NOT registered in qt/cli; +``--resume`` reads back already-written panels_d2 files after checking that +they were produced at the SAME git SHA recorded in ``manifest_d2.json``). +""" + +from __future__ import annotations + +import argparse +import json +import time +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from data.clean.schema import DATE_LEVEL +from qt.panel_freeze import ( + DEFAULT_CONFIG, + PANEL_STORE_NAME, + atomic_write_parquet, + atomic_write_text, + canonical_content_hash, + minute_recipes, + read_frozen_panel, + _git_head_sha, +) + +DEFAULT_BASELINE_ROOT = "artifacts/refactor_baseline" +D2_SUBDIR = "panels_d2" +RELATIVE_TOLERANCE = 1e-12 + + +@dataclass(frozen=True) +class CellComparison: + """One factor's cell-by-cell comparison result (no secrets).""" + + factor_id: str + rows: int + index_equal: bool + nan_only_in_frozen: int + nan_only_in_new: int + n_joint_finite: int + n_cells_beyond_tol: int + max_rel_diff: float + max_abs_diff: float + hashes_equal: bool + + @property + def ok(self) -> bool: + return ( + self.index_equal + and self.nan_only_in_frozen == 0 + and self.nan_only_in_new == 0 + and self.n_cells_beyond_tol == 0 + ) + + +def compare_panels(frozen: pd.Series, new: pd.Series, factor_id: str) -> CellComparison: + """Cell-by-cell comparison per the module docstring's three rules.""" + index_equal = frozen.index.equals(new.index) + if not index_equal: + return CellComparison( + factor_id=factor_id, + rows=int(len(frozen)), + index_equal=False, + nan_only_in_frozen=-1, + nan_only_in_new=-1, + n_joint_finite=0, + n_cells_beyond_tol=-1, + max_rel_diff=float("nan"), + max_abs_diff=float("nan"), + hashes_equal=False, + ) + a = frozen.to_numpy(dtype=float) + b = new.to_numpy(dtype=float) + nan_a = np.isnan(a) + nan_b = np.isnan(b) + joint = ~nan_a & ~nan_b + diff = np.abs(a[joint] - b[joint]) + denom = np.maximum(np.abs(a[joint]), np.abs(b[joint])) + with np.errstate(invalid="ignore", divide="ignore"): + rel = np.where(denom > 0.0, diff / denom, 0.0) + beyond = int((rel > RELATIVE_TOLERANCE).sum()) + return CellComparison( + factor_id=factor_id, + rows=int(len(frozen)), + index_equal=True, + nan_only_in_frozen=int((nan_a & ~nan_b).sum()), + nan_only_in_new=int((nan_b & ~nan_a).sum()), + n_joint_finite=int(joint.sum()), + n_cells_beyond_tol=beyond, + max_rel_diff=float(rel.max()) if rel.size else 0.0, + max_abs_diff=float(diff.max()) if diff.size else 0.0, + hashes_equal=canonical_content_hash(frozen) == canonical_content_hash(new), + ) + + +def _verify_frozen_against_manifest(baseline_root: Path) -> dict[str, str]: + """Check every frozen panel's canonical hash against the D1 manifest. + + Returns {factor_id: canonical_hash}. Any mismatch raises — a clobbered or + regenerated baseline must never be silently reconciled against. + """ + manifest = json.loads((baseline_root / "manifest.json").read_text(encoding="utf-8")) + frozen_hashes: dict[str, str] = {} + for row in manifest["rows"]: + factor_id = row["factor_id"] + path = baseline_root / "panels" / row["file"] + frozen = read_frozen_panel(path, factor_id) + actual = canonical_content_hash(frozen) + if actual != row["canonical_sha256"]: + raise RuntimeError( + f"frozen baseline {row['file']} canonical hash {actual} does not " + f"match the D1 manifest ({row['canonical_sha256']}) — the baseline " + "has been altered since the freeze; refusing to reconcile." + ) + frozen_hashes[factor_id] = actual + return frozen_hashes + + +def render_report(rows: list[CellComparison], header: dict) -> str: + lines = ["# D2 panel reconciliation vs the D1 frozen baseline", ""] + for key in sorted(header): + lines.append(f"- **{key}**: {header[key]}") + lines.append("") + cols = ( + "factor_id | rows | index_equal | nan_only_frozen | nan_only_new | " + "joint_finite | cells_beyond_1e-12 | max_rel_diff | max_abs_diff | " + "hash_equal | verdict" + ).split(" | ") + lines.append("| " + " | ".join(cols) + " |") + lines.append("|" + "|".join("---" for _ in cols) + "|") + for r in rows: + lines.append( + f"| {r.factor_id} | {r.rows} | {r.index_equal} | " + f"{r.nan_only_in_frozen} | {r.nan_only_in_new} | {r.n_joint_finite} | " + f"{r.n_cells_beyond_tol} | {r.max_rel_diff!r} | {r.max_abs_diff!r} | " + f"{r.hashes_equal} | {'OK' if r.ok else 'MISMATCH'} |" + ) + lines.append("") + return "\n".join(lines) + + +def run_panel_reconcile( + config_path: str = DEFAULT_CONFIG, + baseline_root: str | Path = DEFAULT_BASELINE_ROOT, + *, + resume: bool = False, +) -> list[CellComparison]: + """Rebuild the 14 RAW panels on the current tree and reconcile vs the freeze.""" + from qt.config import load_config + from qt.eval_jump_amount_corr import _build_book_factors, _check_preconditions + from qt.pipeline import ( + _build_cache, + _build_universe, + _load_panel, + _log_run_cache_stats, + _make_logger, + _maybe_enrich_covariates, + _maybe_enrich_value, + ) + + started = time.monotonic() + root = Path(baseline_root) + d2_dir = root / D2_SUBDIR + d2_dir.mkdir(parents=True, exist_ok=True) + head_sha = _git_head_sha() + + # provenance for --resume: a panels_d2 file may only be reused if it was + # produced at the SAME git SHA (else it might carry an older engine). + manifest_d2_path = root / "manifest_d2.json" + prior: dict = {} + if manifest_d2_path.exists(): + prior = json.loads(manifest_d2_path.read_text(encoding="utf-8")) + if resume and prior and prior.get("producing_git_sha") != head_sha: + raise RuntimeError( + f"--resume refused: existing panels_d2 were produced at " + f"{prior.get('producing_git_sha')} but HEAD is {head_sha}; delete " + f"{d2_dir} to rebuild from scratch." + ) + + frozen_hashes = _verify_frozen_against_manifest(root) + + cfg = load_config(config_path) + _check_preconditions(cfg) + cfg = cfg.model_copy( + update={"data": cfg.data.model_copy(update={"output_name": PANEL_STORE_NAME})} + ) + log_path = Path(cfg.output.log_dir) / "panel_reconcile.log" + logger = _make_logger(log_path, name="qt.panel_reconcile") + logger.info("panel reconcile: config=%s baseline_root=%s sha=%s", + config_path, root, head_sha) + + cache = _build_cache(cfg) + universe, symbols = _build_universe(cfg, logger, cache) + panel = _load_panel(cfg, symbols, logger, cache) + book_factors = _build_book_factors() + panel = _maybe_enrich_value(cfg, panel, symbols, book_factors, logger, cache) + panel = _maybe_enrich_covariates(cfg, panel, symbols, logger, cache) + _log_run_cache_stats(cache, logger) + panel_dates = pd.Index( + pd.unique(panel.index.get_level_values(DATE_LEVEL)), name=DATE_LEVEL + ) + + results: list[CellComparison] = [] + d2_rows: list[dict] = [] + + def _reconcile_one(factor_id: str, raw: pd.Series | None) -> None: + target = d2_dir / f"{factor_id}.parquet" + if raw is None: # resume path: reuse the already-written d2 panel + logger.info("resume: reading back %s", target.name) + else: + atomic_write_parquet(raw.rename(factor_id), target) + new = read_frozen_panel(target, factor_id) # file read #1 + frozen = read_frozen_panel(root / "panels" / f"{factor_id}.parquet", factor_id) # #2 + comp = compare_panels(frozen, new, factor_id) + results.append(comp) + d2_rows.append( + {"factor_id": factor_id, "canonical_sha256": canonical_content_hash(new), + "frozen_sha256": frozen_hashes[factor_id], "rows": comp.rows} + ) + logger.info( + "reconciled %s: ok=%s max_rel=%.3e nan_frozen_only=%d nan_new_only=%d", + factor_id, comp.ok, comp.max_rel_diff, + comp.nan_only_in_frozen, comp.nan_only_in_new, + ) + + # book factors (raw, pre-processing) — same call shape as the freeze + for factor in book_factors: + target = d2_dir / f"{factor.name}.parquet" + if resume and target.exists(): + _reconcile_one(factor.name, None) + else: + _reconcile_one(factor.name, factor.compute(panel).rename(factor.name)) + + # minute factors — the freeze recipes, now routed through the D2 shims + live_total = 0 + for recipe in minute_recipes(): + target = d2_dir / f"{recipe.factor_id}.parquet" + if resume and target.exists(): + _reconcile_one(recipe.factor_id, None) + continue + load = recipe.build(cfg, symbols, panel, logger) + live = int(load.live_calls) + live_total += live + if live != 0: + raise RuntimeError( + f"{recipe.factor_id}: stk_mins_live_calls={live} != 0 — the " + "reconciliation is cache-only by contract." + ) + raw = load.factor[ + load.factor.index.get_level_values(DATE_LEVEL).isin(panel_dates) + ].rename(recipe.factor_id) + _reconcile_one(recipe.factor_id, raw) + + header = { + "producing_git_sha": head_sha, + "config": str(config_path), + "baseline_root": str(root), + "relative_tolerance": repr(RELATIVE_TOLERANCE), + "stk_mins_live_calls": live_total, + "panels": len(results), + "all_ok": all(r.ok for r in results), + "elapsed_seconds": round(time.monotonic() - started, 1), + "generated_utc": pd.Timestamp.now(tz="UTC").strftime("%Y-%m-%d %H:%M:%S"), + } + atomic_write_text( + json.dumps({"producing_git_sha": head_sha, "rows": d2_rows, "header": header}, + indent=2, sort_keys=True), + manifest_d2_path, + ) + atomic_write_text(render_report(results, header), root / "reconcile_d2.md") + logger.info("panel reconcile complete: all_ok=%s (%ss)", + header["all_ok"], header["elapsed_seconds"]) + return results + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--config", default=DEFAULT_CONFIG) + parser.add_argument("--baseline-root", default=DEFAULT_BASELINE_ROOT) + parser.add_argument( + "--resume", action="store_true", + help="reuse already-written panels_d2 files (same-SHA guarded)", + ) + args = parser.parse_args(argv) + results = run_panel_reconcile( + args.config, args.baseline_root, resume=args.resume + ) + for r in results: + print( + f"{'OK ' if r.ok else 'MISMATCH'} {r.factor_id}: rows={r.rows} " + f"max_rel={r.max_rel_diff:.3e} max_abs={r.max_abs_diff:.3e} " + f"nan_frozen_only={r.nan_only_in_frozen} nan_new_only={r.nan_only_in_new} " + f"hash_equal={r.hashes_equal}" + ) + ok = all(r.ok for r in results) + print(f"reconcile: {'ALL OK' if ok else 'MISMATCH — stop and fix the engine'}") + return 0 if ok else 1 + + +if __name__ == "__main__": # pragma: no cover - thin CLI shim + raise SystemExit(main()) diff --git a/tests/test_panel_reconcile_d2.py b/tests/test_panel_reconcile_d2.py new file mode 100644 index 0000000..8b6d65f --- /dev/null +++ b/tests/test_panel_reconcile_d2.py @@ -0,0 +1,80 @@ +"""Teeth tests for the D2 cell-by-cell panel comparator (network-free). + +A reconciliation whose comparator cannot fail is the ``compare_postmerge.py`` +failure mode; these tests feed the comparator engineered defects and assert it +CONVICTS each one — and that the one legitimate difference class (float +reordering within 1e-12) passes without being confused with hash equality. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from qt.panel_reconcile import RELATIVE_TOLERANCE, compare_panels + + +def _series(values, dates=None, symbols=("AAA", "BBB")): + dates = dates or ["2024-01-02", "2024-01-03"] + index = pd.MultiIndex.from_product( + [pd.to_datetime(dates), list(symbols)], names=["date", "symbol"] + ) + return pd.Series(list(values), index=index, dtype=float, name="f") + + +def test_identical_panels_reconcile_exactly(): + a = _series([1.0, 2.0, np.nan, 4.0]) + comp = compare_panels(a, a.copy(), "f") + assert comp.ok and comp.max_rel_diff == 0.0 and comp.hashes_equal + assert comp.nan_only_in_frozen == 0 and comp.nan_only_in_new == 0 + + +def test_float_reordering_within_budget_passes_but_hash_differs(): + a = _series([1.0, 2.0, np.nan, 4.0]) + b = a.copy() + b.iloc[0] = 1.0 * (1.0 + 1e-14) # sub-budget drift (legit reordering scale) + comp = compare_panels(a, b, "f") + assert comp.ok # within the 1e-12 budget + assert comp.max_rel_diff > 0.0 + assert not comp.hashes_equal # the hash still SEES it (no false comfort) + + +def test_value_drift_beyond_budget_is_convicted(): + a = _series([1.0, 2.0, np.nan, 4.0]) + b = a.copy() + b.iloc[3] = 4.0 * (1.0 + 1e-9) + comp = compare_panels(a, b, "f") + assert not comp.ok + assert comp.n_cells_beyond_tol == 1 + assert comp.max_rel_diff > RELATIVE_TOLERANCE + + +def test_nan_set_change_is_convicted_in_both_directions(): + a = _series([1.0, 2.0, np.nan, 4.0]) + b = _series([1.0, np.nan, np.nan, 4.0]) + comp = compare_panels(a, b, "f") + assert not comp.ok and comp.nan_only_in_new == 1 and comp.nan_only_in_frozen == 0 + comp_rev = compare_panels(b, a, "f") + assert not comp_rev.ok and comp_rev.nan_only_in_frozen == 1 + + +def test_index_mismatch_is_convicted_before_any_value_math(): + a = _series([1.0, 2.0, 3.0, 4.0]) + b = _series([1.0, 2.0, 3.0, 4.0], dates=["2024-01-02", "2024-01-04"]) + comp = compare_panels(a, b, "f") + assert not comp.ok and not comp.index_equal + + +def test_sign_flip_at_zero_magnitude_is_within_denominator_rule(): + # denominator = max(|a|,|b|): a 0.0 vs 0.0 cell contributes rel 0 (0/0 -> 0), + # while 0.0 vs 1e-30 is rel 1.0 and convicted — tiny absolute fabrications + # near zero cannot hide behind a relative rule. + a = _series([0.0, 2.0, 3.0, 4.0]) + b = a.copy() + comp = compare_panels(a, b, "f") + assert comp.ok + b2 = a.copy() + b2.iloc[0] = 1e-30 + comp2 = compare_panels(a, b2, "f") + assert not comp2.ok and comp2.max_rel_diff == pytest.approx(1.0) diff --git a/tests/test_provisional_anchor_d2.py b/tests/test_provisional_anchor_d2.py new file mode 100644 index 0000000..c5c6e7c --- /dev/null +++ b/tests/test_provisional_anchor_d2.py @@ -0,0 +1,77 @@ +"""PROVISIONAL D2 anchor: full-precision phase0 demo metrics (R10 timeline). + +Anchor timeline (design v3.2 §五 / R10): the OLD coarse anchor +``ic 0.9600 / annual 0.8408`` stayed REQUIRED through D1 (dispatch-only, math +untouched). D2 rewrote the factor implementations; after the hand anchors (R12) +and the frozen-baseline cell-by-cell reconciliation (this branch) passed, the +FULL-PRECISION metrics of the same demo run are frozen here as the PROVISIONAL +new anchor — the zero-cost CI sentinel for every D2..D5 PR. D6 promotes it. + +PROVISIONAL means: the values below are the D2-accepted engine's output and +may be re-frozen ONLY as part of an explicitly authorized engine change (with +the same reconciliation discipline), never casually edited to green a build. +Captured on this branch after commits A-C, with the D2 reconciliation showing +max relative drift 0.0 vs the D1 frozen baseline on all 14 closing factors — +i.e. these values are bit-identical to what the pre-D2 engine produced. + +The demo run is offline and deterministic (DemoFeed), so exact float equality +is the right assertion: any drift >= 1 ulp is a real engine change that must +be explained, not absorbed. +""" + +from __future__ import annotations + +import pytest + +from qt.pipeline import run_phase0 + +# Full-precision values captured from the D2-accepted engine (see module +# docstring). The coarse 4dp reading of ic_mean / annual_return matches the +# historical anchor 0.9600 / 0.8408 exactly. +PROVISIONAL_ANCHOR = { + "ic_mean": 0.9600438863169586, + "ic_ir": 7.617904227971471, + "annual_return": 0.8407986461861146, + "sharpe": 10.304260918159038, + "volatility": 0.060904296602865324, + "max_drawdown": 0.0, + "avg_turnover": 0.2121212121212121, + "cost_drag": 0.002333333333333333, +} + + +@pytest.fixture(scope="module") +def phase0_result(): + return run_phase0("config/example.yaml") + + +def test_provisional_anchor_full_precision(phase0_result): + r = phase0_result + actual = { + "ic_mean": r.ic_mean, + "ic_ir": r.ic_ir, + "annual_return": r.performance["annual_return"], + "sharpe": r.performance["sharpe"], + "volatility": r.performance["volatility"], + "max_drawdown": r.performance["max_drawdown"], + "avg_turnover": r.avg_turnover, + "cost_drag": r.cost_drag, + } + mismatches = { + k: (actual[k], expected) + for k, expected in PROVISIONAL_ANCHOR.items() + if actual[k] != expected + } + assert not mismatches, ( + "PROVISIONAL D2 anchor drift (full precision, exact-equality contract): " + f"{mismatches}. An intentional engine change must re-freeze the anchor " + "explicitly with reconciliation evidence — never edit these numbers to " + "green a build." + ) + + +def test_provisional_anchor_matches_the_retired_coarse_anchor(phase0_result): + """The historical 4dp anchor is a strict corollary of the full-precision one.""" + r = phase0_result + assert f"{r.ic_mean:.4f}" == "0.9600" + assert f"{r.performance['annual_return']:.4f}" == "0.8408" From a316b77e0814260b65712790c0c67c665acdaef6 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 07:50:50 -0700 Subject: [PATCH 5/8] fix(qt): hand-anchor calibration from the pre-flight spot-check Three corrections the frozen-panel spot-check convicted (the reason the hand reimplementation exists): (1) the minute-cache path gains its freq=1min partition level; (2) PR-C jump reads FULL-day bars (its engine has no 14:50 cutoff - day-level PIT only, a PR-C-predates-the-convention fact now recorded at the call site); (3) PR-E resamples the FULL day to 5min FIRST and PIT- filters the DERIVED bar by available_time = max(source)+1min, dropping partial buckets WHOLE. Post-fix spot-check: 24/24 rows pass (20 exact, worst rel 5.8e-15). Also widens the peak-family hand read windows to 200 calendar days (baseline-truncation safety), trims the valley-quantile cross-section read to 130 days and amortizes its five anchor rows over three distinct dates (each date costs a full-universe hand cross-section; disclosed). --- qt/hand_anchor_rows.py | 14 +++++++++----- qt/hand_anchors_d2.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/qt/hand_anchor_rows.py b/qt/hand_anchor_rows.py index 9898def..2ba9f33 100644 --- a/qt/hand_anchor_rows.py +++ b/qt/hand_anchor_rows.py @@ -255,15 +255,19 @@ def compare(factor_id, cls, key, hand, note=""): compare("intraday_amp_cut_10", cls, (d, sym), hand, note=f"cross_section={int(fin.sum())}") - # ---------------- valley price quantile (cross-section; 2 dates) -------- + # ---------------- valley price quantile (cross-section; 3 dates) -------- + # 5 rows amortized over 3 distinct dates: each date costs a full-universe + # hand cross-section (qbar for every covered symbol), so the two extra + # interior rows share the first random date (disclosed amortization). series = _panel("valley_price_quantile_20") w = _pick_warmup(series, rng) - r1, r2 = _pick_random(series, rng, 2) + (r1,) = _pick_random(series, rng, 1) e = _pick_exdate(series, rng, H.VPQ_LOOKBACK) - picks = [("warmup_end", w), ("ex_date_window", e), ("random", r1), ("random", r2)] + picks = [("warmup_end", w), ("ex_date_window", e), ("random", r1)] same_date = _finite(series).xs(r1[0], level="date", drop_level=False) - extra_sym = str(same_date.index[rng.randint(len(same_date))][1]) - picks.append(("random", (r1[0], extra_sym))) + for _ in range(2): + extra_sym = str(same_date.index[rng.randint(len(same_date))][1]) + picks.append(("random", (r1[0], extra_sym))) vpq_cache: dict = {} for cls, (d, sym) in picks: if d not in vpq_cache: diff --git a/qt/hand_anchors_d2.py b/qt/hand_anchors_d2.py index 4f3c677..e0f6a5b 100644 --- a/qt/hand_anchors_d2.py +++ b/qt/hand_anchors_d2.py @@ -540,7 +540,7 @@ def hand_intraday_amp_cut(symbols: list[str], target: str, d: pd.Timestamp) -> f def _vpq_qbar_one(symbol: str, d: pd.Timestamp) -> float: - bars = read_minutes(symbol, d - pd.Timedelta(days=200), d) + bars = read_minutes(symbol, d - pd.Timedelta(days=130), d) if bars.empty: return float("nan") work = classify(bars) From 7f6e92e8250d5478b397e76db5d5cbe4e1211809 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 07:54:32 -0700 Subject: [PATCH 6/8] fix(qt): daily hand anchors reach the R12 five-row floor momentum/reversal/overnight: warmup + ex-date + 3 random; liquidity (amount channel, ex-date class not applicable - disclosed, not faked): warmup + 4 random. --- qt/hand_anchor_rows.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qt/hand_anchor_rows.py b/qt/hand_anchor_rows.py index 2ba9f33..40f9c95 100644 --- a/qt/hand_anchor_rows.py +++ b/qt/hand_anchor_rows.py @@ -367,7 +367,10 @@ def compare(factor_id, cls, key, hand, note=""): break if ex_row is not None: rows.append(("ex_date_window", (ex_row, sym), "")) - for _ in range(2 if kind != "liquidity" else 3): + # >= 5 rows per factor (R12): warmup + ex-date + 3 random for the + # price factors, warmup + 4 random for liquidity (amount channel — the + # ex-date class is not applicable and is disclosed, not faked). + for _ in range(3 if kind != "liquidity" else 4): j = int(rng.randint(need + 1, len(dates))) rows.append(("random", (dates[j], sym), "")) for cls, (d, s), note in rows: From 76bb39c34850e85e73f28021698ec8815243c082 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 08:01:04 -0700 Subject: [PATCH 7/8] fix(qt): clip hand-anchor reads to the evaluation plane start The first G run convicted every early-window row (warmup/ex-date anchors in 2021-07/08) while every mid-panel row was exact: the minute cache holds BACKFILLED bars from before 2021-07-01, and the hand reads were pulling that richer history in, while the frozen panels were computed on store.read_range(cfg.data.start=2021-07-01, ...). read_minutes now clips its lower bound to the plane start. _pick_warmup gains a disclosed fallback for factors that only emit rows once the accumulation floor is crossed (amp_cut's combine drops non-finite stat pairs, so its panel has no leading-NaN ramp): the symbol's first emitted row IS the warm-up end. --- qt/hand_anchor_rows.py | 17 +++++++++++++++-- qt/hand_anchors_d2.py | 6 ++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/qt/hand_anchor_rows.py b/qt/hand_anchor_rows.py index 40f9c95..5d209c0 100644 --- a/qt/hand_anchor_rows.py +++ b/qt/hand_anchor_rows.py @@ -48,13 +48,26 @@ def _finite(series: pd.Series) -> pd.Series: def _pick_warmup(series: pd.Series, rng: np.random.RandomState) -> tuple: - """A symbol's FIRST finite row (its warm-up end on the factor's own axis).""" - for sym in rng.permutation(_symbols(series)): + """A symbol's FIRST finite row (its warm-up end on the factor's own axis). + + Prefers a symbol with a real leading-NaN ramp. Some factors emit rows only + once every accumulation floor is already crossed (amp_cut's combine drops + non-finite stat pairs), so no ramp exists on the panel — then the symbol's + FIRST EMITTED row IS the warm-up end and is used as the anchor. + """ + syms = rng.permutation(_symbols(series)) + for sym in syms: sub = series.xs(sym, level="symbol").sort_index() fin = np.isfinite(sub.to_numpy(float)) if fin.any() and not fin[0]: # a real warm-up ramp exists j = int(np.argmax(fin)) return (sub.index[j], str(sym)) + for sym in syms: # fallback: first emitted (already finite) row + sub = series.xs(sym, level="symbol").sort_index() + fin = np.isfinite(sub.to_numpy(float)) + if fin.any(): + j = int(np.argmax(fin)) + return (sub.index[j], str(sym)) raise RuntimeError("no warm-up row found") diff --git a/qt/hand_anchors_d2.py b/qt/hand_anchors_d2.py index e0f6a5b..116b467 100644 --- a/qt/hand_anchors_d2.py +++ b/qt/hand_anchors_d2.py @@ -88,6 +88,12 @@ def read_minutes( only), and PR-E (amp anomaly) resamples the FULL day to 5min FIRST and PIT-filters the DERIVED bars by their own available_time. """ + # Clip to the evaluation plane's start: the cache holds BACKFILLED bars + # from before 2021-07-01, but the frozen panels were computed on + # store.read_range(cfg.data.start, ...) — reading earlier months would + # hand the check a richer history than the engine ever saw (the first + # G run's early-window failures were exactly this). + lo = max(lo, WINDOW_LO) months = pd.period_range(lo, hi, freq="M") parts = [] base = ( From b3d21724065625664bdcf60b7df8dddf4a4d25f3 Mon Sep 17 00:00:00 2001 From: shaofl <2899218482@qq.com> Date: Fri, 24 Jul 2026 08:31:22 -0700 Subject: [PATCH 8/8] fix(qt): disclose a searched-but-not-found guard_boundary anchor class A missing guard_boundary row in hand_anchors_d2.json was previously indistinguishable from a forgotten stratification class (review D2, LOW). The scan now records an explicit guard_boundary_skipped row with the scan size, so the JSON is self-describing. Current run artifacts predate this fix; the D5 re-run will carry the disclosure. --- qt/hand_anchor_rows.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/qt/hand_anchor_rows.py b/qt/hand_anchor_rows.py index 5d209c0..90cac08 100644 --- a/qt/hand_anchor_rows.py +++ b/qt/hand_anchor_rows.py @@ -333,6 +333,21 @@ def compare(factor_id, cls, key, hand, note=""): compare(factor_id, "guard_boundary", found, H.hand_value_ratio(found[1], found[0], field), note=f"{field}<=0 -> NaN") + else: + # Disclose "searched but not found" so the JSON is self-describing: + # a missing guard_boundary row must be distinguishable from a + # forgotten stratification class (review D2, LOW). + results.append( + { + "factor_id": factor_id, "class": "guard_boundary_skipped", + "date": "", "symbol": "", "hand": None, "engine": None, + "rel_diff": None, "ok": True, + "note": (f"scanned {SCAN_SYMBOLS} random symbols; no " + f"{field}<=0 row found — class searched, not omitted"), + } + ) + print(f"SKIP {factor_id:28s} guard_boundary: no {field}<=0 row " + f"in {SCAN_SYMBOLS} scanned symbols (disclosed)") first = series.xs(_symbols(series)[0], level="symbol").index[0] k0 = (first, _symbols(series)[0]) compare(factor_id, "first_row", k0, H.hand_value_ratio(k0[1], k0[0], field),