diff --git a/qt/config.py b/qt/config.py index 454908c..2ee17e9 100644 --- a/qt/config.py +++ b/qt/config.py @@ -318,6 +318,15 @@ class IntradayCfg(_Strict): session_open: str = "09:30:00" execution_model: str = "next_minute_close" execution_window: tuple[str, str] = ("14:51:00", "14:56:59") + # WHICH bar fills is ``execution_model``; WHAT price it fills at is this. + # Default ``bar_vwap`` = that bar's amount/volume, the volume-weighted mean of + # every trade in the execution minute — the honest proxy for an order worked + # across the minute, and not movable by a single small print. ``bar_close`` + # (that minute's last tick) is kept only to reproduce the pre-VWAP ledger + # bit-for-bit. Both are RAW unadjusted prices. A bar whose basis price is + # undefined (NaN close; non-positive/non-finite volume or amount) is an + # explicit block, never a fallback to another price. + execution_price_basis: Literal["bar_vwap", "bar_close"] = "bar_vwap" require_cache_coverage: bool = True missing_execution: Literal["block"] = "block" # I5c: which PIT-safe daily intraday feature the tail-rebalance score uses. diff --git a/qt/intraday_group_report.py b/qt/intraday_group_report.py index 21d1dd1..14bd6a3 100644 --- a/qt/intraday_group_report.py +++ b/qt/intraday_group_report.py @@ -91,6 +91,9 @@ def _intro_lines(result) -> list[str]: f"- decision_time `{cfg.intraday.decision_time}` / execution_window " f"`[{cfg.intraday.execution_window[0]}, {cfg.intraday.execution_window[1]}]`; " "returns are execution-to-execution, NEVER close-to-close", + f"- execution_price_basis: `{cfg.intraday.execution_price_basis}` " + "(`bar_vwap` = the selected 1min bar's amount/volume, RAW unadjusted; " + "`bar_close` = that bar's single closing tick)", f"- trading cost: `cost.fee_rate={cfg.cost.fee_rate}`, " f"`slippage_rate={cfg.cost.slippage_rate}` (cost line = turnover × fee_rate " "inside SimExecution; no extra ad-hoc cost layer)", diff --git a/qt/intraday_tail_framework.py b/qt/intraday_tail_framework.py index fe2f304..eee0863 100644 --- a/qt/intraday_tail_framework.py +++ b/qt/intraday_tail_framework.py @@ -158,6 +158,7 @@ def _exec_cfg_from(cfg: RootConfig) -> IntradayExecutionConfig: data_lag=ic.data_lag, execution_model=ic.execution_model, execution_window=tuple(ic.execution_window), + execution_price_basis=ic.execution_price_basis, ) @@ -735,6 +736,11 @@ def _write_report(result: I5aResult, *, elapsed: float) -> None: lines.append( f"- execution_window: `[{ec.execution_window[0]}, {ec.execution_window[1]}]`" ) + lines.append( + f"- execution_price_basis: `{ec.execution_price_basis}` " + "(`bar_vwap` = the selected 1min bar's amount/volume, RAW unadjusted; " + "`bar_close` = that bar's single closing tick)" + ) lines.append( f"- score feature: `{result.score_feature}` " f"(key=`{result.score_feature_key}`)" @@ -742,7 +748,8 @@ def _write_report(result: I5aResult, *, elapsed: float) -> None: lines.append("") lines.append("**Returns are execution-to-execution, NOT close-to-close.** Holding " "return of a period = exec_price(next) / exec_price(this) - 1, priced " - "at the first valid 1min close in the execution window.") + "at the first valid 1min bar in the execution window on the " + f"`{ec.execution_price_basis}` basis.") lines.append("") _append_factor_section(lines, result) lines.append("## Minute-cache coverage & data provenance") diff --git a/runtime/backtest/event_models.py b/runtime/backtest/event_models.py index a381e59..86b634b 100644 --- a/runtime/backtest/event_models.py +++ b/runtime/backtest/event_models.py @@ -13,7 +13,12 @@ Both schedules come from :func:`runtime.backtest.events.monthly_anchor_pairs`, so daily and intraday rebalance on the same calendar; only the time basis differs. -The intraday model never prices off the daily panel. + +The intraday model never takes a PRICE from the daily panel: decisions and fills +read minute bars only, so no EOD price can leak into a 14:50 decision. It does +read one non-price field from that panel — ``adj_factor`` — to divide out +corporate actions when measuring a holding return, since the cached minute bars +are raw. See :meth:`IntradayTailEventModel.holding_returns`. """ from __future__ import annotations @@ -110,9 +115,11 @@ class IntradayTailEventModel: The rebalance schedule is the same monthly calendar as the daily model, but each period's decision is timestamped at ``decision_time`` and its execution - at the start of the execution window. Per-symbol entry/exit prices are the - earliest valid 1min close in the execution window (``next_minute_close``); - holding returns are ``exec_price(exit) / exec_price(entry) - 1``. + at the start of the execution window. Per-symbol entry/exit prices come from + the earliest valid 1min bar in the execution window (``next_minute_close``), + priced on the config's ``execution_price_basis`` (default that bar's VWAP = + ``amount / volume``); holding returns are + ``exec_price(exit) / exec_price(entry) - 1``. Base (I5a) feasibility rule: a symbol can be traded at the rebalance ONLY if it has a valid (non-NaN) execution bar at the entry anchor; a missing/NaN @@ -120,12 +127,57 @@ class IntradayTailEventModel: is blocked by this rule. I5b adds OPT-IN execution-time price-limit feasibility (``price_limit_check``): - on top of the bar-exists rule, the selected execution-minute RAW close is - compared to that symbol/date's raw ``stk_limit`` band — a buy is blocked at the - upper limit, a sell is blocked at the lower limit, directionally. The - comparison is RAW-vs-RAW only: the raw 1min close (the intraday cache stores - unadjusted bars) against raw ``stk_limit``; it never reads a qfq / daily close - or a daily-close-derived limit flag. A missing/NaN limit row never silently + on top of the bar-exists rule, the RAW price the trade actually pays — the + ``execution_price_basis``, a bar VWAP by default — is compared to that + symbol/date's raw ``stk_limit`` band. A buy is blocked at the upper limit, a + sell at the lower limit, directionally. + + Why the gate reads the VWAP and not the bar close. A limit-up execution minute + has exactly two shapes, and they are the feasibility question: + + * LOCKED (封死涨停) — every trade in the minute prints at the limit, so the + VWAP equals the limit price up to rounding. There is no seller except at + the limit and the queue is hopeless: the buy must be blocked. + * OPENED (盘中打开) — some trades printed below the limit, so the VWAP sits + below it. Those prints are direct evidence that a fill WAS achievable: + the buy must go through. + + The bar close cannot make that distinction and misclassifies both edges — a + minute that closed at the limit but opened during it gets over-blocked, and a + minute that closed below but was locked most of the way gets under-blocked. + The VWAP encodes exactly "was there volume available below the limit". + + Calibration follows from that, and it is the part that is easy to get wrong: + ``limit_tolerance`` must stay at ROUNDING scale and must NOT be widened to + "catch near-limit bars". Widening it re-blocks fills that demonstrably were + achievable, which is the opposite of the point. Measured on real cached 14:51 + bars joined to raw ``stk_limit`` (13,699 bars; 149 closing at the up limit): + the 146 LOCKED ones sit a median 0.000000% and at most 0.0021% below the + limit — pure ``amount`` rounding — while the 3 OPENED ones sit 0.013%-0.017% + below it. The populations separate by an order of magnitude. At the default + ``limit_tolerance=1e-6`` every OPENED minute is correctly allowed and 2.1% of + LOCKED minutes are misclassified as opened by sub-tick rounding; widening to + 0.01 RMB would instead misclassify 100% of OPENED minutes as locked. + + That residual errs OPTIMISTIC and is disclosed rather than tuned away: a + locked minute misread as opened lets through a buy that was NOT achievable, + so the backtest can book a fill the market would have refused. Its size is + bounded — limit-closing bars are 149/13,699 ≈ 1.1% of all execution bars, and + 2.1% of those are misread, so ≈0.02% of bars overall. The threshold is + deliberately NOT recalibrated to remove it: re-fitting a cutoff on a + 149-observation sample is the post-hoc tuning this project refuses elsewhere, + and it is not worth an exception for a 0.02% effect. If that bound is ever + revisited, it needs a fresh, larger sample — not this one. + + An "opened but with tiny volume below the limit" bar is a CAPACITY question, + not a feasibility one: it belongs to the I5f capacity layer, which sizes the + trade against the execution minute's traded ``amount``. This gate answers only + whether a fill was possible at all. + + The comparison is RAW-vs-RAW only: the raw 1min execution price (the intraday + cache stores unadjusted bars, and a bar VWAP = raw amount / raw volume is raw + too) against raw ``stk_limit``; it never reads a qfq / daily close or a + daily-close-derived limit flag. A missing/NaN limit row never silently counts as a passed check: in strict mode (``require_price_limit_coverage``) a missing REQUIRED row fails at construction (before any result); in lenient mode it is counted/disclosed and the name falls back to the bar-exists rule (the @@ -174,6 +226,13 @@ def __init__( fills = [] self._exec_prices = prices self._fills = fills + # Cumulative adj_factor at each ANCHOR date, for the return computation + # only (see holding_returns). Sourced from the daily panel, which carries + # adj_factor as a required core column; restricted to anchors so the map + # stays small. NOT a price: reading it leaks no EOD price into the 14:50 + # decision or the fill, both of which remain minute-only. + self._adj = self._index_adj_factors(calendar_panel, anchor_dates) + self._missing_adj: set[tuple[pd.Timestamp, str]] = set() # -- I5b price-limit feasibility state ----------------------------- # self._price_limit_check = bool(price_limit_check) @@ -185,6 +244,13 @@ def __init__( self._limits: dict[tuple[pd.Timestamp, str], tuple[float, float]] = ( self._index_limits(price_limits) if self._price_limit_check else {} ) + # The RAW CLOSE of each selected execution bar, keyed by (date, symbol). + # NOT the gate's input (the gate compares the executed price); used only to + # label OPENED limit minutes for diagnostics. Carried on the fills, so it + # is available on the ``precomputed_prices`` path too. + self._limit_refs: dict[tuple[pd.Timestamp, str], float] = ( + self._index_limit_refs(self._fills) if self._price_limit_check else {} + ) # entry/rebalance anchor dates (the first element of each pair); the exit # date of the final pair is a pure pricing anchor and never a decision, so # it needs no limit coverage. @@ -198,6 +264,11 @@ def __init__( self._up_blocked_buys: dict[tuple[pd.Timestamp, str], float] = {} self._down_blocked_sells: dict[tuple[pd.Timestamp, str], float] = {} self._unchecked_limits: set[tuple[pd.Timestamp, str]] = set() + # OPENED limit minutes: the bar closed at a limit but traded through it, so + # the VWAP gate lets the fill stand where a close-based gate would block. + # Purely diagnostic — recorded so that divergence is reportable, not silent. + self._opened_limit_ups: dict[tuple[pd.Timestamp, str], float] = {} + self._opened_limit_downs: dict[tuple[pd.Timestamp, str], float] = {} if self._price_limit_check and self._require_limit_coverage: self._assert_limit_coverage() @@ -222,6 +293,63 @@ def _index_limits( out[key] = (float(up), float(down)) return out + @staticmethod + def _index_adj_factors( + panel: pd.DataFrame, anchor_dates: list + ) -> dict[tuple[pd.Timestamp, str], float]: + """``{(anchor date, symbol): cumulative adj_factor}`` from the daily panel. + + Only strictly positive, non-NaN factors are indexed; anything else leaves + the pair absent so the caller BLOCKS that return rather than assuming 1.0. + """ + if "adj_factor" not in panel.columns or panel.empty or not anchor_dates: + return {} + wanted = {pd.Timestamp(d).normalize() for d in anchor_dates} + dates = panel.index.get_level_values("date") + sub = panel.loc[dates.normalize().isin(wanted), "adj_factor"] + out: dict[tuple[pd.Timestamp, str], float] = {} + for (d, s), v in sub.items(): + if pd.isna(v): + continue + f = float(v) + if f > 0.0: + out[(pd.Timestamp(d).normalize(), str(s))] = f + return out + + def _adj_at(self, date: pd.Timestamp, symbol: str) -> float: + """Cumulative adj_factor at an anchor, or NaN when unusable/absent.""" + return self._adj.get( + (pd.Timestamp(date).normalize(), str(symbol)), float("nan") + ) + + @staticmethod + def _index_limit_refs( + fills: list[ExecutionFill], + ) -> dict[tuple[pd.Timestamp, str], float]: + """Index the selected execution bars' RAW closes by (date, symbol). + + A fill with no usable raw close (no bar, or a NaN close) contributes no + entry, so the gate reports that pair as unchecked rather than passing it. + """ + out: dict[tuple[pd.Timestamp, str], float] = {} + for f in fills: + ref = f.limit_reference_price + if ref is None or pd.isna(ref): + continue + out[(pd.Timestamp(f.date).normalize(), str(f.symbol))] = float(ref) + return out + + def _closed_at(self, key: tuple[pd.Timestamp, str], limit: float) -> bool: + """Did the selected bar's RAW CLOSE sit at ``limit``? (diagnostic only.) + + Used solely to label an OPENED limit minute — one that ended at the limit + but traded through it. It never affects ``can_buy``/``can_sell``. + """ + ref = self._limit_refs.get(key) + if ref is None: + return False + return abs(ref - float(limit)) <= self._limit_tol + def _required_limit_pairs(self) -> list[tuple[pd.Timestamp, str]]: """(entry anchor date, symbol) pairs that need a raw limit row. @@ -295,20 +423,47 @@ def _exec_price(self, date: pd.Timestamp, symbol: str) -> float: def holding_returns( self, period: HoldingPeriod, symbols: list[str] ) -> pd.Series: - """exec-to-exec gross return per symbol over (entry, exit]. + """exec-to-exec gross return per symbol over (entry, exit], ADJUSTED. + + The execution prices are RAW minute prices (the intraday cache stores + unadjusted bars), so a ratio of two of them across an ex-dividend or split + date reads the mechanical price drop as a loss. The return is therefore + computed on adjusted prices:: + + (raw_exit * adj_factor(exit)) / (raw_entry * adj_factor(entry)) - 1 + + The per-symbol anchor that ``data/clean/adjust.py`` divides by cancels in + this ratio, so no qfq series needs re-deriving and the result is exact and + window-invariant — the same property that makes ``front_adjust`` safe for + batch == incremental. A symbol missing an execution price at EITHER anchor is omitted (so the - engine's ``settle`` treats it as flat — it earns nothing for the period). - Never substitutes a daily close. + engine's ``settle`` treats it as flat). A symbol whose ``adj_factor`` is + missing, NaN or non-positive at either anchor is ALSO omitted and counted + in :meth:`missing_adj_factor_pairs` — never silently treated as 1.0, which + would reintroduce the very bias this corrects. Never substitutes a daily + close. + + Only the RETURN is adjusted. The I5b price-limit gate stays RAW-vs-RAW + (raw execution price against raw ``stk_limit``); adjustment must not leak + into it, since exchange limits are quoted on raw prices. """ entry, exit_ = period.entry_date, period.exit_date out: dict[str, float] = {} for sym in symbols: a = self._exec_price(entry, sym) b = self._exec_price(exit_, sym) - if pd.notna(a) and pd.notna(b) and a != 0.0: - out[str(sym)] = b / a - 1.0 - # else: blocked at an anchor -> omitted -> flat via settle. + if not (pd.notna(a) and pd.notna(b) and a != 0.0): + continue # blocked at an anchor -> omitted -> flat via settle. + fa = self._adj_at(entry, sym) + fb = self._adj_at(exit_, sym) + if pd.isna(fa) or pd.isna(fb): + # No usable adjustment: the corporate-action-free return cannot be + # formed, so the name earns nothing this period and the gap is + # disclosed. Assuming 1.0 here is exactly the defect being fixed. + self._missing_adj.add((pd.Timestamp(entry).normalize(), str(sym))) + continue + out[str(sym)] = (b * fb) / (a * fa) - 1.0 return pd.Series(out, dtype=float) def feasibility( @@ -319,11 +474,16 @@ def feasibility( Base rule (I5a): tradable in a direction iff a valid execution bar exists at the entry anchor; a missing bar blocks both directions BEFORE any limit logic. I5b layer (when ``price_limit_check``): a buy is additionally blocked - if the selected execution-minute raw close sits at/above the raw upper - limit, and a sell if it sits at/below the raw lower limit (raw-vs-raw, with - ``limit_tolerance`` as the equality band). Limit-up blocks BUY only; - limit-down blocks SELL only — the other direction still executes if the bar - exists. + if the RAW execution price sits at/above the raw upper limit, and a sell if + it sits at/below the raw lower limit (raw-vs-raw, with ``limit_tolerance`` + as the equality band). Limit-up blocks BUY only; limit-down blocks SELL + only — the other direction still executes if the bar exists. + + The gate's input is the price the trade PAYS. A locked minute's VWAP is the + limit price (every print is there); an opened minute's VWAP is below it + precisely because volume traded below the limit, which is what makes the + fill achievable. See the class docstring for the measured separation and + for why ``limit_tolerance`` must stay at rounding scale. """ entry = pd.Timestamp(period.entry_date).normalize() can_buy: dict[str, bool] = {} @@ -339,12 +499,20 @@ def feasibility( band = self._limits.get(key) if band is not None: up, down = band - if float(price) >= up - self._limit_tol: + ref = float(price) + if ref >= up - self._limit_tol: buy_ok = False self._up_blocked_buys[key] = up - if float(price) <= down + self._limit_tol: + elif self._closed_at(key, up): + # Opened limit-up: the minute ENDED at the limit but traded + # below it, so the fill stands. Counted so the divergence + # from a close-based gate is auditable, never silent. + self._opened_limit_ups[key] = up + if ref <= down + self._limit_tol: sell_ok = False self._down_blocked_sells[key] = down + elif self._closed_at(key, down): + self._opened_limit_downs[key] = down else: # No usable raw limit row: in lenient mode we must not pretend # the limit was checked. Record it as unchecked and fall back @@ -384,6 +552,28 @@ def missing_limit_rows(self) -> int: """Count of evaluated (date, symbol) pairs with no usable raw limit row.""" return len(self._unchecked_limits) + def missing_adj_factor_pairs(self) -> int: + """(entry anchor, symbol) pairs whose return was dropped for lack of a factor. + + Both execution prices existed, but ``adj_factor`` was missing / NaN / + non-positive at an anchor, so no corporate-action-free return could be + formed. Disclosed rather than defaulted to 1.0. + """ + return len(self._missing_adj) + + def opened_limit_up_minutes(self) -> int: + """Buys ALLOWED at a bar that closed at the up limit but traded below it. + + The exact set where the VWAP gate diverges from a close-based gate: the + minute ended locked, yet volume printed under the limit, so the fill was + achievable. Reported so the divergence is auditable rather than silent. + """ + return len(self._opened_limit_ups) + + def opened_limit_down_minutes(self) -> int: + """Sells ALLOWED at a bar that closed at the down limit but traded above it.""" + return len(self._opened_limit_downs) + def up_limit_blocked_buy_keys(self) -> set[tuple[pd.Timestamp, str]]: """(rebalance date, symbol) keys whose BUY was blocked by a raw up-limit. diff --git a/runtime/intraday_execution.py b/runtime/intraday_execution.py index 9a48158..8b39ef6 100644 --- a/runtime/intraday_execution.py +++ b/runtime/intraday_execution.py @@ -7,8 +7,11 @@ signal cutoff T 14:50:00 — the decision may use only bars with available_time <= cutoff (I3's job). execution timestamp T 14:51:00 — the trade fills at the NEXT minute bar - (``next_minute_close``), i.e. on data that - is INTENTIONALLY after the signal cutoff. + (``next_minute_close`` selects WHICH bar), + i.e. on data that is INTENTIONALLY after + the signal cutoff. WHAT price that bar + fills at is the separate + ``execution_price_basis`` decision below. holding period exec(T) -> exec(T_next) — return is measured from this rebalance's execution price to the next rebalance's execution price, NEVER @@ -18,8 +21,9 @@ dataclasses turning (target weights per rebalance date, 1min bars, exec config) into execution prices, execution-to-execution holding returns, and an explainable blocked log. It does NOT touch factors/alpha math, the daily backtest, or the -config models; a missing minute bar / NaN price / no bar in the execution window -produces a BLOCKED record — never a silent daily-close fallback. Daily EOD data +config models; a missing minute bar / no bar in the execution window / a bar +whose configured price basis is undefined produces a BLOCKED record — never a +silent fallback to another price (bar close or daily close). Daily EOD data (``daily_basic`` etc.) is by construction not consulted here: only intraday bars are read, so T-day EOD values cannot leak into a 14:50 decision/execution. """ @@ -39,6 +43,21 @@ SUPPORTED_EXECUTION_MODELS: tuple[str, ...] = ("next_minute_close",) _FUTURE_EXECUTION_MODELS: tuple[str, ...] = ("tail_vwap", "closing_call_proxy") +# WHICH bar is selected (``execution_model``) and WHAT price that bar fills at +# (``execution_price_basis``) are separate decisions. +# bar_vwap — the selected bar's ``amount / volume``: the volume-weighted mean +# of EVERY trade printed in that minute, i.e. the honest proxy for +# an order worked across the minute, and hard to move with a single +# small print. This is the DEFAULT. +# bar_close — the selected bar's close: a single tick (that minute's last +# print). Kept only to reproduce the pre-VWAP ledger bit-for-bit. +# Both are RAW unadjusted prices: the intraday cache stores raw bars, and +# ``amount``/``volume`` are raw traded value / shares, so their ratio is raw too. +# No adjustment factor is applied here or downstream of here. +PRICE_BASIS_VWAP = "bar_vwap" +PRICE_BASIS_CLOSE = "bar_close" +SUPPORTED_PRICE_BASES: tuple[str, ...] = (PRICE_BASIS_VWAP, PRICE_BASIS_CLOSE) + # blocked reasons (explainable; never a silent daily-close substitution). REASON_NO_BAR = "no_execution_bar" REASON_MISSING_PRICE = "missing_price" @@ -52,15 +71,22 @@ class IntradayExecutionConfig: feature-availability lag consumed by the I3 cutoff (``available_time = bar_end + data_lag``); it is NOT re-applied to execution pricing. Execution fills at the earliest 1min bar whose ``bar_end`` falls in - ``[execution_window_start, execution_window_end]`` (default 14:51–14:56:59). + ``[execution_window_start, execution_window_end]`` (default 14:51–14:56:59), + at that bar's ``execution_price_basis`` price (default its VWAP). """ decision_time: str = "14:50:00" data_lag: str = "1min" execution_model: str = "next_minute_close" execution_window: tuple[str, str] = ("14:51:00", "14:56:59") + execution_price_basis: str = PRICE_BASIS_VWAP def __post_init__(self) -> None: + if self.execution_price_basis not in SUPPORTED_PRICE_BASES: + raise ValueError( + f"Unsupported execution_price_basis {self.execution_price_basis!r}; " + f"supported: {SUPPORTED_PRICE_BASES}." + ) if self.execution_model not in SUPPORTED_EXECUTION_MODELS: future = ( " (planned, not yet implemented)" @@ -83,7 +109,14 @@ def __post_init__(self) -> None: @dataclass(frozen=True) class ExecutionFill: - """One symbol's execution outcome on one rebalance date (immutable).""" + """One symbol's execution outcome on one rebalance date (immutable). + + ``exec_price`` is what the trade PAYS (the configured price basis) and is what + the I5b price-limit gate compares. ``limit_reference_price`` is the selected + bar's RAW close, carried only so the gate can LABEL an opened limit minute + (closed at the limit, traded through it) for diagnostics — it never decides + feasibility. It is ``None`` when no bar was selected or its close is NaN. + """ symbol: str date: pd.Timestamp @@ -91,6 +124,7 @@ class ExecutionFill: exec_price: float | None blocked: bool reason: str | None = None + limit_reference_price: float | None = None @dataclass(frozen=True) @@ -108,6 +142,37 @@ def blocked(self) -> list[ExecutionFill]: return [f for f in self.fills if f.blocked] +def bar_execution_price(bar: pd.Series, basis: str) -> float | None: + """The RAW fill price of one selected 1min ``bar``, or ``None`` if undefined. + + ``bar_vwap`` (default) returns ``amount / volume`` — the volume-weighted mean + of every trade printed in that minute. It is UNDEFINED when ``volume`` or + ``amount`` is missing/NaN/non-finite or non-positive (a minute with no traded + shares has no traded average price). ``None`` means exactly that: the caller + must BLOCK the fill. It must never be softened into the bar close, and never + into a daily close — a silent price substitution is the one degradation this + execution layer exists to prevent. + + ``bar_close`` returns that bar's close (a single tick) and reproduces the + pre-VWAP behaviour bit-for-bit, including its exact NaN test. + """ + if basis == PRICE_BASIS_CLOSE: + price = bar["close"] + return None if pd.isna(price) else float(price) + if basis != PRICE_BASIS_VWAP: + raise ValueError( + f"Unsupported execution_price_basis {basis!r}; " + f"supported: {SUPPORTED_PRICE_BASES}." + ) + volume = float(bar["volume"]) if pd.notna(bar["volume"]) else float("nan") + amount = float(bar["amount"]) if pd.notna(bar["amount"]) else float("nan") + if not (np.isfinite(volume) and np.isfinite(amount)): + return None + if volume <= 0.0 or amount <= 0.0: + return None + return amount / volume + + def resolve_fill( symbol: str, date: pd.Timestamp, @@ -117,9 +182,11 @@ def resolve_fill( """Resolve the execution fill for ``symbol`` on ``date`` from its 1min bars. ``day_bars`` are the 1min bars for THIS symbol on THIS date (must carry - ``bar_end`` + ``close``). ``next_minute_close`` takes the EARLIEST bar whose - ``bar_end`` is in the execution window; a missing bar or NaN close yields a - BLOCKED fill with an explainable reason (never a daily-close fallback). + ``bar_end`` plus the columns the price basis reads). ``next_minute_close`` + takes the EARLIEST bar whose ``bar_end`` is in the execution window; a missing + bar, or a bar whose ``execution_price_basis`` price is undefined (NaN close / + non-positive or non-finite volume or amount for the VWAP), yields a BLOCKED + fill with an explainable reason — never a fallback to another price. """ day = pd.Timestamp(date).normalize() win_start = day + pd.Timedelta(cfg.execution_window[0]) @@ -134,14 +201,20 @@ def resolve_fill( return ExecutionFill(symbol, day, None, None, True, REASON_NO_BAR) bar = cand.iloc[0] # next_minute_close: earliest available bar in the window - price = bar["close"] - if pd.isna(price): + bar_end = pd.Timestamp(bar["bar_end"]) + # The raw close travels with the fill regardless of the price basis: the + # price-limit gate needs the price that is comparable to a limit price, which + # a VWAP is not (see IntradayTailEventModel). + raw_close = bar_execution_price(bar, PRICE_BASIS_CLOSE) + price = bar_execution_price(bar, cfg.execution_price_basis) + if price is None: + # Same block path/reason as a NaN close has always taken: the bar existed + # (exec_time is kept) but has no usable price, so the name is untradable + # this rebalance. No other price is substituted. return ExecutionFill( - symbol, day, pd.Timestamp(bar["bar_end"]), None, True, REASON_MISSING_PRICE + symbol, day, bar_end, None, True, REASON_MISSING_PRICE, raw_close ) - return ExecutionFill( - symbol, day, pd.Timestamp(bar["bar_end"]), float(price), False, None - ) + return ExecutionFill(symbol, day, bar_end, float(price), False, None, raw_close) def build_execution_prices( diff --git a/tests/test_exec_vwap_basis.py b/tests/test_exec_vwap_basis.py new file mode 100644 index 0000000..6b7d0ee --- /dev/null +++ b/tests/test_exec_vwap_basis.py @@ -0,0 +1,702 @@ +"""Execution price basis: the tail fill is the execution bar's VWAP by default. + +The intraday tail model picks WHICH 1min bar fills (``next_minute_close``: the +earliest bar in ``[14:51, 14:56:59]``). WHAT price that bar fills at is the +separate ``execution_price_basis``, and the default is now ``bar_vwap`` = +``amount / volume``: the volume-weighted mean of every trade printed in that +minute, rather than the single closing tick. + +What is locked here: + +1. defaults / config — ``bar_vwap`` is the default at both the runtime config and + the pydantic ``IntradayCfg``; an unknown basis fails readably; the pydantic + Literal and the runtime supported set cannot drift apart; the runner actually + forwards the setting (otherwise the knob would be dead); +2. arithmetic — hand-computed VWAP cases where ``amount/volume`` differs from the + bar's close AND from its open/high/low, so a basis mix-up cannot pass; +3. undefined VWAP — non-positive / non-finite / missing ``volume`` or ``amount`` + is an EXPLICIT block on the existing missing-price path, never a fallback to + the bar close and never to the daily close (the daily panel here carries a + deliberately contradictory close to prove that); +4. I5b ordering + raw basis — a missing bar still blocks both directions BEFORE + any limit logic; an undefined VWAP blocks before it too; the limit gate + compares the RAW price that actually executes (the VWAP), and the daily close + never enters the comparison; +5. ``bar_close`` reproduces the pre-VWAP behaviour (the I4 locks in + ``test_intraday_execution.py`` assert the original values under that basis). + +All network-free: synthetic bars / panels / limit rows. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import get_args + +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal +from pydantic import ValidationError + +from data.clean.intraday_schema import normalize_intraday_bars +from data.clean.schema import normalize_panel +from portfolio.construct import TopNEqualWeight +from qt.config import IntradayCfg, load_config +from qt.intraday_tail_framework import _exec_cfg_from +from qt.pipeline import _FrameScores +from runtime.backtest.engine import BacktestEngine +from runtime.backtest.event_models import IntradayTailEventModel +from runtime.backtest.sim_execution import SimExecution +from runtime.intraday_execution import ( + PRICE_BASIS_CLOSE, + PRICE_BASIS_VWAP, + REASON_MISSING_PRICE, + REASON_NO_BAR, + SUPPORTED_PRICE_BASES, + IntradayExecutionConfig, + bar_execution_price, + build_execution_prices, + resolve_fill, +) +from universe.static import StaticUniverse + +_CONFIG_DIR = Path(__file__).resolve().parent.parent / "config" +_I5A_CONFIG = _CONFIG_DIR / "phase_i5a_intraday_tail_framework.yaml" + +_DATES = [ + "2024-01-30", "2024-01-31", + "2024-02-28", "2024-02-29", + "2024-03-28", "2024-03-29", +] +_JAN, _FEB, _MAR = ( + pd.Timestamp("2024-01-31"), pd.Timestamp("2024-02-29"), pd.Timestamp("2024-03-29"), +) + +_VWAP = IntradayExecutionConfig() # default +_CLOSE = IntradayExecutionConfig(execution_price_basis=PRICE_BASIS_CLOSE) + + +# --------------------------------------------------------------------------- # +# builders — bar specs are (symbol, timestamp, close, volume, amount) +# --------------------------------------------------------------------------- # +def _bars(specs: list[tuple[str, str, float, float, float]]) -> pd.DataFrame: + rows = [ + { + "time": pd.Timestamp(t), "symbol": s, + # open/high/low are deliberately OFF the close and off the VWAP, so a + # test that accidentally reads any of them is visible. + "open": c - 0.25, "high": c + 0.75, "low": c - 0.75, "close": c, + "volume": v, "amount": a, "source_trade_time": t, + } + for (s, t, c, v, a) in specs + ] + return normalize_intraday_bars(pd.DataFrame(rows), freq="1min", data_lag="1min") + + +def _day_bars(bars: pd.DataFrame, symbol: str, date: pd.Timestamp) -> pd.DataFrame: + work = bars.reset_index() + work["date"] = work["bar_end"].dt.normalize() + return work[(work["symbol"] == symbol) & (work["date"] == pd.Timestamp(date))] + + +def _daily_panel(symbols: list[str], close: float) -> pd.DataFrame: + rows = [ + { + "date": pd.Timestamp(d), "symbol": s, + "open": close, "high": close, "low": close, "close": close, + "volume": 1.0, "amount": 1.0, "adj_factor": 1.0, + } + for d in _DATES for s in symbols + ] + return normalize_panel(pd.DataFrame(rows)) + + +def _scores(values: dict[str, float]) -> _FrameScores: + idx = pd.MultiIndex.from_tuples( + [(pd.Timestamp(d), s) for d in _DATES for s in values], + names=["date", "symbol"], + ) + return _FrameScores( + pd.Series([values[s] for _ in _DATES for s in values], index=idx, name="score") + ) + + +def _limits(rows: list[tuple]) -> pd.DataFrame: + return pd.DataFrame( + [ + {"date": pd.Timestamp(d), "symbol": s, "up_limit": up, "down_limit": dn} + for (d, s, up, dn) in rows + ] + ) + + +def _period(model: IntradayTailEventModel, date: pd.Timestamp): + return next(p for p in model.holding_periods() if p.date == date) + + +def _fill(bars: pd.DataFrame, symbol: str, date: pd.Timestamp, cfg): + return resolve_fill(symbol, date, _day_bars(bars, symbol, date), cfg) + + +# --------------------------------------------------------------------------- # +# 1. defaults, config validation, wiring +# --------------------------------------------------------------------------- # +def test_default_basis_is_bar_vwap_at_runtime_and_in_config(): + assert IntradayExecutionConfig().execution_price_basis == PRICE_BASIS_VWAP + assert IntradayCfg().execution_price_basis == PRICE_BASIS_VWAP + + +def test_unknown_basis_fails_readably(): + with pytest.raises(ValueError, match="execution_price_basis"): + IntradayExecutionConfig(execution_price_basis="daily_close") + with pytest.raises(ValidationError, match="execution_price_basis"): + IntradayCfg(execution_price_basis="daily_close") + # the pure pricing function refuses an unknown basis rather than guessing. + bar = pd.Series({"close": 10.0, "volume": 100.0, "amount": 1200.0}) + with pytest.raises(ValueError, match="execution_price_basis"): + bar_execution_price(bar, "daily_close") + + +def test_config_literal_cannot_drift_from_runtime_supported_set(): + literal = get_args(IntradayCfg.model_fields["execution_price_basis"].annotation) + assert set(literal) == set(SUPPORTED_PRICE_BASES) + + +def test_all_shipped_configs_still_validate(): + for path in sorted(_CONFIG_DIR.glob("*.yaml")): + load_config(str(path)) # raises on failure + + +def test_runner_forwards_the_price_basis(): + """The knob must reach ``IntradayExecutionConfig`` — otherwise it is dead.""" + cfg = load_config(str(_I5A_CONFIG)) + assert _exec_cfg_from(cfg).execution_price_basis == PRICE_BASIS_VWAP + pinned = cfg.model_copy( + update={ + "intraday": cfg.intraday.model_copy( + update={"execution_price_basis": PRICE_BASIS_CLOSE} + ) + } + ) + assert _exec_cfg_from(pinned).execution_price_basis == PRICE_BASIS_CLOSE + + +# --------------------------------------------------------------------------- # +# 2. VWAP arithmetic — hand-computed, and distinguishable from every other price +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + "close, volume, amount, expected_vwap", + [ + # 1_200 RMB traded over 100 shares -> 12.0, while the last tick was 11.0. + (11.0, 100.0, 1_200.0, 12.0), + # 3e6 RMB over 200k shares -> 15.0, while the last tick was 20.0. + (20.0, 200_000.0, 3_000_000.0, 15.0), + ], +) +def test_fill_price_is_bar_amount_over_volume(close, volume, amount, expected_vwap): + bars = _bars([("A", f"{_JAN.date()} 14:51:00", close, volume, amount)]) + fill = _fill(bars, "A", _JAN, _VWAP) + + assert not fill.blocked + assert fill.exec_time == pd.Timestamp(f"{_JAN.date()} 14:51:00") + assert fill.exec_price == pytest.approx(expected_vwap) + # ... and it is NOT the close, nor any other price printed on that bar, so the + # assertion has real discriminating power. + bar = _day_bars(bars, "A", _JAN).iloc[0] + for column in ("close", "open", "high", "low"): + assert fill.exec_price != pytest.approx(float(bar[column])) + + +def test_bar_close_basis_prices_the_same_bar_at_its_close(): + """Same bar, other basis: proves the switch actually changes the fill.""" + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 11.0, 100.0, 1_200.0)]) + assert _fill(bars, "A", _JAN, _CLOSE).exec_price == pytest.approx(11.0) + assert _fill(bars, "A", _JAN, _VWAP).exec_price == pytest.approx(12.0) + + +def test_vwap_equals_close_only_when_amount_is_close_times_volume(): + """Why the I5a/I5b/I5d fixtures are basis-neutral — by construction, not luck.""" + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 13.5, 400.0, 13.5 * 400.0)]) + assert _fill(bars, "A", _JAN, _VWAP).exec_price == pytest.approx(13.5) + assert _fill(bars, "A", _JAN, _CLOSE).exec_price == pytest.approx(13.5) + + +def test_window_selection_is_unchanged_by_the_basis(): + """The basis prices the bar; it never re-picks which bar executes.""" + bars = _bars([ + ("A", f"{_JAN.date()} 14:53:00", 11.0, 100.0, 1_200.0), # earliest in window + ("A", f"{_JAN.date()} 14:55:00", 50.0, 100.0, 9_900.0), # later: ignored + ]) + for cfg in (_VWAP, _CLOSE): + assert _fill(bars, "A", _JAN, cfg).exec_time == pd.Timestamp( + f"{_JAN.date()} 14:53:00" + ) + assert _fill(bars, "A", _JAN, _VWAP).exec_price == pytest.approx(12.0) + # a bar outside the window is still no bar at all. + outside = _bars([("A", f"{_JAN.date()} 14:00:00", 11.0, 100.0, 1_200.0)]) + blocked = _fill(outside, "A", _JAN, _VWAP) + assert blocked.blocked and blocked.reason == REASON_NO_BAR + + +# --------------------------------------------------------------------------- # +# 3. undefined VWAP -> explicit block, never a fallback price +# --------------------------------------------------------------------------- # +_UNDEFINED = [ + pytest.param(0.0, 1_200.0, id="zero-volume"), + pytest.param(-100.0, 1_200.0, id="negative-volume"), + pytest.param(float("nan"), 1_200.0, id="nan-volume"), + pytest.param(float("inf"), 1_200.0, id="inf-volume"), + pytest.param(100.0, 0.0, id="zero-amount"), + pytest.param(100.0, -5.0, id="negative-amount"), + pytest.param(100.0, float("nan"), id="nan-amount"), + pytest.param(100.0, float("inf"), id="inf-amount"), +] + + +@pytest.mark.parametrize("volume, amount", _UNDEFINED) +def test_undefined_vwap_is_blocked_and_never_falls_back_to_the_bar_close( + volume, amount +): + close = 11.0 + bars = _bars([("A", f"{_JAN.date()} 14:51:00", close, volume, amount)]) + + fill = _fill(bars, "A", _JAN, _VWAP) + assert fill.blocked + # joins the EXISTING missing-price block path: same reason, and the bar's + # timestamp is kept because the bar did exist. + assert fill.reason == REASON_MISSING_PRICE + assert fill.exec_time == pd.Timestamp(f"{_JAN.date()} 14:51:00") + assert fill.exec_price is None + # The close is a perfectly usable number on this very bar — a fallback would + # have produced exactly it. It must not appear. + assert _fill(bars, "A", _JAN, _CLOSE).exec_price == pytest.approx(close) + assert fill.exec_price != close + # the pricing primitive says "undefined", not "zero". + bar = _day_bars(bars, "A", _JAN).iloc[0] + assert bar_execution_price(bar, PRICE_BASIS_VWAP) is None + + +@pytest.mark.parametrize("volume, amount", _UNDEFINED) +def test_undefined_vwap_leaves_nan_in_the_price_matrix(volume, amount): + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 11.0, volume, amount)]) + prices, fills = build_execution_prices(bars, [_JAN], ["A"], _VWAP) + assert pd.isna(prices.loc[_JAN, "A"]) + assert not (prices == 11.0).any().any() # the close never leaks into the matrix + assert [f.reason for f in fills] == [REASON_MISSING_PRICE] + + +def test_undefined_vwap_blocks_both_directions_and_earns_nothing(): + # A's execution minute has no traded shares; B is normal. The daily panel + # carries a contradictory close (999.0) that must never be substituted. + panel = _daily_panel(["A", "B"], close=999.0) + bars = _bars( + [(s, f"{d.date()} 14:51:00", 11.0, 0.0 if s == "A" else 100.0, 1_200.0) + for d in (_JAN, _FEB, _MAR) for s in ("A", "B")] + ) + model = IntradayTailEventModel(calendar_panel=panel, bars=bars, cfg=_VWAP) + + period = _period(model, _JAN) + can_buy, can_sell = model.feasibility(period, ["A", "B"]) + assert can_buy["A"] is False and can_sell["A"] is False + assert can_buy["B"] is True and can_sell["B"] is True + # A is omitted from holding returns (settle treats it as flat) rather than + # earning a close-priced return; B prices at its VWAP (12.0, flat over time). + returns = model.holding_returns(period, ["A", "B"]) + assert "A" not in returns.index + assert returns["B"] == pytest.approx(0.0) + # neither the minute close (11.0) nor the daily close (999.0) is anywhere in + # the execution price matrix. + prices = model.execution_prices() + assert pd.isna(prices.loc[_JAN, "A"]) + assert not prices.isin([11.0, 999.0]).any().any() + assert prices["B"].tolist() == pytest.approx([12.0, 12.0, 12.0]) + assert {f.reason for f in model.blocked_fills()} == {REASON_MISSING_PRICE} + + +# --------------------------------------------------------------------------- # +# 4. I5b price-limit gate — ordering unchanged, RAW-vs-RAW on the executed price +# --------------------------------------------------------------------------- # +def _gate(bars, lim, cfg=_VWAP, **kwargs): + """Build the gate on the DEFAULT limit_tolerance unless a test overrides it. + + Tests here deliberately do NOT pin the tolerance: widening the default must + show up as a failure here (see the mutation evidence in the report). + """ + panel = _daily_panel(["A"], close=50.0) + model = IntradayTailEventModel( + calendar_panel=panel, bars=bars, cfg=cfg, price_limits=lim, + price_limit_check=True, require_price_limit_coverage=True, **kwargs, + ) + return model, model.feasibility(_period(model, _JAN), ["A"]) + + +# A limit-up execution minute has exactly two shapes and the gate must tell them +# apart. BOTH are tested: a one-sided test cannot distinguish a correct gate from +# one that simply blocks everything near a limit. +def test_locked_limit_up_minute_blocks_the_buy(): + """封死涨停: every print at the limit -> VWAP == limit -> no fill was available.""" + volume = 1_000.0 + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 11.0, volume, 11.0 * volume)]) + lim = _limits([(_JAN, "A", 11.0, 5.0)]) + + model, (can_buy, can_sell) = _gate(bars, lim) + assert model.execution_prices().loc[_JAN, "A"] == pytest.approx(11.0) + assert can_buy["A"] is False # blocked ... + assert can_sell["A"] is True # ... and only the buy + assert model.up_limit_blocked_buys() == 1 # existing I5b diagnostic + assert model.opened_limit_up_minutes() == 0 + + +def test_opened_limit_up_minute_allows_the_buy(): + """盘中打开: real prints below the limit -> a fill WAS achievable -> no block. + + Reproduced from real cached data — 601858.SH 2023-05-15 14:51: low 45.57, + close == up_limit 46.13, amount/volume = 46.048583. The minute ended at the + limit but traded through it on the way, so the buy must go through. This is a + DELIBERATE divergence from a close-based gate, which would over-block it. + """ + volume = 1_000_000.0 + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 46.13, volume, 46.048583 * volume)]) + lim = _limits([(_JAN, "A", 46.13, 37.75)]) + + model, (can_buy, can_sell) = _gate(bars, lim) + assert model.execution_prices().loc[_JAN, "A"] == pytest.approx(46.048583) + assert can_buy["A"] is True # NOT blocked — volume traded below + assert can_sell["A"] is True + assert model.up_limit_blocked_buys() == 0 + # the divergence from a close-based gate is counted, never silent + assert model.opened_limit_up_minutes() == 1 + # and it hinges on a gap far above rounding scale (0.0814 RMB = 0.18%) + assert 46.13 - 46.048583 > 0.08 + + +def test_locked_limit_down_minute_blocks_the_sell(): + """Mirror: every print at the lower limit -> VWAP == limit -> no bid to hit.""" + volume = 1_000.0 + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 5.0, volume, 5.0 * volume)]) + lim = _limits([(_JAN, "A", 10.0, 5.0)]) + model, (can_buy, can_sell) = _gate(bars, lim) + assert can_sell["A"] is False and can_buy["A"] is True + assert model.down_limit_blocked_sells() == 1 + assert model.opened_limit_down_minutes() == 0 + + +def test_opened_limit_down_minute_allows_the_sell(): + """Mirror: prints above the lower limit -> a sell was achievable -> no block.""" + volume = 1_000.0 + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 5.0, volume, 5.02 * volume)]) + lim = _limits([(_JAN, "A", 10.0, 5.0)]) + model, (can_buy, can_sell) = _gate(bars, lim) + assert model.execution_prices().loc[_JAN, "A"] == pytest.approx(5.02) + assert can_sell["A"] is True and can_buy["A"] is True + assert model.down_limit_blocked_sells() == 0 + assert model.opened_limit_down_minutes() == 1 + + +def test_tolerance_stays_at_rounding_scale_not_a_near_limit_band(): + """The calibration rule, as an executable fact. + + Real cached 14:51 bars closing at the up limit: the LOCKED ones sit at most + 0.0021% below the limit (`amount` rounding), the OPENED ones 0.013%-0.017% + below — an order of magnitude apart. A rounding-scale band separates them; a + "near-limit" band does not, it just re-blocks achievable fills. + """ + volume = 3_000.0 + # locked, but `amount` rounded to the yuan -> vwap = 10.999666... (0.003% off) + locked = _bars([("A", f"{_JAN.date()} 14:51:00", 11.0, volume, 32_999.0)]) + # opened: genuine prints below the limit -> vwap 10.94 (0.5% off) + opened = _bars([("A", f"{_JAN.date()} 14:51:00", 11.0, volume, 10.94 * volume)]) + lim = _limits([(_JAN, "A", 11.0, 5.0)]) + + # A rounding-scale band blocks the locked minute ... + assert _gate(locked, lim, limit_tolerance=0.001)[1][0]["A"] is False + # ... and still lets the opened one through. + assert _gate(opened, lim, limit_tolerance=0.001)[1][0]["A"] is True + # Widening to a "near limit" band destroys the distinction: the opened minute, + # where volume demonstrably traded below the limit, gets blocked too. + assert _gate(opened, lim, limit_tolerance=0.1)[1][0]["A"] is False + +def test_missing_bar_still_blocks_before_any_limit_logic(): + panel = _daily_panel(["A"], close=50.0) + bars = _bars([("A", f"{_FEB.date()} 14:51:00", 9.0, 100.0, 1_000.0)]) # no Jan bar + lim = _limits([(_FEB, "A", 100.0, 1.0)]) + model = IntradayTailEventModel( + calendar_panel=panel, bars=bars, cfg=_VWAP, price_limits=lim, + price_limit_check=True, require_price_limit_coverage=True, + ) + can_buy, can_sell = model.feasibility(_period(model, _JAN), ["A"]) + assert can_buy["A"] is False and can_sell["A"] is False + assert model.up_limit_blocked_buys() == 0 # the gate never ran + assert model.down_limit_blocked_sells() == 0 + + +def test_undefined_vwap_blocks_before_the_limit_gate_and_needs_no_limit_row(): + """An unpriceable bar cannot reach the gate, so it requires no coverage. + + Strict coverage therefore must NOT demand a limit row for it (that would be a + spurious hard failure), and its block must stay a price block, not a limit one. + """ + panel = _daily_panel(["A"], close=50.0) + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 9.0, 0.0, 1_000.0)]) # vwap undefined + model = IntradayTailEventModel( + calendar_panel=panel, bars=bars, cfg=_VWAP, price_limits=None, + price_limit_check=True, require_price_limit_coverage=True, # no raise + ) + assert model.limit_coverage() == {"required": 0, "present": 0, "missing": 0} + can_buy, can_sell = model.feasibility(_period(model, _JAN), ["A"]) + assert can_buy["A"] is False and can_sell["A"] is False + assert model.up_limit_blocked_buys() == 0 + assert model.missing_limit_rows() == 0 # never even consulted + + +def test_vwap_basis_diverges_from_close_basis_only_where_intended(): + """Enumerate BOTH directions of the behaviour change — nothing silent. + + Versus a close-based gate the VWAP gate is MORE permissive on opened limit + minutes (by design: volume traded below the limit, so the fill was + achievable) and STRICTER on unpriceable bars (undefined VWAP blocks both + ways). Every other shape agrees. There is deliberately no one-sided + "never more permissive" invariant here — that would be false, and asserting + it would hide the very change the user asked for. + """ + syms = ["LOCKED", "OPENED", "NORMAL", "NOVOL"] + panel = _daily_panel(syms, close=50.0) + specs = [ + ("LOCKED", 11.0, 1_000.0, 11_000.0), # every print at the limit + ("OPENED", 11.0, 1_000.0, 10_940.0), # closed at the limit, traded below + ("NORMAL", 9.0, 1_000.0, 9_400.0), # nowhere near a limit + ("NOVOL", 9.0, 0.0, 0.0), # unpriceable on the VWAP basis + ] + bars = _bars([(s, f"{_JAN.date()} 14:51:00", c, v, a) for (s, c, v, a) in specs]) + lim = _limits([(_JAN, s, 11.0, 5.0) for s in syms]) + + def _buys(cfg): + model = IntradayTailEventModel( + calendar_panel=panel, bars=bars, cfg=cfg, price_limits=lim, + price_limit_check=True, require_price_limit_coverage=False, + ) + return model.feasibility(_period(model, _JAN), syms)[0] + + vwap, close = _buys(_VWAP), _buys(_CLOSE) + assert (close["OPENED"], vwap["OPENED"]) == (False, True) # more permissive + assert (close["NOVOL"], vwap["NOVOL"]) == (True, False) # stricter + assert vwap["LOCKED"] is False and close["LOCKED"] is False # agree: blocked + assert vwap["NORMAL"] is True and close["NORMAL"] is True # agree: allowed + + +def test_limit_gate_ignores_the_daily_close_under_vwap(): + """Perturbing the daily close cannot move an intraday RAW-vs-RAW decision.""" + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 9.0, 100.0, 1_000.0)]) # vwap 10.0 + lim = _limits([(_JAN, "A", 10.0, 5.0)]) + outcomes = [] + for daily_close in (50.0, 5_000.0): + model = IntradayTailEventModel( + calendar_panel=_daily_panel(["A"], close=daily_close), bars=bars, + cfg=_VWAP, price_limits=lim, price_limit_check=True, + require_price_limit_coverage=True, + ) + outcomes.append(model.feasibility(_period(model, _JAN), ["A"])) + assert model.execution_prices().loc[_JAN, "A"] == pytest.approx(10.0) + assert outcomes[0] == outcomes[1] + + +# --------------------------------------------------------------------------- # +# 5. engine level — the basis moves the book, and only the basis +# --------------------------------------------------------------------------- # +def _engine(panel, bars, cfg): + return BacktestEngine( + model=IntradayTailEventModel(calendar_panel=panel, bars=bars, cfg=cfg), + universe=StaticUniverse(["A", "B"]), + scores=_scores({"A": 2.0, "B": 1.0}), + constructor=TopNEqualWeight(1, long_only=True), + execution=SimExecution(fee_rate=0.001), + selection_panel=panel, + initial_nav=1.0, + cash_return=0.0, + ) + + +def test_engine_book_follows_the_vwap_not_the_close(): + panel = _daily_panel(["A", "B"], close=100.0) + # A's closes are flat at 10.0 while its VWAP doubles from 8.0 to 16.0: the two + # bases cannot be confused in the settled book. + bars = _bars([ + ("A", f"{_JAN.date()} 14:51:00", 10.0, 100.0, 800.0), # vwap 8.0 + ("A", f"{_FEB.date()} 14:51:00", 10.0, 100.0, 1_600.0), # vwap 16.0 + ("A", f"{_MAR.date()} 14:51:00", 10.0, 100.0, 1_600.0), # vwap 16.0 + ("B", f"{_JAN.date()} 14:51:00", 20.0, 100.0, 2_000.0), + ("B", f"{_FEB.date()} 14:51:00", 20.0, 100.0, 2_000.0), + ("B", f"{_MAR.date()} 14:51:00", 20.0, 100.0, 2_000.0), + ]) + vwap_model = IntradayTailEventModel(calendar_panel=panel, bars=bars, cfg=_VWAP) + close_model = IntradayTailEventModel(calendar_panel=panel, bars=bars, cfg=_CLOSE) + jan = _period(vwap_model, _JAN) + assert vwap_model.holding_returns(jan, ["A"])["A"] == pytest.approx(1.0) # 16/8-1 + assert close_model.holding_returns(jan, ["A"])["A"] == pytest.approx(0.0) # 10/10-1 + + vwap_nav = _engine(panel, bars, _VWAP).run() + close_nav = _engine(panel, bars, _CLOSE).run() + assert not vwap_nav["nav"].equals(close_nav["nav"]) + assert float(vwap_nav["nav"].iloc[-1]) > float(close_nav["nav"].iloc[-1]) + + +def test_bases_agree_exactly_when_amount_equals_close_times_volume(): + """The equivalence that keeps the I5a/I5b/I5d fixtures basis-neutral.""" + panel = _daily_panel(["A", "B"], close=100.0) + specs = [ + ("A", _JAN, 10.0), ("A", _FEB, 12.0), ("A", _MAR, 9.0), + ("B", _JAN, 20.0), ("B", _FEB, 21.0), ("B", _MAR, 19.0), + ] + bars = _bars( + [(s, f"{d.date()} 14:51:00", c, 250.0, c * 250.0) for (s, d, c) in specs] + ) + vwap_engine, close_engine = _engine(panel, bars, _VWAP), _engine(panel, bars, _CLOSE) + assert_frame_equal(vwap_engine.run(), close_engine.run()) + assert_frame_equal(vwap_engine.holdings_log(), close_engine.holdings_log()) + assert_frame_equal(vwap_engine.feasibility_log(), close_engine.feasibility_log()) + + +def test_undefined_vwap_does_not_silently_trade_at_the_close_in_the_engine(): + """A whole rebalance's names unpriceable -> no position, not a close-priced one.""" + panel = _daily_panel(["A", "B"], close=100.0) + bars = _bars([ + # A's January execution minute printed no shares; February/March are fine. + ("A", f"{_JAN.date()} 14:51:00", 10.0, 0.0, 0.0), + ("A", f"{_FEB.date()} 14:51:00", 10.0, 100.0, 1_600.0), + ("A", f"{_MAR.date()} 14:51:00", 10.0, 100.0, 1_600.0), + ("B", f"{_JAN.date()} 14:51:00", 20.0, 100.0, 2_000.0), + ("B", f"{_FEB.date()} 14:51:00", 20.0, 100.0, 2_000.0), + ("B", f"{_MAR.date()} 14:51:00", 20.0, 100.0, 2_000.0), + ]) + def _jan_weight_of_a(cfg) -> float: + engine = _engine(panel, bars, cfg) + nav = engine.run() + assert np.isfinite(nav["nav"]).all() + holdings = engine.holdings_log() + jan = holdings[holdings["date"] == _JAN].set_index("symbol")["weight"] + return float(jan.get("A", 0.0)) + + # A is the top-scored name on both bases. On bar_close its January bar prices + # fine (10.0), so it IS bought — which is exactly the fill a fallback would + # have manufactured. On bar_vwap the same bar is unpriceable, so A is simply + # not bought: the block is real, not a re-priced trade. + assert _jan_weight_of_a(_CLOSE) == pytest.approx(1.0) + assert _jan_weight_of_a(_VWAP) == pytest.approx(0.0) + +# --------------------------------------------------------------------------- # +# 6. holding returns are ADJUSTED (raw minute prices span corporate actions) +# --------------------------------------------------------------------------- # +def _ex_date_panel(factors: dict) -> pd.DataFrame: + """Daily panel whose adj_factor per (date, symbol) comes from ``factors``.""" + rows = [ + { + "date": pd.Timestamp(d), "symbol": s, + "open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0, + "volume": 1.0, "amount": 1.0, + "adj_factor": factors.get((pd.Timestamp(d), s), 1.0), + } + for d in _DATES for s in ("A", "B") + ] + return normalize_panel(pd.DataFrame(rows)) + + +def test_holding_return_divides_out_an_ex_date_price_drop(): + """A 2-for-1 split between anchors must read ~0, not -50%. + + Raw minute price halves overnight while the cumulative adj_factor doubles. + The unadjusted ratio would book a mechanical -50% "loss"; the adjusted one is + flat, which is the true holding return. + """ + factors = {(_JAN, "A"): 1.0, (_FEB, "A"): 2.0, (_MAR, "A"): 2.0} + panel = _ex_date_panel(factors) + bars = _bars([ + # A: raw 40.00 -> 20.00 across the ex-date (a pure 2:1 split) + ("A", f"{_JAN.date()} 14:51:00", 40.0, 1_000.0, 40_000.0), + ("A", f"{_FEB.date()} 14:51:00", 20.0, 1_000.0, 20_000.0), + ("A", f"{_MAR.date()} 14:51:00", 20.0, 1_000.0, 20_000.0), + # B: no corporate action, flat price -> flat return (control) + ("B", f"{_JAN.date()} 14:51:00", 30.0, 1_000.0, 30_000.0), + ("B", f"{_FEB.date()} 14:51:00", 30.0, 1_000.0, 30_000.0), + ("B", f"{_MAR.date()} 14:51:00", 30.0, 1_000.0, 30_000.0), + ]) + model = IntradayTailEventModel(calendar_panel=panel, bars=bars, cfg=_VWAP) + jan = _period(model, _JAN) + returns = model.holding_returns(jan, ["A", "B"]) + + # (20 * 2) / (40 * 1) - 1 == 0 : the split is divided out. + assert returns["A"] == pytest.approx(0.0) + # the unadjusted computation this replaces would have booked -50% + raw_ratio = 20.0 / 40.0 - 1.0 + assert raw_ratio == pytest.approx(-0.5) + assert returns["A"] != pytest.approx(raw_ratio) + assert returns["B"] == pytest.approx(0.0) # control unaffected + assert model.missing_adj_factor_pairs() == 0 + # the RAW execution prices themselves are untouched (fills still pay raw) + assert model.execution_prices().loc[_JAN, "A"] == pytest.approx(40.0) + assert model.execution_prices().loc[_FEB, "A"] == pytest.approx(20.0) + + +def test_holding_return_adjusts_a_partial_dividend_not_just_a_split(): + """A ~2% cash dividend: raw reads -2%, adjusted reads ~0.""" + factors = {(_JAN, "A"): 1.0, (_FEB, "A"): 1.02, (_MAR, "A"): 1.02} + panel = _ex_date_panel(factors) + bars = _bars([ + ("A", f"{_JAN.date()} 14:51:00", 51.0, 1_000.0, 51_000.0), + ("A", f"{_FEB.date()} 14:51:00", 50.0, 1_000.0, 50_000.0), + ("A", f"{_MAR.date()} 14:51:00", 50.0, 1_000.0, 50_000.0), + ("B", f"{_JAN.date()} 14:51:00", 30.0, 1_000.0, 30_000.0), + ("B", f"{_FEB.date()} 14:51:00", 30.0, 1_000.0, 30_000.0), + ("B", f"{_MAR.date()} 14:51:00", 30.0, 1_000.0, 30_000.0), + ]) + model = IntradayTailEventModel(calendar_panel=panel, bars=bars, cfg=_VWAP) + got = model.holding_returns(_period(model, _JAN), ["A"])["A"] + assert got == pytest.approx((50.0 * 1.02) / (51.0 * 1.0) - 1.0) + assert got == pytest.approx(0.0, abs=1e-9) + assert got != pytest.approx(50.0 / 51.0 - 1.0) # the raw (biased) value + + +@pytest.mark.parametrize("bad", [None, float("nan"), 0.0, -1.0]) +def test_missing_adj_factor_is_dropped_and_disclosed_never_treated_as_one(bad): + """No usable factor -> the return is dropped and counted, never assumed 1.0.""" + factors = {(_JAN, "A"): 1.0, (_FEB, "A"): 2.0, (_MAR, "A"): 2.0} + if bad is None: + factors.pop((_FEB, "A")) # row present but factor absent -> 1.0 default + panel = _ex_date_panel(factors) # ... so drop the row entirely instead + panel = panel.drop(index=(_FEB, "A")) + else: + factors[(_FEB, "A")] = bad + panel = _ex_date_panel(factors) + bars = _bars([ + ("A", f"{_JAN.date()} 14:51:00", 40.0, 1_000.0, 40_000.0), + ("A", f"{_FEB.date()} 14:51:00", 20.0, 1_000.0, 20_000.0), + ("B", f"{_JAN.date()} 14:51:00", 30.0, 1_000.0, 30_000.0), + ("B", f"{_FEB.date()} 14:51:00", 30.0, 1_000.0, 30_000.0), + ]) + model = IntradayTailEventModel(calendar_panel=panel, bars=bars, cfg=_VWAP) + returns = model.holding_returns(_period(model, _JAN), ["A", "B"]) + + assert "A" not in returns.index # dropped, not booked at -50% + assert model.missing_adj_factor_pairs() == 1 # ... and disclosed + assert returns["B"] == pytest.approx(0.0) # unaffected names still price + + +def test_adjustment_does_not_leak_into_the_raw_price_limit_gate(): + """Exchange limits are quoted on RAW prices — the gate must stay raw-vs-raw. + + The symbol carries adj_factor 2.0, so an adjusted price would be 22.0 and + would clear an 11.0 limit. The gate must still compare the raw 11.0 VWAP and + block the buy. + """ + factors = {(_JAN, "A"): 2.0, (_FEB, "A"): 2.0, (_MAR, "A"): 2.0} + panel = _ex_date_panel(factors) + bars = _bars([("A", f"{_JAN.date()} 14:51:00", 11.0, 1_000.0, 11_000.0)]) + lim = _limits([(_JAN, "A", 11.0, 5.0)]) + model = IntradayTailEventModel( + calendar_panel=panel, bars=bars, cfg=_VWAP, price_limits=lim, + price_limit_check=True, require_price_limit_coverage=True, + ) + can_buy, _ = model.feasibility(_period(model, _JAN), ["A"]) + assert model.execution_prices().loc[_JAN, "A"] == pytest.approx(11.0) # raw + assert can_buy["A"] is False # blocked on the RAW comparison + assert model.up_limit_blocked_buys() == 1 diff --git a/tests/test_intraday_execution.py b/tests/test_intraday_execution.py index b46f74f..dce2b72 100644 --- a/tests/test_intraday_execution.py +++ b/tests/test_intraday_execution.py @@ -1,4 +1,11 @@ -"""Intraday tail-rebalance execution tests (I4): cutoff vs exec, exec-to-exec.""" +"""Intraday tail-rebalance execution tests (I4): cutoff vs exec, exec-to-exec. + +These are the PRE-VWAP regression locks: every case here pins +``execution_price_basis="bar_close"`` and must keep asserting exactly the values +it asserted before the VWAP basis landed (the ``_norm`` fixture deliberately +gives ``amount/volume != close``, so a basis mix-up cannot pass silently). The +default ``bar_vwap`` basis is covered in ``test_exec_vwap_basis.py``. +""" from __future__ import annotations @@ -19,6 +26,11 @@ ) +# The pre-VWAP price basis, pinned explicitly so these locks keep asserting the +# original values no matter what the default becomes. +_CLOSE_BASIS = IntradayExecutionConfig(execution_price_basis="bar_close") + + def _norm(rows, data_lag="1min"): """rows = [(time_str, symbol, close), ...] -> normalized 1min bars.""" cl = [r[2] for r in rows] @@ -70,7 +82,7 @@ def test_signal_cutoff_and_execution_timestamp_are_separate(): bars = _norm(rows) fill = resolve_fill( "000001.SZ", pd.Timestamp("2024-01-02"), - _day(bars, "000001.SZ"), IntradayExecutionConfig(), + _day(bars, "000001.SZ"), _CLOSE_BASIS, ) # execution INTENTIONALLY uses the 14:51 bar, which is AFTER the 14:50 cutoff assert not fill.blocked @@ -98,7 +110,7 @@ def test_next_minute_close_uses_1451_bar(): ] fill = resolve_fill( "000001.SZ", pd.Timestamp("2024-01-02"), - _day(_norm(rows), "000001.SZ"), IntradayExecutionConfig(), + _day(_norm(rows), "000001.SZ"), _CLOSE_BASIS, ) assert fill.exec_time == pd.Timestamp("2024-01-02 14:51:00") assert fill.exec_price == 10.0 @@ -111,7 +123,7 @@ def test_missing_1451_uses_first_bar_in_window(): ] fill = resolve_fill( "000001.SZ", pd.Timestamp("2024-01-02"), - _day(_norm(rows), "000001.SZ"), IntradayExecutionConfig(), + _day(_norm(rows), "000001.SZ"), _CLOSE_BASIS, ) assert fill.exec_time == pd.Timestamp("2024-01-02 14:53:00") # first in window assert fill.exec_price == 11.0 @@ -121,7 +133,7 @@ def test_no_bar_in_window_is_blocked(): rows = [("2024-01-02 14:00:00", "000001.SZ", 10.0)] # nothing in 14:51-14:56:59 fill = resolve_fill( "000001.SZ", pd.Timestamp("2024-01-02"), - _day(_norm(rows), "000001.SZ"), IntradayExecutionConfig(), + _day(_norm(rows), "000001.SZ"), _CLOSE_BASIS, ) assert fill.blocked assert fill.reason == REASON_NO_BAR @@ -132,7 +144,7 @@ def test_nan_price_is_blocked_missing_price(): rows = [("2024-01-02 14:51:00", "000001.SZ", np.nan)] fill = resolve_fill( "000001.SZ", pd.Timestamp("2024-01-02"), - _day(_norm(rows), "000001.SZ"), IntradayExecutionConfig(), + _day(_norm(rows), "000001.SZ"), _CLOSE_BASIS, ) assert fill.blocked assert fill.reason == REASON_MISSING_PRICE @@ -156,7 +168,7 @@ def test_holding_return_is_execution_to_execution_not_close(): pd.Timestamp("2024-01-02"): pd.Series({"000001.SZ": 1.0}), pd.Timestamp("2024-01-03"): pd.Series({"000001.SZ": 1.0}), } - res = simulate_tail_rebalance(weights, bars) + res = simulate_tail_rebalance(weights, bars, _CLOSE_BASIS) entry = pd.Timestamp("2024-01-02") exec_to_exec = 12.0 / 10.0 - 1.0 # 0.20 close_to_close = 11.0 / 10.5 - 1.0 # ~0.0476 @@ -184,7 +196,7 @@ def test_blocked_symbol_excluded_and_logged(): pd.Timestamp("2024-01-02"): pd.Series({"000001.SZ": 0.5, "000002.SZ": 0.5}), pd.Timestamp("2024-01-03"): pd.Series({"000001.SZ": 1.0}), } - res = simulate_tail_rebalance(weights, bars) + res = simulate_tail_rebalance(weights, bars, _CLOSE_BASIS) entry = pd.Timestamp("2024-01-02") # only A contributes: 0.5 * 0.10 ; B excluded (its weight earns nothing) assert res.period_returns[entry] == pytest.approx(0.5 * 0.10) @@ -200,7 +212,7 @@ def test_build_execution_prices_matrix_and_log(): bars = _norm(rows) prices, fills = build_execution_prices( bars, [pd.Timestamp("2024-01-02")], ["000001.SZ", "000002.SZ"], - IntradayExecutionConfig(), + _CLOSE_BASIS, ) assert prices.loc[pd.Timestamp("2024-01-02"), "000001.SZ"] == 10.0 assert pd.isna(prices.loc[pd.Timestamp("2024-01-02"), "000002.SZ"])