From 00a0ca168be4dbf315bf9fa91d899d9c0d1eff7f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 20 Jul 2026 08:57:01 -0400 Subject: [PATCH] Blocking Table 1.4 dollar-row fit + QRF tail-concentration gates; export-guard both CGD route legs (#462) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit populace#462: the live Build M default shipped non_sch_d_capital_gains at $74.6B against its own $10.2B SOI Pub 1304 Table 1.4 target (+634.8%, recorded in the release's calibration_diagnostics.json) — a puf_tax_detail weighted-QRF donor-tail point mass ($594,484 repeated ceiling value; top 100 records = 89% of weighted mass) — while schedule_d_capital_gain_distributions was absent from the export and guarded by nothing. Fix 1 (gates): target_fit_gate + TargetFitRequirement in populace.build.gates make target-row CLASSES within-tolerance-blocking by name pattern (conjunctive selectors), so rows the exact-name critical registers never enumerated cannot certify. Wired twice: the release tool's _release_gate_failures and the populace-data release contract both require every national Table 1.4 dollar row within |25%| (the established broad-fit bound), with no incumbent-improvement escape. On the live Build M surface this fails exactly the two defect rows (capital_gain_distributions_amount +634.8%, net_capital_gains_amount -25.6%) and passes the other nine dollar rows (worst passer -10.9%). Fix 1b (gates): tail_concentration_gate blocks a sparse imputed dollar column whose top-100 weighted records carry >75% of its weighted |mass| (Build M signature: 89% across 2,295 carriers). The release tool derives the checked set from the source_stages.json fit_weighted_qrf stage outputs, runs the gate on the calibrated export frame in the terminal batched gate group, writes qrf_tail_concentration.json alongside the other gate artifacts, and adds --allow-qrf-tail-concentration as the diagnostic escape hatch. Fix 2 (export contract, the #361 remedy class): both #282 CGD route legs are export-guarded — schedule_d_capital_gain_distributions joins the release input coverage manifest as a post-reference hard requirement (red on today's artifacts by design, the #368 SSI-asset pattern), and US_CGD_ROUTE_REQUIRED_INPUTS pins both legs against demotion to reviewed exclusions in assert_release_input_coverage_manifest_current. The QRF imputation fix itself (zero-inflation / donor-support guard) remains Build N #449 scope. Relates #462, #361, #457, #449. Co-Authored-By: Claude Fable 5 --- .../src/populace/build/__init__.py | 6 + .../src/populace/build/gates.py | 441 ++++++++++++++++ .../us/release_input_coverage_manifest.json | 10 +- .../src/populace/build/us_runtime/__init__.py | 2 + .../us_runtime/release_input_coverage.py | 38 +- packages/populace-build/tests/test_gates.py | 487 ++++++++++++++++++ .../tests/test_release_input_coverage.py | 54 ++ .../tests/test_us_fiscal_refresh_builder.py | 203 ++++++++ .../src/populace/data/contract.py | 45 ++ packages/populace-data/tests/test_contract.py | 125 ++++- packages/populace-data/tests/test_release.py | 17 +- tools/build_us_fiscal_refresh_release.py | 203 +++++++- ...uild_us_release_input_coverage_manifest.py | 30 +- 13 files changed, 1644 insertions(+), 17 deletions(-) diff --git a/packages/populace-build/src/populace/build/__init__.py b/packages/populace-build/src/populace/build/__init__.py index 12803967..603a70d2 100644 --- a/packages/populace-build/src/populace/build/__init__.py +++ b/packages/populace-build/src/populace/build/__init__.py @@ -64,6 +64,7 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: GateReport, GateResult, TargetCoverageRequirement, + TargetFitRequirement, aggregate_admin_gate, default_valued_columns_gate, enum_domain_gate, @@ -81,6 +82,8 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: source_coverage_gate, source_stage_input_coverage_gate, support_gate, + tail_concentration_gate, + target_fit_gate, target_profile_coverage_gate, target_surface_gate, weights_audit_gate, @@ -146,6 +149,7 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "STAGING_SCHEMA_VERSION", "StagingTelemetry", "TargetCoverageRequirement", + "TargetFitRequirement", "LedgerConsumerArtifact", "LedgerTargetMapping", "LedgerTargetSelection", @@ -172,6 +176,8 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "select_ledger_targets_from_jsonl", "summarize_rotations", "support_gate", + "tail_concentration_gate", + "target_fit_gate", "target_profile_coverage_gate", "target_spec_from_ledger_fact", "target_surface_gate", diff --git a/packages/populace-build/src/populace/build/gates.py b/packages/populace-build/src/populace/build/gates.py index 23aa1931..4297066f 100644 --- a/packages/populace-build/src/populace/build/gates.py +++ b/packages/populace-build/src/populace/build/gates.py @@ -67,8 +67,11 @@ "per_family_fit_gate", "source_coverage_gate", "source_stage_input_coverage_gate", + "tail_concentration_gate", + "target_fit_gate", "target_profile_coverage_gate", "TargetCoverageRequirement", + "TargetFitRequirement", "relative_error_loss", "target_surface_gate", "weights_audit_gate", @@ -2018,3 +2021,441 @@ def weights_audit_gate( "unused_allowed_unweighted": unused_allowed, }, ) + + +@dataclass(frozen=True) +class TargetFitRequirement: + """One within-tolerance-blocking requirement over calibrated target rows. + + The exact-name critical-target registers (the release tool's + ``US_CRITICAL_TARGET_FIT_REQUIREMENTS``, the release contract's + ``_US_CRITICAL_TARGET_FIT_REQUIREMENTS``) can only block rows someone + enumerated in advance. populace#462 is what slips through: the live Build M + release shipped ``irs_soi.ty2023.table_1_4.all.capital_gain_distributions_ + amount@2024`` at +634.8% relative error — recorded in its own diagnostics — + because no register named that row. A fit requirement names a row *class* + (e.g. every national SOI Pub 1304 Table 1.4 dollar row) so new rows in the + class are blocking the day the feed adds them. + + Selector semantics differ deliberately from + :class:`TargetCoverageRequirement`: selectors here are **conjunctive across + fields** and disjunctive within a field. A row matches when, for every + non-empty selector field, it satisfies at least one entry of that field. + Coverage requirements ask "is at least one of these concepts present?" + (any-of); fit requirements ask "does every row shaped like this fit?" — + which needs name-shape ANDs (prefix ``irs_soi.`` AND substring + ``.table_1_4.`` AND suffix ``_amount@2024``). + + Attributes: + requirement_id: Stable id for the requirement (manifest key). + label: Human-readable class label used in failure lines. + max_abs_relative_error: The blocking tolerance: every matched row's + ``|relative_error|`` must be at or below this. Must be finite and + positive. + accepted_names: Exact row names that match. + accepted_name_prefixes: Row-name prefixes (``str.startswith``). + accepted_name_substrings: Row-name substrings (case-sensitive). + accepted_name_suffixes: Row-name suffixes (``str.endswith``). + min_matches: Minimum rows the surface must contain for the requirement + (default 1). A pattern that matches nothing is vacuous — a renamed + table or a dropped feed family must fail loudly, not gate nothing. + notes: Free-text rationale recorded alongside the requirement. + """ + + requirement_id: str + label: str + max_abs_relative_error: float + accepted_names: tuple[str, ...] = () + accepted_name_prefixes: tuple[str, ...] = () + accepted_name_substrings: tuple[str, ...] = () + accepted_name_suffixes: tuple[str, ...] = () + min_matches: int = 1 + notes: str = "" + + def __post_init__(self) -> None: + if not self.requirement_id: + raise ValueError("TargetFitRequirement.requirement_id is required.") + if not self.label: + raise ValueError( + f"TargetFitRequirement {self.requirement_id!r}: label is required." + ) + max_abs = float(self.max_abs_relative_error) + if not np.isfinite(max_abs) or max_abs <= 0.0: + raise ValueError( + f"TargetFitRequirement {self.requirement_id!r}: " + "max_abs_relative_error must be finite and positive, got " + f"{self.max_abs_relative_error!r}." + ) + if self.min_matches < 1: + raise ValueError( + f"TargetFitRequirement {self.requirement_id!r}: " + "min_matches must be at least 1." + ) + if not ( + self.accepted_names + or self.accepted_name_prefixes + or self.accepted_name_substrings + or self.accepted_name_suffixes + ): + raise ValueError( + f"TargetFitRequirement {self.requirement_id!r}: at least one " + "accepted selector is required." + ) + + +def _matches_target_fit_requirement( + name: str, requirement: TargetFitRequirement +) -> bool: + if requirement.accepted_names and name not in requirement.accepted_names: + return False + if requirement.accepted_name_prefixes and not any( + name.startswith(prefix) for prefix in requirement.accepted_name_prefixes + ): + return False + if requirement.accepted_name_substrings and not any( + substring in name for substring in requirement.accepted_name_substrings + ): + return False + if requirement.accepted_name_suffixes and not any( + name.endswith(suffix) for suffix in requirement.accepted_name_suffixes + ): + return False + return True + + +def _finite_number(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + value = float(value) + if not np.isfinite(value): + return None + return value + + +def target_fit_gate( + target_rows: Iterable[object], + requirements: Iterable[TargetFitRequirement], + *, + reviewed_exclusions: Mapping[str, str] | None = None, + name: str = "target_fit", +) -> GateResult: + """Every target row in a named class fits within its blocking tolerance. + + The populace#462 catcher, generalized: the Build M live release recorded a + +634.8% relative error on a national SOI Pub 1304 Table 1.4 dollar target + (and −25.6% on the table's net-capital-gains line) inside its own + ``calibration_diagnostics.json`` and still certified, because dollar-target + tolerance blocking existed only for individually enumerated rows. This gate + takes the calibration's own per-target rows and a set of + :class:`TargetFitRequirement` row classes, recomputes each matched row's + relative error from ``target`` and ``final_estimate`` (never trusting a + recorded ``relative_error`` it can't reproduce), and fails the release when + any matched row's ``|relative_error|`` exceeds its class tolerance. + + Args: + target_rows: Per-target diagnostics rows. Each row may be a mapping or + an attribute object (both the calibrate ``TargetDiagnostic`` shape + and the published ``calibration_diagnostics.json`` row shape work) + carrying ``name``, ``target``, and ``final_estimate``, plus an + optional recorded ``relative_error`` that is cross-checked for + staleness when present. + requirements: The row classes to block on. + reviewed_exclusions: Row name -> REASON for rows allowed to breach + their class tolerance (an adjudicated, tracked defect). An entry + for a row that no requirement matches, or for a row now within + tolerance everywhere it matches, is rot and fails the gate. + name: Gate name for the manifest (defaults to ``"target_fit"``). + + Returns: + Pass iff every requirement matches at least ``min_matches`` rows and + every matched, non-excluded row is within its class tolerance, with no + stale or dangling exclusions. + + Raises: + ValueError: If a reviewed exclusion has an empty reason. + TypeError: If ``reviewed_exclusions`` is not a mapping. + """ + exclusions = _reviewed_exclusion_reasons(reviewed_exclusions) + requirements = tuple(requirements) + + rows: dict[str, object] = {} + failures: list[str] = [] + n_rows = 0 + for row in target_rows: + n_rows += 1 + row_name = str(_coverage_field(row, "name", "") or "") + if not row_name: + continue + if row_name in rows: + failures.append( + f"{row_name}: duplicate target row in the fit surface; " + "two rows under one name would let one mask the other's fit." + ) + continue + rows[row_name] = row + + matches_by_requirement: dict[str, list[str]] = {} + matched_names: set[str] = set() + for requirement in requirements: + matches = sorted( + row_name + for row_name in rows + if _matches_target_fit_requirement(row_name, requirement) + ) + matches_by_requirement[requirement.requirement_id] = matches + matched_names.update(matches) + if len(matches) < requirement.min_matches: + failures.append( + f"{requirement.requirement_id}: target surface has " + f"{len(matches)} match(es), needs {requirement.min_matches} " + f"for {requirement.label} — a fit requirement that matches " + "nothing gates nothing." + ) + + # Row-level auditability, computed once per matched row: the relative + # error is recomputed from target and final_estimate (the same convention + # both critical-target registers use: a zero target compares the absolute + # difference), and a recorded relative_error that disagrees is stale. + relative_errors: dict[str, float] = {} + for row_name in sorted(matched_names): + row = rows[row_name] + target_value = _finite_number(_coverage_field(row, "target", None)) + final_estimate = _finite_number(_coverage_field(row, "final_estimate", None)) + if target_value is None or final_estimate is None: + raw_target = _coverage_field(row, "target", None) + raw_final = _coverage_field(row, "final_estimate", None) + kind = ( + "non-finite" + if isinstance(raw_target, int | float) + and isinstance(raw_final, int | float) + and not isinstance(raw_target, bool) + and not isinstance(raw_final, bool) + else "non-numeric" + ) + failures.append( + f"{row_name}: {kind} target/final_estimate " + f"(target={raw_target!r}, final_estimate={raw_final!r}); " + "an unauditable row cannot certify." + ) + continue + if target_value == 0.0: + computed = final_estimate - target_value + else: + computed = (final_estimate - target_value) / target_value + relative_errors[row_name] = float(computed) + recorded = _coverage_field(row, "relative_error", None) + if recorded is not None: + recorded_number = _finite_number(recorded) + if recorded_number is None: + failures.append( + f"{row_name}: non-numeric recorded relative_error {recorded!r}." + ) + elif not np.isclose(recorded_number, computed, rtol=1e-9, atol=1e-9): + failures.append( + f"{row_name}: stale relative_error {recorded_number!r}; " + f"computed {computed:.6g} from target and final_estimate." + ) + + breached_exclusions: dict[str, str] = {} + for requirement in requirements: + max_abs = float(requirement.max_abs_relative_error) + for row_name in matches_by_requirement[requirement.requirement_id]: + if row_name not in relative_errors: + continue + error = relative_errors[row_name] + if abs(error) <= max_abs: + continue + if row_name in exclusions: + breached_exclusions[row_name] = exclusions[row_name] + continue + row = rows[row_name] + failures.append( + f"{row_name}: relative_error={error:.6g} exceeds " + f"{max_abs:.6g} for {requirement.label} " + f"({requirement.requirement_id}); " + f"target={_coverage_field(row, 'target', None)!r}, " + f"final_estimate={_coverage_field(row, 'final_estimate', None)!r}." + ) + + unmatched_exclusions = sorted(set(exclusions) - matched_names) + if unmatched_exclusions: + failures.append( + f"Reviewed exclusions not matched by any fit requirement: " + f"{unmatched_exclusions}." + ) + stale_exclusions = sorted( + row_name + for row_name in exclusions + if row_name in matched_names and row_name not in breached_exclusions + ) + if stale_exclusions: + failures.append( + f"Stale reviewed exclusions — the row fits its tolerance now, " + f"remove the exclusion: {stale_exclusions}." + ) + + worst = sorted( + relative_errors, + key=lambda row_name: abs(relative_errors[row_name]), + reverse=True, + ) + return GateResult( + name=name, + passed=not failures, + failures=tuple(failures), + details={ + "rows_checked": n_rows, + "requirements_checked": len(requirements), + "matches_by_requirement": matches_by_requirement, + "relative_errors": relative_errors, + "worst_offenders": worst[:10], + "reviewed_exclusions": breached_exclusions, + "stale_exclusions": stale_exclusions, + "unmatched_exclusions": unmatched_exclusions, + }, + ) + + +def tail_concentration_gate( + column_values: Mapping[str, Iterable[float]], + column_weights: Mapping[str, Iterable[float]], + *, + top_k: int = 100, + max_top_share: float = 0.75, + min_nonzero_records: int = 500, + reviewed_exclusions: Mapping[str, str] | None = None, +) -> GateResult: + """No sparse imputed dollar column hides its mass in a handful of records. + + The populace#462 signature: the Build M ``puf_tax_detail`` weighted-QRF + stage broadcast a donor-tail point mass (a repeated $594,484 ceiling + value) onto ``non_sch_d_capital_gains``, so 100 of the 2,295 weighted + carrier records carried 89% of the shipped $74.6B — 7.3x the SOI dollar + target — while the paired returns-count target was hit exactly. Support + clipping cannot catch this (every draw is inside the donor's realized + range) and count targets cannot either (the number of carriers is + correct); the tell is weighted-mass concentration. This gate computes, for + each column, the share of total weighted absolute mass (``|value| * + weight``) carried by the ``top_k`` records and fails when that share + exceeds ``max_top_share``. + + Calibrated to the incident: with the defaults, the Build M column fails at + 89% >> 75%, while every plausibly heavy-tailed sparse dollar column keeps + top-100-of-500+ carriers well below the threshold; a genuinely + concentrated column belongs behind a reviewed exclusion, not a silent + pass. + + Args: + column_values: Column -> per-record values (a sparse QRF-imputed + dollar variable's export column). Signed columns are checked on + absolute mass. + column_weights: Column -> per-record weights, aligned with the + column's values. Every checked column must have weights — an + unweighted concentration check would miss weight-driven mass. + top_k: How many top weighted records form the tail (default 100, the + #462 measurement). + max_top_share: Blocking share threshold in (0, 1) (default 0.75). + min_nonzero_records: Columns with fewer weighted carriers (records + with non-zero mass) are reported as thin and not checked — with + ``top_k`` a material fraction of the carriers, a high share is + arithmetic, not evidence. Must exceed ``top_k``. + reviewed_exclusions: Column -> REASON for columns allowed to stay + concentrated (a documented, tracked defect or a genuinely + concentrated instrument). A column now below the threshold is a + stale entry and fails; an entry for a column absent from this + surface is dormant and only reported. + + Returns: + Pass iff every checked, non-excluded column's top-``top_k`` weighted + mass share is at or below ``max_top_share`` and no exclusion is stale. + + Raises: + ValueError: On invalid thresholds, a checked column with no or + misaligned weights, or an exclusion without a reason. + TypeError: If ``reviewed_exclusions`` is not a mapping. + """ + if top_k < 1: + raise ValueError(f"top_k must be at least 1, got {top_k!r}.") + if not (0.0 < max_top_share < 1.0): + raise ValueError(f"max_top_share must be in (0, 1), got {max_top_share!r}.") + if min_nonzero_records <= top_k: + raise ValueError( + "min_nonzero_records must exceed top_k so the tail is a strict " + f"subset of the carriers, got min_nonzero_records=" + f"{min_nonzero_records!r} with top_k={top_k!r}." + ) + exclusions = _reviewed_exclusion_reasons(reviewed_exclusions) + + failures: list[str] = [] + top_share: dict[str, float] = {} + carrier_counts: dict[str, int] = {} + thin_columns: dict[str, int] = {} + used_exclusions: dict[str, str] = {} + stale_exclusions: list[str] = [] + + for column in sorted(column_values): + if column not in column_weights: + raise ValueError( + f"{column}: no weights declared for the tail-concentration " + "check; concentration without record weights is unauditable." + ) + values = np.asarray(column_values[column], dtype=np.float64).reshape(-1) + weights = np.asarray(column_weights[column], dtype=np.float64).reshape(-1) + if values.shape[0] != weights.shape[0]: + raise ValueError( + f"{column}: values and weights must align, got " + f"{values.shape[0]} values vs {weights.shape[0]} weights." + ) + finite = np.isfinite(values) & np.isfinite(weights) + mass = np.abs(values[finite]) * weights[finite] + mass = mass[mass > 0.0] + carriers = int(mass.size) + if carriers < min_nonzero_records: + thin_columns[column] = carriers + continue + total = float(mass.sum()) + tail = float(np.partition(mass, -top_k)[-top_k:].sum()) + share = tail / total + top_share[column] = float(share) + carrier_counts[column] = carriers + if column in exclusions: + if share > max_top_share: + used_exclusions[column] = exclusions[column] + else: + stale_exclusions.append(column) + continue + if share > max_top_share: + failures.append( + f"{column}: top {top_k} weighted records carry {share:.1%} of " + f"the weighted |mass| across {carriers} carriers (threshold " + f"{max_top_share:.0%}) — a donor-tail point mass broadcast by " + "a sparse imputation (the populace#462 " + "non_sch_d_capital_gains signature); fix the imputation or " + "record a reviewed exclusion." + ) + + if stale_exclusions: + failures.append( + f"Stale reviewed exclusions — the column is below the " + f"concentration threshold now, remove the exclusion: " + f"{sorted(stale_exclusions)}." + ) + dormant_exclusions = sorted(set(exclusions) - set(column_values)) + + return GateResult( + name="tail_concentration", + passed=not failures, + failures=tuple(failures), + details={ + "columns_checked": len(top_share), + "top_k": int(top_k), + "max_top_share": float(max_top_share), + "min_nonzero_records": int(min_nonzero_records), + "top_share": top_share, + "carrier_counts": carrier_counts, + "thin_columns": thin_columns, + "reviewed_exclusions": used_exclusions, + "stale_exclusions": sorted(stale_exclusions), + "dormant_exclusions": dormant_exclusions, + }, + ) diff --git a/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json b/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json index 824ba9e6..6d7594d8 100644 --- a/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json +++ b/packages/populace-build/src/populace/build/us/release_input_coverage_manifest.json @@ -333,6 +333,10 @@ "salt_refund_income": { "status": "required" }, + "schedule_d_capital_gain_distributions": { + "note": "Schedule D line 13 route leg of the #282 capital-gain-distributions split (memo component of long_term_capital_gains, written by the capital_gain_distributions source stage); required with NO reviewed exclusion per PolicyEngine/populace#462 so a release whose export drops the route (the Build M live default shipped it at $0 while non_sch_d_capital_gains carried 7.3x its SOI target) fails the coverage gate. Currently absent — this is the intended red gate until the Build N rebuild carries the split through.", + "status": "required" + }, "selected_marketplace_plan_benchmark_ratio": { "status": "required" }, @@ -519,11 +523,11 @@ } }, "counts": { - "required": 158, + "required": 159, "reviewed_exclusion": 8, - "total": 166 + "total": 167 }, - "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, and meets_ssi_disability_criteria required by shipped validation probes. status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/populace#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", + "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, and the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/populace#462). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/populace#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", "description": "Declared full-coverage contract for a US release: every input column the reference eCPS exports must be persisted as a key with non-default signal, or carry a reviewed exclusion. Enforced as a hard release gate (populace.build.us_runtime.release_input_coverage) that generalizes assert_required_us_release_source_columns from 5 columns to the full eCPS input surface.", "issue": "PolicyEngine/populace#368", "reference": { diff --git a/packages/populace-build/src/populace/build/us_runtime/__init__.py b/packages/populace-build/src/populace/build/us_runtime/__init__.py index 40abb4b1..8c7222d3 100644 --- a/packages/populace-build/src/populace/build/us_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/us_runtime/__init__.py @@ -561,6 +561,7 @@ from populace.build.us_runtime.release_input_coverage import ( POST_REFERENCE_ECPS_REQUIRED_INPUTS, SSI_COUNTABLE_RESOURCE_ASSETS, + US_CGD_ROUTE_REQUIRED_INPUTS, US_RELEASE_INPUT_COVERAGE_RESOURCE, ReformCoverageProbe, ReleaseInputColumn, @@ -1644,6 +1645,7 @@ "assert_validation_leaf_registry_current", "SSI_COUNTABLE_RESOURCE_ASSETS", "POST_REFERENCE_ECPS_REQUIRED_INPUTS", + "US_CGD_ROUTE_REQUIRED_INPUTS", "US_RELEASE_INPUT_COVERAGE_RESOURCE", "ReformCoverageProbe", "ReleaseInputColumn", diff --git a/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py b/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py index 06d3b9b5..af58e25d 100644 --- a/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py +++ b/packages/populace-build/src/populace/build/us_runtime/release_input_coverage.py @@ -106,6 +106,7 @@ ) __all__ = [ + "US_CGD_ROUTE_REQUIRED_INPUTS", "US_RELEASE_INPUT_COVERAGE_RESOURCE", "POST_REFERENCE_ECPS_REQUIRED_INPUTS", "RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS", @@ -124,8 +125,9 @@ # The frozen reference artifact predates the retired pipeline's export of the # pure FLSA overtime-premium input, OBBBA's distinct qualifying passenger- -# vehicle interest leaf, and the final pipeline's five desired retirement- -# contribution inputs, and its final SIPP-imputed SSI disability criterion. +# vehicle interest leaf, the final pipeline's five desired retirement- +# contribution inputs, its final SIPP-imputed SSI disability criterion, and +# the #282 capital-gain-distributions route split's Schedule-D leg. # These are hard requirements because their shipped validation rows otherwise # become structural zeroes. POST_REFERENCE_ECPS_REQUIRED_INPUTS = frozenset( @@ -137,6 +139,7 @@ "traditional_ira_contributions_desired", "roth_ira_contributions_desired", "self_employed_pension_contributions_desired", + "schedule_d_capital_gain_distributions", *US_SSI_DISABILITY_CRITERIA_OUTPUT_COLUMNS, } ) @@ -197,6 +200,23 @@ "bond_assets", ) +#: The two legs of the #282 capital-gain-distributions route split +#: (populace#462 / #361 remedy): the direct route reported on Form 1040 line 7 +#: without Schedule D (``non_sch_d_capital_gains``, PUF E01100, the SOI Pub +#: 1304 Table 1.4 "capital gain distributions" concept) and the Schedule D +#: line 13 route (``schedule_d_capital_gain_distributions``, a memo component +#: of ``long_term_capital_gains`` written by the ``capital_gain_distributions`` +#: source stage). The Build M live default shipped the direct leg 7.3x over +#: its SOI dollar target while the Schedule-D leg was absent from the export +#: entirely. Both ship as hard requirements with NO reviewed exclusion, so a +#: release whose export drops either route leg fails the coverage gate — red +#: on today's artifacts by design until the Build N rebuild carries the split +#: through. +US_CGD_ROUTE_REQUIRED_INPUTS = ( + "non_sch_d_capital_gains", + "schedule_d_capital_gain_distributions", +) + @dataclass(frozen=True) class ReleaseInputColumn: @@ -656,6 +676,20 @@ def assert_release_input_coverage_manifest_current( "required manifest column (#368)." ) + for route_leg in US_CGD_ROUTE_REQUIRED_INPUTS: + if route_leg in reviewed: + failures.append( + f"{route_leg}: capital-gain-distributions route leg is a " + "reviewed exclusion, but #462 requires both #282 route legs " + "be hard requirements with no exclusion so a release that " + "drops either route from the export fails the coverage gate." + ) + elif route_leg not in required: + failures.append( + f"{route_leg}: capital-gain-distributions route leg must be a " + "required manifest column (#462)." + ) + for column in RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS: if column in reviewed: failures.append( diff --git a/packages/populace-build/tests/test_gates.py b/packages/populace-build/tests/test_gates.py index 7811d23c..cae32697 100644 --- a/packages/populace-build/tests/test_gates.py +++ b/packages/populace-build/tests/test_gates.py @@ -14,6 +14,7 @@ GateReport, GateResult, TargetCoverageRequirement, + TargetFitRequirement, aggregate_admin_gate, default_valued_columns_gate, enum_domain_gate, @@ -29,6 +30,8 @@ source_coverage_gate, source_stage_input_coverage_gate, support_gate, + tail_concentration_gate, + target_fit_gate, target_profile_coverage_gate, target_surface_gate, weights_audit_gate, @@ -1429,3 +1432,487 @@ def test_details_map_each_missing_leaf_to_its_consumers(self) -> None: assert result.details["missing_consumers"] == { "qualified_tuition_expenses": ["soi_education_credits"] } + + +def _fit_row(name: str, target: float, final_estimate: float) -> dict: + """A diagnostics-shaped target row (the contract/diagnostics mapping form).""" + relative_error = ( + (final_estimate - target) / target if target != 0.0 else final_estimate - target + ) + return { + "name": name, + "target": target, + "final_estimate": final_estimate, + "relative_error": relative_error, + } + + +#: The live Build M SOI Pub 1304 Table 1.4 national dollar rows +#: (populace#462): the +634.8% capital-gain-distributions defect and the +#: -25.6% net-capital-gains miss shipped while every exact-name critical +#: requirement passed. These rows are the replay fixture the blanket +#: fit requirement must catch. +_TABLE_1_4_REQUIREMENT = TargetFitRequirement( + requirement_id="soi_table_1_4_national_dollar_rows", + label="SOI Pub 1304 Table 1.4 national dollar rows", + accepted_name_prefixes=("irs_soi.",), + accepted_name_substrings=(".table_1_4.",), + accepted_name_suffixes=("_amount@2024",), + max_abs_relative_error=0.25, +) + +_BUILD_M_CGD_ROW = _fit_row( + "irs_soi.ty2023.table_1_4.all.capital_gain_distributions_amount@2024", + 10_155_465_319.0, + 74_617_447_202.0, +) +_BUILD_M_NET_CAPITAL_GAINS_ROW = _fit_row( + "irs_soi.ty2023.table_1_4.all.net_capital_gains_amount@2024", + 1_270_864_366_489.0, + 945_431_772_792.0, +) +_BUILD_M_WAGES_ROW = _fit_row( + "irs_soi.ty2023.table_1_4.all.wages_salaries_amount@2024", + 10_773_360_188_645.0, + 10_774_383_029_502.0, +) +_BUILD_M_CGD_RETURNS_ROW = _fit_row( + "irs_soi.ty2023.table_1_4.all.capital_gain_distributions_returns@2024", + 3_209_131.0, + 3_209_038.0, +) +_HT2_AMOUNT_ROW = _fit_row( + "irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_amount@2024", + 2_105_345_646_000.0, + 2_067_762_165_736.424, +) + + +class _FitRowObject: + """Attribute-shaped row, mirroring calibrate TargetDiagnostic objects.""" + + def __init__(self, payload: dict) -> None: + for key, value in payload.items(): + setattr(self, key, value) + + +class TestTargetFitRequirement: + def test_requires_a_selector(self) -> None: + with pytest.raises(ValueError, match="selector"): + TargetFitRequirement( + requirement_id="x", + label="x", + max_abs_relative_error=0.25, + ) + + def test_requires_positive_finite_tolerance(self) -> None: + with pytest.raises(ValueError, match="max_abs_relative_error"): + TargetFitRequirement( + requirement_id="x", + label="x", + accepted_names=("a",), + max_abs_relative_error=0.0, + ) + with pytest.raises(ValueError, match="max_abs_relative_error"): + TargetFitRequirement( + requirement_id="x", + label="x", + accepted_names=("a",), + max_abs_relative_error=float("nan"), + ) + + def test_requires_id_label_and_min_matches(self) -> None: + with pytest.raises(ValueError, match="requirement_id"): + TargetFitRequirement( + requirement_id="", + label="x", + accepted_names=("a",), + max_abs_relative_error=0.25, + ) + with pytest.raises(ValueError, match="label"): + TargetFitRequirement( + requirement_id="x", + label="", + accepted_names=("a",), + max_abs_relative_error=0.25, + ) + with pytest.raises(ValueError, match="min_matches"): + TargetFitRequirement( + requirement_id="x", + label="x", + accepted_names=("a",), + max_abs_relative_error=0.25, + min_matches=0, + ) + + +class TestTargetFitGate: + def test_within_tolerance_table_1_4_rows_pass(self) -> None: + result = target_fit_gate( + (_BUILD_M_WAGES_ROW, _HT2_AMOUNT_ROW, _BUILD_M_CGD_RETURNS_ROW), + (_TABLE_1_4_REQUIREMENT,), + ) + assert result.passed + assert result.details["matches_by_requirement"] == { + "soi_table_1_4_national_dollar_rows": [ + "irs_soi.ty2023.table_1_4.all.wages_salaries_amount@2024" + ] + } + + def test_build_m_capital_gain_distributions_breach_fails(self) -> None: + result = target_fit_gate( + (_BUILD_M_CGD_ROW, _BUILD_M_WAGES_ROW), + (_TABLE_1_4_REQUIREMENT,), + ) + assert not result.passed + assert len(result.failures) == 1 + failure = result.failures[0] + assert "capital_gain_distributions_amount@2024" in failure + assert "SOI Pub 1304 Table 1.4 national dollar rows" in failure + assert "6.347" in failure + assert "0.25" in failure + + def test_build_m_net_capital_gains_breach_fails_too(self) -> None: + result = target_fit_gate( + (_BUILD_M_CGD_ROW, _BUILD_M_NET_CAPITAL_GAINS_ROW, _BUILD_M_WAGES_ROW), + (_TABLE_1_4_REQUIREMENT,), + ) + assert not result.passed + assert len(result.failures) == 2 + assert any("net_capital_gains_amount@2024" in line for line in result.failures) + + def test_selector_fields_are_conjunctive(self) -> None: + # The ht2 national amount row shares the prefix and the suffix but not + # the ``.table_1_4.`` substring; the table_1_4 returns row shares the + # prefix and substring but not the ``_amount@`` suffix. Neither may + # match a requirement naming the Table 1.4 dollar-row class. + breached_returns = _fit_row( + "irs_soi.ty2023.table_1_4.all.estate_trust_net_loss_returns@2024", + 36_592.0, + 218_052.0, + ) + breached_ht2 = _fit_row( + "irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_amount@2024", + 2_105_345_646_000.0, + 1_000_000_000_000.0, + ) + result = target_fit_gate( + (breached_returns, breached_ht2, _BUILD_M_WAGES_ROW), + (_TABLE_1_4_REQUIREMENT,), + ) + assert result.passed + + def test_object_rows_match_mapping_rows(self) -> None: + result = target_fit_gate( + (_FitRowObject(_BUILD_M_CGD_ROW),), + (_TABLE_1_4_REQUIREMENT,), + ) + assert not result.passed + assert "capital_gain_distributions_amount@2024" in result.failures[0] + + def test_empty_surface_fails_min_matches(self) -> None: + result = target_fit_gate((_HT2_AMOUNT_ROW,), (_TABLE_1_4_REQUIREMENT,)) + assert not result.passed + assert "soi_table_1_4_national_dollar_rows" in result.failures[0] + assert "0 match(es)" in result.failures[0] + + def test_min_matches_floor_is_enforced(self) -> None: + requirement = TargetFitRequirement( + requirement_id="table_1_4_dollar_rows", + label="Table 1.4 dollar rows", + accepted_name_substrings=(".table_1_4.",), + accepted_name_suffixes=("_amount@2024",), + max_abs_relative_error=0.25, + min_matches=2, + ) + result = target_fit_gate((_BUILD_M_WAGES_ROW,), (requirement,)) + assert not result.passed + assert "needs 2" in result.failures[0] + + def test_stale_recorded_relative_error_fails(self) -> None: + row = dict(_BUILD_M_CGD_ROW) + row["relative_error"] = 0.0 + result = target_fit_gate((row,), (_TABLE_1_4_REQUIREMENT,)) + assert not result.passed + assert any("stale relative_error" in line for line in result.failures) + + def test_missing_recorded_relative_error_is_not_stale(self) -> None: + row = { + "name": _BUILD_M_WAGES_ROW["name"], + "target": _BUILD_M_WAGES_ROW["target"], + "final_estimate": _BUILD_M_WAGES_ROW["final_estimate"], + } + result = target_fit_gate((row,), (_TABLE_1_4_REQUIREMENT,)) + assert result.passed + + def test_non_numeric_target_fails(self) -> None: + row = dict(_BUILD_M_WAGES_ROW) + row["target"] = None + result = target_fit_gate((row,), (_TABLE_1_4_REQUIREMENT,)) + assert not result.passed + assert any("non-numeric" in line for line in result.failures) + + def test_non_finite_final_estimate_fails(self) -> None: + row = dict(_BUILD_M_WAGES_ROW) + row["final_estimate"] = float("inf") + result = target_fit_gate((row,), (_TABLE_1_4_REQUIREMENT,)) + assert not result.passed + assert any("non-finite" in line for line in result.failures) + + def test_zero_target_uses_absolute_difference(self) -> None: + requirement = TargetFitRequirement( + requirement_id="zero_target", + label="zero-target row", + accepted_names=("zero.row@2024",), + max_abs_relative_error=0.25, + ) + within = _fit_row("zero.row@2024", 0.0, 0.1) + assert target_fit_gate((within,), (requirement,)).passed + beyond = _fit_row("zero.row@2024", 0.0, 0.3) + assert not target_fit_gate((beyond,), (requirement,)).passed + + def test_reviewed_exclusion_passes_and_is_recorded(self) -> None: + result = target_fit_gate( + (_BUILD_M_CGD_ROW, _BUILD_M_WAGES_ROW), + (_TABLE_1_4_REQUIREMENT,), + reviewed_exclusions={ + _BUILD_M_CGD_ROW["name"]: "populace#462: tracked QRF defect." + }, + ) + assert result.passed + assert result.details["reviewed_exclusions"] == { + _BUILD_M_CGD_ROW["name"]: "populace#462: tracked QRF defect." + } + + def test_reviewed_exclusion_requires_reason(self) -> None: + with pytest.raises(ValueError, match="reason"): + target_fit_gate( + (_BUILD_M_CGD_ROW,), + (_TABLE_1_4_REQUIREMENT,), + reviewed_exclusions={_BUILD_M_CGD_ROW["name"]: ""}, + ) + + def test_stale_reviewed_exclusion_fails(self) -> None: + result = target_fit_gate( + (_BUILD_M_WAGES_ROW,), + (_TABLE_1_4_REQUIREMENT,), + reviewed_exclusions={ + _BUILD_M_WAGES_ROW["name"]: "was out of tolerance once." + }, + ) + assert not result.passed + assert any("Stale reviewed exclusion" in line for line in result.failures) + + def test_unmatched_reviewed_exclusion_fails(self) -> None: + result = target_fit_gate( + (_BUILD_M_WAGES_ROW,), + (_TABLE_1_4_REQUIREMENT,), + reviewed_exclusions={"never.matched@2024": "dangling entry."}, + ) + assert not result.passed + assert any( + "not matched by any fit requirement" in line for line in result.failures + ) + + def test_excluded_rows_still_count_toward_min_matches(self) -> None: + result = target_fit_gate( + (_BUILD_M_CGD_ROW,), + (_TABLE_1_4_REQUIREMENT,), + reviewed_exclusions={ + _BUILD_M_CGD_ROW["name"]: "populace#462: tracked QRF defect." + }, + ) + assert result.passed + + def test_details_carry_relative_errors_and_worst_offenders(self) -> None: + result = target_fit_gate( + (_BUILD_M_CGD_ROW, _BUILD_M_NET_CAPITAL_GAINS_ROW, _BUILD_M_WAGES_ROW), + (_TABLE_1_4_REQUIREMENT,), + ) + errors = result.details["relative_errors"] + assert errors[_BUILD_M_CGD_ROW["name"]] == pytest.approx(6.3475, abs=5e-4) + worst = list(result.details["worst_offenders"]) + assert worst[0] == _BUILD_M_CGD_ROW["name"] + + +class TestTailConcentrationGate: + def _build_m_signature(self) -> tuple[np.ndarray, np.ndarray]: + """The populace#462 non_sch_d_capital_gains tail point-mass, to scale. + + 2,295 nonzero records; the top 100 weighted records carry ~89% of the + weighted dollar mass via a repeated $594,484 donor-ceiling value. + """ + body_values = np.full(2_195, 2_979.0) + body_weights = np.full(2_195, 1_400.0) + tail_values = np.full(100, 594_484.0) + tail_weights = np.full(100, 1_117.0) + values = np.concatenate([body_values, tail_values]) + weights = np.concatenate([body_weights, tail_weights]) + return values, weights + + def test_build_m_point_mass_signature_fails(self) -> None: + values, weights = self._build_m_signature() + result = tail_concentration_gate( + {"non_sch_d_capital_gains": values}, + {"non_sch_d_capital_gains": weights}, + ) + assert not result.passed + failure = result.failures[0] + assert "non_sch_d_capital_gains" in failure + assert "top 100" in failure + share = result.details["top_share"]["non_sch_d_capital_gains"] + assert share == pytest.approx(0.89, abs=0.02) + + def test_dispersed_mass_passes(self) -> None: + rng = np.random.default_rng(462) + values = rng.lognormal(mean=8.0, sigma=1.0, size=2_295) + weights = np.full(2_295, 1_400.0) + result = tail_concentration_gate( + {"non_sch_d_capital_gains": values}, + {"non_sch_d_capital_gains": weights}, + ) + assert result.passed + share = result.details["top_share"]["non_sch_d_capital_gains"] + assert share < 0.5 + + def test_share_at_threshold_passes(self) -> None: + # 10 records of mass 3 and 90 of mass 1: top-10 share is exactly 0.25. + values = np.concatenate([np.full(10, 3.0), np.full(90, 1.0)]) + weights = np.ones(100) + result = tail_concentration_gate( + {"column": values}, + {"column": weights}, + top_k=10, + max_top_share=0.25, + min_nonzero_records=50, + ) + assert result.passed + + def test_thin_columns_are_skipped_and_reported(self) -> None: + values = np.concatenate([np.full(10, 594_484.0), np.zeros(90)]) + weights = np.ones(100) + result = tail_concentration_gate( + {"thin": values}, + {"thin": weights}, + top_k=5, + min_nonzero_records=50, + ) + assert result.passed + assert result.details["thin_columns"] == {"thin": 10} + + def test_all_zero_column_is_thin_not_failing(self) -> None: + result = tail_concentration_gate( + {"empty": np.zeros(1_000)}, + {"empty": np.ones(1_000)}, + ) + assert result.passed + assert result.details["thin_columns"] == {"empty": 0} + + def test_zero_weight_records_carry_no_mass(self) -> None: + # Nonzero values on zero-weight records are not weighted carriers: 40 + # weighted carriers is under the floor, so the column is thin. + values = np.concatenate([np.full(40, 100.0), np.full(160, 100.0)]) + weights = np.concatenate([np.ones(40), np.zeros(160)]) + result = tail_concentration_gate( + {"column": values}, + {"column": weights}, + top_k=5, + min_nonzero_records=50, + ) + assert result.passed + assert result.details["thin_columns"] == {"column": 40} + + def test_signed_values_use_absolute_mass(self) -> None: + values, weights = self._build_m_signature() + result = tail_concentration_gate( + {"signed": -values}, + {"signed": weights}, + ) + assert not result.passed + + def test_missing_weights_are_refused(self) -> None: + with pytest.raises(ValueError, match="weights"): + tail_concentration_gate({"column": np.ones(600)}, {}) + + def test_misaligned_weights_are_refused(self) -> None: + with pytest.raises(ValueError, match="align"): + tail_concentration_gate( + {"column": np.ones(600)}, + {"column": np.ones(599)}, + ) + + def test_top_k_must_sit_inside_the_support_floor(self) -> None: + with pytest.raises(ValueError, match="min_nonzero_records"): + tail_concentration_gate( + {"column": np.ones(600)}, + {"column": np.ones(600)}, + top_k=100, + min_nonzero_records=100, + ) + + def test_share_threshold_bounds_are_validated(self) -> None: + with pytest.raises(ValueError, match="max_top_share"): + tail_concentration_gate( + {"column": np.ones(600)}, + {"column": np.ones(600)}, + max_top_share=1.0, + ) + with pytest.raises(ValueError, match="top_k"): + tail_concentration_gate( + {"column": np.ones(600)}, + {"column": np.ones(600)}, + top_k=0, + ) + + def test_reviewed_exclusion_passes_and_is_recorded(self) -> None: + values, weights = self._build_m_signature() + result = tail_concentration_gate( + {"non_sch_d_capital_gains": values}, + {"non_sch_d_capital_gains": weights}, + reviewed_exclusions={ + "non_sch_d_capital_gains": "populace#462: Build N QRF fix owns this." + }, + ) + assert result.passed + assert result.details["reviewed_exclusions"] == { + "non_sch_d_capital_gains": "populace#462: Build N QRF fix owns this." + } + + def test_reviewed_exclusion_requires_reason(self) -> None: + values, weights = self._build_m_signature() + with pytest.raises(ValueError, match="reason"): + tail_concentration_gate( + {"non_sch_d_capital_gains": values}, + {"non_sch_d_capital_gains": weights}, + reviewed_exclusions={"non_sch_d_capital_gains": ""}, + ) + + def test_stale_reviewed_exclusion_fails(self) -> None: + values = np.ones(2_295) + weights = np.full(2_295, 1_400.0) + result = tail_concentration_gate( + {"column": values}, + {"column": weights}, + reviewed_exclusions={"column": "was concentrated once."}, + ) + assert not result.passed + assert any("Stale reviewed exclusion" in line for line in result.failures) + + def test_exclusion_for_absent_column_is_dormant_not_failing(self) -> None: + result = tail_concentration_gate( + {"column": np.ones(600)}, + {"column": np.ones(600)}, + reviewed_exclusions={"other_column": "different release line."}, + ) + assert result.passed + assert result.details["dormant_exclusions"] == ["other_column"] + + def test_details_are_json_ready(self) -> None: + import json as json_module + + values, weights = self._build_m_signature() + result = tail_concentration_gate( + {"non_sch_d_capital_gains": values}, + {"non_sch_d_capital_gains": weights}, + ) + json_module.dumps(dict(result.details)) diff --git a/packages/populace-build/tests/test_release_input_coverage.py b/packages/populace-build/tests/test_release_input_coverage.py index 0786db61..b52ca2f3 100644 --- a/packages/populace-build/tests/test_release_input_coverage.py +++ b/packages/populace-build/tests/test_release_input_coverage.py @@ -32,6 +32,7 @@ import populace.build.us_runtime.reform_coverage_smoke as smoke_module from populace.build.us_runtime import ( SSI_COUNTABLE_RESOURCE_ASSETS, + US_CGD_ROUTE_REQUIRED_INPUTS, US_QBI_OUTPUT_COLUMNS, US_RELEASE_INPUT_COVERAGE_RESOURCE, ReformCoverageProbe, @@ -1307,3 +1308,56 @@ def test_generated_manifest_names_no_retired_data_package(self) -> None: # trip the live-tree guard it mirrors (test_us_plan does the same). assert ("policyengine-" + "us-data") not in rendered assert ("policyengine_" + "us_data") not in rendered + + +class TestCapitalGainDistributionRouteGuarantee: + """populace#462 / #361 remedy: BOTH capital-gain-distribution route legs + are export-guarded. The Build M live default shipped the direct-route leg + (``non_sch_d_capital_gains``) 7.3x over its SOI dollar target while the + Schedule-D route (``schedule_d_capital_gain_distributions``) was absent + from the export entirely — and the coverage manifest guarded neither + against demotion.""" + + def test_route_constant_names_both_legs(self) -> None: + assert US_CGD_ROUTE_REQUIRED_INPUTS == ( + "non_sch_d_capital_gains", + "schedule_d_capital_gain_distributions", + ) + + def test_both_route_variables_are_required_without_exclusion(self) -> None: + manifest = load_release_input_coverage_manifest() + for column in US_CGD_ROUTE_REQUIRED_INPUTS: + assert column in manifest.required_columns + assert column not in manifest.reviewed_exclusions + + @pytest.mark.parametrize( + "column", + [ + "non_sch_d_capital_gains", + "schedule_d_capital_gain_distributions", + ], + ) + def test_route_variable_cannot_regress_to_reviewed_exclusion( + self, column: str + ) -> None: + manifest = load_release_input_coverage_manifest() + assert column in manifest.declared_columns + demoted = ReleaseInputCoverageManifest( + reference=manifest.reference, + columns=tuple( + ReleaseInputColumn( + name=entry.name, + status="reviewed_exclusion", + reason="regression attempt", + issue="PolicyEngine/populace#462", + ) + if entry.name == column + else entry + for entry in manifest.columns + ), + probes=manifest.probes, + schema_version=manifest.schema_version, + ) + + with pytest.raises(ValueError, match=column): + assert_release_input_coverage_manifest_current(manifest=demoted) diff --git a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py index 41a21db9..7787718c 100644 --- a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py @@ -971,6 +971,13 @@ def diagnostic(name, target, final_estimate): 24_475_100.0, 26_000_000.0, ), + # The SOI Table 1.4 national dollar blanket (populace#462) needs at + # least one Table 1.4 amount row on the surface, within tolerance. + diagnostic( + "irs_soi.ty2023.table_1_4.all.wages_salaries_amount", + 10_773_360_188_645.0, + 10_774_383_029_502.0, + ), ) @@ -7357,3 +7364,199 @@ def test_ssi_swap_delta_dense_cap_ratio_admits_measured_dense_equilibrium() -> N )["within_bound"] is False ) + + +def _table_1_4_diagnostic(builder, name: str, target: float, final: float): + return SimpleNamespace( + name=f"{name}@{builder.PERIOD}", + target=target, + initial_estimate=target, + final_estimate=final, + relative_error=(final - target) / target, + ) + + +def test_release_gate_failures_block_table_1_4_dollar_breaches() -> None: + builder = _load_builder_module() + breached = ( + # The live Build M defect pair (populace#462): +634.8% on the + # capital-gain-distributions dollar row, -25.6% on net capital gains. + _table_1_4_diagnostic( + builder, + "irs_soi.ty2023.table_1_4.all.capital_gain_distributions_amount", + 10_155_465_319.0, + 74_617_447_202.0, + ), + _table_1_4_diagnostic( + builder, + "irs_soi.ty2023.table_1_4.all.net_capital_gains_amount", + 1_270_864_366_489.0, + 945_431_772_792.0, + ), + ) + result = SimpleNamespace( + skipped=(), + diagnostics=_passing_critical_diagnostics(builder) + breached, + initial_loss=10.0, + final_loss=5.0, + ) + + failures = builder._release_gate_failures(result, {"dropped_target_names": []}) + + assert len(failures) == 2 + assert all( + failure.startswith("SOI Table 1.4 national dollar fit failed: ") + for failure in failures + ) + joined = "\n".join(failures) + assert "capital_gain_distributions_amount@2024" in joined + assert "net_capital_gains_amount@2024" in joined + assert "6.3475" in joined + + +def test_release_gate_failures_ignore_table_1_4_returns_rows() -> None: + builder = _load_builder_module() + # A wildly-missed returns (count) row is outside the dollar blanket: the + # live Build M estate_trust_net_loss_returns row landed at +495.9% and is + # a distinct defect class, not this gate's scope. + returns_row = _table_1_4_diagnostic( + builder, + "irs_soi.ty2023.table_1_4.all.estate_trust_net_loss_returns", + 36_592.0, + 218_052.0, + ) + result = SimpleNamespace( + skipped=(), + diagnostics=_passing_critical_diagnostics(builder) + (returns_row,), + initial_loss=10.0, + final_loss=5.0, + ) + + assert builder._release_gate_failures(result, {"dropped_target_names": []}) == [] + + +def test_release_gate_failures_require_a_table_1_4_dollar_surface() -> None: + builder = _load_builder_module() + without_table_1_4 = tuple( + diagnostic + for diagnostic in _passing_critical_diagnostics(builder) + if ".table_1_4." not in diagnostic.name + ) + result = SimpleNamespace( + skipped=(), + diagnostics=without_table_1_4, + initial_loss=10.0, + final_loss=5.0, + ) + + failures = builder._release_gate_failures(result, {"dropped_target_names": []}) + + assert any("soi_table_1_4_national_dollar_rows" in failure for failure in failures) + + +def test_qrf_imputed_source_outputs_come_from_the_stage_manifest() -> None: + builder = _load_builder_module() + + outputs = builder._qrf_imputed_source_outputs() + + assert "non_sch_d_capital_gains" in outputs + assert "taxable_interest_income" in outputs + assert len(outputs) >= 60 + # The capital_gain_distributions stage is a share split, not a QRF fit. + assert "schedule_d_capital_gain_distributions" not in outputs + + +def _qrf_export_frame(builder, non_sch_d_values: np.ndarray) -> Frame: + n = int(non_sch_d_values.size) + ids = np.arange(1, n + 1, dtype="int64") + taxable_interest = np.zeros(n) + taxable_interest[: n // 2] = 1_000.0 + person = pd.DataFrame( + { + "person_id": ids, + "person_household_id": ids, + "person_tax_unit_id": ids, + "person_spm_unit_id": ids, + "person_family_id": ids, + "person_marital_unit_id": ids, + "non_sch_d_capital_gains": non_sch_d_values, + "taxable_interest_income": taxable_interest, + } + ) + return Frame( + { + "person": person, + "household": pd.DataFrame({"household_id": ids}), + "tax_unit": pd.DataFrame({"tax_unit_id": ids}), + "spm_unit": pd.DataFrame({"spm_unit_id": ids}), + "family": pd.DataFrame({"family_id": ids}), + "marital_unit": pd.DataFrame({"marital_unit_id": ids}), + }, + builder.US_SCHEMA, + { + "household": builder.Weights( + values=np.ones(n), + kind=WeightKind.DESIGN, + ) + }, + ) + + +def test_qrf_tail_concentration_gate_flags_the_build_m_point_mass() -> None: + builder = _load_builder_module() + # 500 carriers of 12,000 person records (4.2% — sparse); the top 100 carry + # the repeated $594,484 donor ceiling, ~98% of the weighted mass. + values = np.zeros(12_000) + values[:100] = 594_484.0 + values[100:500] = 2_979.0 + + gate, surface = builder._qrf_tail_concentration_gate( + _qrf_export_frame(builder, values) + ) + + assert not gate.passed + assert any("non_sch_d_capital_gains" in line for line in gate.failures) + assert surface["checked_sparse_columns"] == ["non_sch_d_capital_gains"] + assert "taxable_interest_income" in surface["dense_columns"] + assert "short_term_capital_gains" in surface["absent_columns"] + + +def test_qrf_tail_concentration_gate_passes_dispersed_sparse_mass() -> None: + builder = _load_builder_module() + values = np.zeros(12_000) + values[:500] = 2_979.0 + + gate, _ = builder._qrf_tail_concentration_gate(_qrf_export_frame(builder, values)) + + assert gate.passed + + +def test_allow_qrf_tail_concentration_flag_parses(monkeypatch) -> None: + builder = _load_builder_module() + + monkeypatch.setattr( + sys, + "argv", + [ + "build_us_fiscal_refresh_release.py", + "--ledger-facts", + "facts.jsonl", + "--out", + "release", + ], + ) + assert not builder._parse_args().allow_qrf_tail_concentration + + monkeypatch.setattr( + sys, + "argv", + [ + "build_us_fiscal_refresh_release.py", + "--ledger-facts", + "facts.jsonl", + "--out", + "release", + "--allow-qrf-tail-concentration", + ], + ) + assert builder._parse_args().allow_qrf_tail_concentration diff --git a/packages/populace-data/src/populace/data/contract.py b/packages/populace-data/src/populace/data/contract.py index c5b87eb1..87b18873 100644 --- a/packages/populace-data/src/populace/data/contract.py +++ b/packages/populace-data/src/populace/data/contract.py @@ -222,6 +222,27 @@ "target_roles": ("medical_expense_deduction_total",), "allow_incumbent_improvement": False, }, + # populace#462: every national SOI Pub 1304 Table 1.4 dollar row is + # within-tolerance-blocking by NAME PATTERN, not enumeration. The Build M + # live default shipped the Table 1.4 capital-gain-distributions dollar row + # at +634.8% relative error — recorded in its own diagnostics — because no + # exact-name entry above covered it. 0.25 is the established broad-fit + # bound (the incumbent-improvement hard stop): on the live Build M surface + # it fails exactly the two defect rows (capital_gain_distributions_amount + # +634.8%, net_capital_gains_amount -25.6%) and passes the other nine + # Table 1.4 dollar rows. No incumbent-improvement escape: a national + # dollar row beyond broad fit never certifies. + { + "requirement_id": "soi_table_1_4_national_dollar_rows", + "label": "SOI Pub 1304 Table 1.4 national dollar rows", + "max_abs_relative_error": 0.25, + "names": (), + "families": (), + "target_roles": (), + "name_substrings": (".table_1_4.",), + "name_suffixes": ("_amount@2024",), + "allow_incumbent_improvement": False, + }, ) _US_CRITICAL_TARGET_IMPROVEMENT_MAX_ABS_RELATIVE_ERROR = 0.25 @@ -839,6 +860,27 @@ def _check_calibration_diagnostics(diagnostics: Mapping, failures: list[str]) -> ) +def _matches_requirement_name_pattern(name: str, requirement: Mapping) -> bool: + """Whether a row name matches a requirement's name-shape selectors. + + Pattern selectors (``name_substrings`` / ``name_suffixes``) are + conjunctive across fields and disjunctive within a field, so a + requirement can name a row *class* — every national SOI Table 1.4 dollar + row (substring ``.table_1_4.`` AND suffix ``_amount@2024``) — instead of + enumerating rows one by one (the populace#462 gap: the enumerated + register cannot block a row nobody listed). + """ + substrings = tuple(requirement.get("name_substrings", ())) + suffixes = tuple(requirement.get("name_suffixes", ())) + if not substrings and not suffixes: + return False + if substrings and not any(substring in name for substring in substrings): + return False + if suffixes and not any(name.endswith(suffix) for suffix in suffixes): + return False + return True + + def _check_us_critical_target_fit(diagnostics: Mapping, failures: list[str]) -> None: targets = diagnostics.get("targets") if not isinstance(targets, list): @@ -855,6 +897,9 @@ def _check_us_critical_target_fit(diagnostics: Mapping, failures: list[str]) -> and not _is_congressional_district_layout_target(target) and ( target.get("name") in names + or _matches_requirement_name_pattern( + str(target.get("name") or ""), requirement + ) or ( isinstance(target.get("metadata"), Mapping) and target["metadata"].get("target_role") in target_roles diff --git a/packages/populace-data/tests/test_contract.py b/packages/populace-data/tests/test_contract.py index c981833a..f9df2bba 100644 --- a/packages/populace-data/tests/test_contract.py +++ b/packages/populace-data/tests/test_contract.py @@ -30,7 +30,7 @@ SOURCE_COVERAGE_SHA = "9" * 64 TARGET_SURFACE_SHA = "e" * 64 REGISTRY_VERSION = "registryabc123" -TARGET_COUNT = 17 +TARGET_COUNT = 18 DEDUCTION_CRITICAL_TARGETS = ( ( @@ -237,6 +237,19 @@ def _calibration_diagnostics() -> dict: ), *additional_critical_credit_rows(), *deduction_critical_target_rows(), + # The SOI Table 1.4 national dollar blanket (populace#462) needs + # at least one Table 1.4 dollar row on the surface, within its + # 25% blocking tolerance (the live Build M wages row). + _target_row( + "irs_soi.ty2023.table_1_4.all.wages_salaries_amount@2024", + target_name="irs_soi.ty2023.table_1_4.all.wages_salaries_amount", + target=10_773_360_188_645.0, + initial_estimate=10_500_000_000_000.0, + final_estimate=10_774_383_029_502.0, + relative_error=(10_774_383_029_502.0 - 10_773_360_188_645.0) + / 10_773_360_188_645.0, + family="irs_soi", + ), ], } @@ -404,7 +417,7 @@ def _source_coverage_diagnostics() -> dict: }, "irs_soi": { "label": "IRS Statistics of Income", - "target_count": 15, + "target_count": 16, "sources": ["IRS SOI Historic Table 2"], "reference_urls": ["https://example.test/soi"], }, @@ -594,8 +607,7 @@ def test_us_release_ignores_congressional_district_layout_critical_fit( salt_target = next( row for row in diagnostics["targets"] - if row["name"] - == "irs_soi.ty2022.historic_table_2.us.all." + if row["name"] == "irs_soi.ty2022.historic_table_2.us.all." "limited_state_local_taxes_amount@2024" ) cd_layout_target = dict(salt_target) @@ -1416,3 +1428,108 @@ def test_all_failures_reported_at_once(release_dir: Path) -> None: def test_a_missing_directory_is_a_contract_error(tmp_path: Path) -> None: with pytest.raises(ReleaseContractError, match="is not a directory"): validate_release_dir(tmp_path / "releases" / "nope") + + +def test_us_release_rejects_table_1_4_national_dollar_breach( + release_dir: Path, +) -> None: + diagnostics = _calibration_diagnostics() + target = next( + row + for row in diagnostics["targets"] + if row["name"] == "irs_soi.ty2023.table_1_4.all.wages_salaries_amount@2024" + ) + # Replay the live Build M defect (populace#462) onto the fixture's Table + # 1.4 row: the capital-gain-distributions dollar row shipped at +634.8% + # relative error, recorded in the release's own diagnostics. + target["name"] = ( + "irs_soi.ty2023.table_1_4.all.capital_gain_distributions_amount@2024" + ) + target["target_name"] = ( + "irs_soi.ty2023.table_1_4.all.capital_gain_distributions_amount" + ) + target["target"] = 10_155_465_319.0 + target["compiled_target"] = 10_155_465_319.0 + target["initial_estimate"] = 10_155_465_319.0 + target["final_estimate"] = 74_617_447_202.0 + target["relative_error"] = (74_617_447_202.0 - 10_155_465_319.0) / 10_155_465_319.0 + _write_json_and_refresh_manifest_hash( + release_dir, + filename="calibration_diagnostics.json", + artifact_key="calibration_diagnostics", + payload=diagnostics, + ) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + + failures = "\n".join(excinfo.value.failures) + assert "SOI Pub 1304 Table 1.4 national dollar rows" in failures + assert "capital_gain_distributions_amount" in failures + assert "relative_error=6.3475" in failures + + +def test_us_release_requires_table_1_4_national_dollar_rows( + release_dir: Path, +) -> None: + diagnostics = _calibration_diagnostics() + target = next( + row + for row in diagnostics["targets"] + if row["name"] == "irs_soi.ty2023.table_1_4.all.wages_salaries_amount@2024" + ) + # Rename the only Table 1.4 dollar row out of the class (keeping the row + # count intact): a diagnostics surface with no national Table 1.4 dollar + # row must not certify — a dropped or renamed feed family gates nothing. + target["name"] = "irs_soi.ty2023.table_1_9.all.wages_salaries_amount@2024" + target["target_name"] = "irs_soi.ty2023.table_1_9.all.wages_salaries_amount" + _write_json_and_refresh_manifest_hash( + release_dir, + filename="calibration_diagnostics.json", + artifact_key="calibration_diagnostics", + payload=diagnostics, + ) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(release_dir) + + failures = "\n".join(excinfo.value.failures) + assert "soi_table_1_4_national_dollar_rows" in failures + + +def test_us_release_table_1_4_returns_rows_are_outside_the_dollar_blanket( + release_dir: Path, +) -> None: + diagnostics = _calibration_diagnostics() + target = next( + row for row in diagnostics["targets"] if row["name"] == "population@2024" + ) + # The live Build M estate-trust net-loss RETURNS row landed at +495.9%; a + # count row is a distinct defect class the dollar blanket must not gate. + target["name"] = "irs_soi.ty2023.table_1_4.all.estate_trust_net_loss_returns@2024" + target["target_name"] = "irs_soi.ty2023.table_1_4.all.estate_trust_net_loss_returns" + target["target"] = 36_592.0 + target["compiled_target"] = 36_592.0 + target["initial_estimate"] = 36_592.0 + target["final_estimate"] = 218_052.0 + target["relative_error"] = (218_052.0 - 36_592.0) / 36_592.0 + target["registry"] = {"family": "irs_soi"} + _write_json_and_refresh_manifest_hash( + release_dir, + filename="calibration_diagnostics.json", + artifact_key="calibration_diagnostics", + payload=diagnostics, + ) + source_coverage = json.loads( + (release_dir / US_SOURCE_COVERAGE_DIAGNOSTICS_FILE).read_text() + ) + del source_coverage["fiscal_target_sources"]["cbo"] + source_coverage["fiscal_target_sources"]["irs_soi"]["target_count"] += 1 + _write_json_and_refresh_manifest_hash( + release_dir, + filename=US_SOURCE_COVERAGE_DIAGNOSTICS_FILE, + artifact_key="us_source_coverage", + payload=source_coverage, + ) + + validate_release_dir(release_dir) diff --git a/packages/populace-data/tests/test_release.py b/packages/populace-data/tests/test_release.py index 46015fb9..3d9bf011 100644 --- a/packages/populace-data/tests/test_release.py +++ b/packages/populace-data/tests/test_release.py @@ -42,7 +42,7 @@ def _no_slack_webhook(monkeypatch): CALIBRATION_SHA = "ac31f2be76a0f8dc4da89b6935aa4b8b1b2e1bd4eb3d03b809333084f25b376e" TARGET_SURFACE_SHA = "e" * 64 REGISTRY_VERSION = "registryabc123" -TARGET_COUNT = 17 +TARGET_COUNT = 18 DEDUCTION_CRITICAL_TARGETS = ( ( @@ -153,6 +153,19 @@ def _calibration_diagnostics() -> dict: ), *additional_critical_credit_rows(), *deduction_critical_target_rows(), + # The SOI Table 1.4 national dollar blanket (populace#462) needs + # at least one Table 1.4 dollar row on the surface, within its + # 25% blocking tolerance (the live Build M wages row). + _target_row( + "irs_soi.ty2023.table_1_4.all.wages_salaries_amount@2024", + target_name="irs_soi.ty2023.table_1_4.all.wages_salaries_amount", + target=10_773_360_188_645.0, + initial_estimate=10_500_000_000_000.0, + final_estimate=10_774_383_029_502.0, + relative_error=(10_774_383_029_502.0 - 10_773_360_188_645.0) + / 10_773_360_188_645.0, + family="irs_soi", + ), ], } @@ -320,7 +333,7 @@ def _source_coverage_diagnostics() -> dict: }, "irs_soi": { "label": "IRS Statistics of Income", - "target_count": 15, + "target_count": 16, "sources": ["IRS SOI Historic Table 2"], "reference_urls": ["https://example.test/soi"], }, diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 32b11b8e..a9c60a76 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -53,10 +53,13 @@ from populace.build.gates import ( GateResult, + TargetFitRequirement, default_valued_columns_gate, input_mass_parity_gate, nonconstant_columns_gate, parity_gate, + tail_concentration_gate, + target_fit_gate, target_profile_coverage_gate, ) from populace.build.ledger_artifact import load_ledger_consumer_artifact @@ -507,6 +510,32 @@ def _automatic_gc_suspended(): }, ) +#: Blanket within-tolerance blocking for every national SOI Pub 1304 Table 1.4 +#: dollar row (populace#462). The exact-name register above only blocks rows +#: someone enumerated; Build M shipped the Table 1.4 capital-gain-distributions +#: dollar row at +634.8% relative error (and net capital gains at -25.6%) with +#: both recorded in its own calibration_diagnostics.json, because no register +#: named them. 0.25 is the established broad-fit bound (the per-family hard +#: threshold and the incumbent-improvement hard stop): on the live Build M +#: surface it fails exactly the two defect rows and passes the other nine +#: Table 1.4 dollar rows (worst passer: taxable_social_security_amount at +#: -10.9%). Deliberately no incumbent-improvement escape — a national dollar +#: row beyond broad fit is never certifiable. +US_SOI_TABLE_1_4_NATIONAL_DOLLAR_MAX_ABS_RELATIVE_ERROR = 0.25 +US_SOI_TABLE_1_4_NATIONAL_DOLLAR_FIT_REQUIREMENT = TargetFitRequirement( + requirement_id="soi_table_1_4_national_dollar_rows", + label="SOI Pub 1304 Table 1.4 national dollar rows", + accepted_name_prefixes=("irs_soi.",), + accepted_name_substrings=(".table_1_4.",), + accepted_name_suffixes=(f"_amount@{PERIOD}",), + max_abs_relative_error=US_SOI_TABLE_1_4_NATIONAL_DOLLAR_MAX_ABS_RELATIVE_ERROR, + notes=( + "populace#462: the Build M live default shipped non_sch_d_capital_gains " + "at $74.6B against its $10.2B SOI target with no blocking tolerance on " + "the dollar row." + ), +) + DIRECT_ACTIVE_ALIASES = ( "census-pep-2024-national-age-sex", "census-pep-2024-state-age-sex", @@ -889,6 +918,18 @@ def _parse_args() -> argparse.Namespace: "certification blocker." ), ) + parser.add_argument( + "--allow-qrf-tail-concentration", + action="store_true", + help=( + "Diagnostic escape hatch (populace#462): record the QRF " + "tail-concentration gate result — sparse QRF-imputed dollar " + "columns whose top-k weighted records carry an implausible share " + "of the weighted mass (the non_sch_d_capital_gains donor-ceiling " + "point mass) — without failing the build. Release builds must " + "leave this unset." + ), + ) parser.add_argument( "--skip-reform-coverage-smoke", action="store_true", @@ -4702,6 +4743,95 @@ def _export_input_mass_gate( ) +#: QRF tail-concentration gate parameters (populace#462). top_k=100 and the +#: 0.75 share threshold are calibrated to the incident: the Build M +#: non_sch_d_capital_gains column carried 89% of its weighted mass in its top +#: 100 records (a repeated $594,484 donor-ceiling value) across 2,295 +#: carriers. Columns are checked only when sparse (nonzero on at most 5% of +#: their entity's records — the regime where a conditional QRF without a +#: participation margin tail-broadcasts) and wide enough (at least 500 +#: weighted carriers) for a top-100 share to be evidence rather than +#: arithmetic. +US_QRF_TAIL_CONCENTRATION_TOP_K = 100 +US_QRF_TAIL_CONCENTRATION_MAX_TOP_SHARE = 0.75 +US_QRF_TAIL_CONCENTRATION_MIN_NONZERO_RECORDS = 500 +US_QRF_SPARSE_NONZERO_SHARE_MAX = 0.05 + + +def _qrf_imputed_source_outputs() -> frozenset[str]: + """Variables produced by a ``fit_weighted_qrf`` source-stage operation. + + Derived from the declarative stage manifest (``us/source_stages.json``) + rather than a hand list, so a new QRF-imputed stage output is covered by + the tail-concentration gate the day the manifest declares it. + """ + return frozenset( + output + for stage in US_SOURCE_MANIFEST.stages + if any(operation.kind == "fit_weighted_qrf" for operation in stage.operations) + for output in stage.outputs + ) + + +def _qrf_tail_concentration_gate( + export_frame: Frame, +) -> tuple[GateResult, dict[str, object]]: + """Tail-concentration gate over the sparse QRF-imputed export columns. + + Runs :func:`populace.build.gates.tail_concentration_gate` on every + QRF-imputed source-stage output the export persists that is sparse + (nonzero share at most :data:`US_QRF_SPARSE_NONZERO_SHARE_MAX` of its + entity's records), at the export's calibrated weights. Returns the gate + result plus the surface metadata (which QRF outputs were checked, dense, + absent, or non-numeric) for the release artifact. + """ + qrf_outputs = sorted(_qrf_imputed_source_outputs()) + values: dict[str, np.ndarray] = {} + weights: dict[str, np.ndarray] = {} + absent: list[str] = [] + dense: list[str] = [] + non_numeric: list[str] = [] + entity_weights: dict[str, np.ndarray] = {} + for column in qrf_outputs: + try: + entity = export_frame.column_entity(column) + except ValueError: + absent.append(column) + continue + series = export_frame.table(entity)[column] + if pd.api.types.is_bool_dtype(series): + non_numeric.append(column) + continue + column_values = pd.to_numeric(series, errors="coerce").fillna(0.0) + array = column_values.to_numpy(dtype=np.float64) + nonzero_share = float((array != 0.0).mean()) if array.size else 0.0 + if nonzero_share > US_QRF_SPARSE_NONZERO_SHARE_MAX: + dense.append(column) + continue + if entity not in entity_weights: + entity_weights[entity] = np.asarray( + export_frame.resolve_weights(entity).values, dtype=np.float64 + ) + values[column] = array + weights[column] = entity_weights[entity] + gate = tail_concentration_gate( + values, + weights, + top_k=US_QRF_TAIL_CONCENTRATION_TOP_K, + max_top_share=US_QRF_TAIL_CONCENTRATION_MAX_TOP_SHARE, + min_nonzero_records=US_QRF_TAIL_CONCENTRATION_MIN_NONZERO_RECORDS, + ) + surface: dict[str, object] = { + "qrf_imputed_outputs": len(qrf_outputs), + "checked_sparse_columns": sorted(values), + "dense_columns": dense, + "absent_columns": absent, + "non_numeric_columns": non_numeric, + "sparse_nonzero_share_max": US_QRF_SPARSE_NONZERO_SHARE_MAX, + } + return gate, surface + + def _person_population(frame: Frame) -> float: return float(frame.resolve_weights("person").values.sum()) @@ -5955,6 +6085,21 @@ def _release_gate_failures( incumbent_diagnostics=incumbent_diagnostics, ) ) + # populace#462: every national SOI Pub 1304 Table 1.4 dollar row is + # within-tolerance-blocking, by name pattern rather than enumeration, so + # rows the exact-name register above never listed (the +634.8% + # capital-gain-distributions defect) cannot certify. No incumbent- + # improvement escape: a national dollar row beyond broad fit never ships. + soi_table_1_4_gate = target_fit_gate( + getattr(result, "diagnostics", ()) or (), + (US_SOI_TABLE_1_4_NATIONAL_DOLLAR_FIT_REQUIREMENT,), + name="soi_table_1_4_national_dollar_fit", + ) + if not soi_table_1_4_gate.passed: + failures.extend( + f"SOI Table 1.4 national dollar fit failed: {failure}" + for failure in soi_table_1_4_gate.failures + ) if not math.isfinite(result.initial_loss) or not math.isfinite(result.final_loss): failures.append("Calibration loss is non-finite.") elif result.final_loss > result.initial_loss: @@ -9132,8 +9277,62 @@ def main() -> None: f"Input mass parity failed: {failure}" for failure in export_input_mass_gate.failures ) - # Batched pre-export raise: the calibration battery, input coverage, and - # export-mass parity have ALL been evaluated at this point, so one failed + # populace#462: tail-concentration gate over the sparse QRF-imputed dollar + # columns at the export's calibrated weights. The Build M defect — 89% of + # the shipped non_sch_d_capital_gains mass in 100 records via a repeated + # $594,484 donor-ceiling value — is invisible to support clipping (every + # draw inside donor range), count targets (carrier count exact), and mass + # parity (column excluded from the reference band), but is unmistakable as + # top-k weighted-mass share. + try: + qrf_tail_gate, qrf_tail_surface = _qrf_tail_concentration_gate(export_frame) + except Exception as exc: + # Same degraded-mode contract as the coverage gate above. + if not terminal_gate_failures: + raise + terminal_gate_failures.append( + "QRF tail concentration failed: evaluation error under earlier " + f"gate failures: {type(exc).__name__}: {exc}" + ) + qrf_tail_gate = None + qrf_tail_surface = None + if qrf_tail_gate is not None: + qrf_tail_path = release_dir / "qrf_tail_concentration.json" + qrf_tail_path.write_text( + json.dumps( + { + "schema_version": 1, + "enforced": not args.allow_qrf_tail_concentration, + "surface": qrf_tail_surface, + "tail_concentration": { + "passed": qrf_tail_gate.passed, + "failures": list(qrf_tail_gate.failures), + "details": dict(qrf_tail_gate.details), + }, + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + if telemetry is not None: + telemetry.attach_artifact("qrf_tail_concentration", qrf_tail_path) + if not qrf_tail_gate.passed and not args.allow_qrf_tail_concentration: + if telemetry is not None: + telemetry.stage( + "export_dataset", + status="failed", + message="QRF tail-concentration gate failed.", + failures=list(qrf_tail_gate.failures), + force_upload=True, + ) + terminal_gate_failures.extend( + f"QRF tail concentration failed: {failure}" + for failure in qrf_tail_gate.failures + ) + # Batched pre-export raise: the calibration battery, input coverage, + # export-mass parity, and QRF tail concentration have ALL been evaluated + # at this point, so one failed # run reports every failing pre-export group at once (Build M attempts 9 # and 10 each burned a ~2h run to surface one of these groups serially). # The reform-coverage smoke and take-up contract keep their own raises diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index 0ca69d09..898afa25 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -62,8 +62,9 @@ # The pinned reference H5 predates the retired pipeline's FLSA-premium export, # OBBBA's distinct qualifying passenger-vehicle interest leaf, its five final -# desired retirement-contribution inputs, and its final SIPP-imputed SSI -# disability criterion. These later inputs are hard requirements because the +# desired retirement-contribution inputs, its final SIPP-imputed SSI +# disability criterion, and the #282 capital-gain-distributions route split's +# Schedule-D leg. These later inputs are hard requirements because the # shipped validation provisions must bind. POST_REFERENCE_ECPS_REQUIRED_INPUTS = ( "fsla_overtime_premium", @@ -74,8 +75,25 @@ "roth_ira_contributions_desired", "self_employed_pension_contributions_desired", "meets_ssi_disability_criteria", + "schedule_d_capital_gain_distributions", ) +# Per-column annotations for post-reference hard requirements whose absence +# from today's artifacts is the intended red gate (the #368 SSI-asset +# pattern, extended by #462 to the capital-gain-distributions route). +POST_REFERENCE_COLUMN_NOTES = { + "schedule_d_capital_gain_distributions": ( + "Schedule D line 13 route leg of the #282 capital-gain-distributions " + "split (memo component of long_term_capital_gains, written by the " + "capital_gain_distributions source stage); required with NO reviewed " + "exclusion per PolicyEngine/populace#462 so a release whose export " + "drops the route (the Build M live default shipped it at $0 while " + "non_sch_d_capital_gains carried 7.3x its SOI target) fails the " + "coverage gate. Currently absent — this is the intended red gate " + "until the Build N rebuild carries the split through." + ), +} + QBI_INPUTS = ( "estate_income_would_be_qualified", "farm_operations_income_would_be_qualified", @@ -1263,6 +1281,8 @@ def build_manifest() -> dict: "gate fails until the asset stage is restored (Deliverable " "2). Currently absent — this is the intended red gate." ) + elif name in POST_REFERENCE_COLUMN_NOTES: + column["note"] = POST_REFERENCE_COLUMN_NOTES[name] columns[name] = column required = sorted(n for n, c in columns.items() if c["status"] == "required") @@ -1312,9 +1332,11 @@ def build_manifest() -> dict: "ecps_parity_reference.json populated layers, plus the documented " "post-reference fsla_overtime_premium, " "qualified_passenger_vehicle_loan_interest, five desired " - "retirement-contribution inputs, and " + "retirement-contribution inputs, " "meets_ssi_disability_criteria required by shipped validation " - "probes. " + "probes, and the #282 Schedule-D capital-gain-distributions " + "route leg schedule_d_capital_gain_distributions " + "(PolicyEngine/populace#462). " "status='reviewed_exclusion' for ecps_parity_known_gaps.json entries " "(reason+issue from that register); EXCEPT every primary-source " "restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS "