From 915e21860b4c7e67358dca54b8f680fec3d91477 Mon Sep 17 00:00:00 2001 From: Corrin Lakeland Date: Wed, 5 Aug 2026 20:02:42 +1200 Subject: [PATCH] Repair data that violates this project's own field contracts A full-database validation sweep (every row through full_clean()) found 31 rows whose values were never valid against the models as declared here. Django enforces choices and blank= in its validation layer, which the write paths that produced these rows did not run, so the database accepted them. purchasing/0009: deletes 12 purchase-order lines that hold nothing at all (blank description, qty 1, no cost, nothing received, no job, no Xero id, no item codes, no raw payload, price_tbc unset); describes the 5 blank- description lines that DO carry data, one of which has 2 units received at $119.50 against a job; and maps the single status 'void' -> 'deleted'. quoting/0004: unsets 13 mapped_metal_type values that were never valid choices ('unspecified' x10, 'steel' x2, 'tungsten' x1) and clears their parser_version so the parser re-derives them, skipping any row an operator has hand-validated. Verified by replaying both migrations against a restore of production inside a transaction: 12/5/1/13 rows affected, leaving zero blank descriptions, zero 'void' statuses and zero out-of-enum metal types. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_013wDoeUfmKL3AnXzpYToyPk --- ...0009_repair_blank_lines_and_void_status.py | 86 +++++++++++++++++++ .../0004_clear_invalid_metal_types.py | 65 ++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 apps/purchasing/migrations/0009_repair_blank_lines_and_void_status.py create mode 100644 apps/quoting/migrations/0004_clear_invalid_metal_types.py diff --git a/apps/purchasing/migrations/0009_repair_blank_lines_and_void_status.py b/apps/purchasing/migrations/0009_repair_blank_lines_and_void_status.py new file mode 100644 index 000000000..709b0286b --- /dev/null +++ b/apps/purchasing/migrations/0009_repair_blank_lines_and_void_status.py @@ -0,0 +1,86 @@ +"""Repair purchase-order rows that violate this project's own field contracts. + +Found by a full-database validation sweep (every row of every model through +`full_clean()`) run while preparing the v2 rewrite's data migration. Nothing +here is a v2 requirement: each row below fails validation against the models +as declared in THIS repository. The database never rejected them because +`CharField` choices and `blank` are enforced by Django's validation layer, +which the write paths that produced these rows did not run. + +Three repairs, in order: + +1. Twelve purchase-order lines that hold nothing at all — blank description, + quantity 1, no unit cost, nothing received, no job, no Xero line id, no + item codes, no metal/alloy, no dimensions/specifics/location, no raw + import payload, and price_tbc unset — are deleted. They are UI artefacts + ("add line" pressed, never filled) and contribute 0.00 to every total. + Verified against a production restore before writing: no Stock row + references any of them (Stock.source_purchase_order_line is the only + inbound foreign key, and it is SET_NULL regardless). + +2. The remaining blank-description lines carry real data and keep their + rows, gaining a marker description instead. One has two units received at + $119.50 against a job; another is flagged price_tbc; another is allocated + to a job. Deleting these would destroy genuine purchase records, so the + predicate in step 1 is deliberately conservative: any signal of human + intent or financial activity disqualifies a row from deletion. + +3. One purchase order carries status 'void', which has never appeared in + this model's choices. It becomes 'deleted', the choice that means the + same thing. + +Irreversible: reverse cannot tell a row this migration described from one +that was always described, nor resurrect a deleted row, so reverse is a +no-op rather than a wrong restore (house pattern: 0007_text_unset_is_null). +""" + +from django.db import migrations + +# A line is junk only if it carries no financial activity and no trace of +# human intent. Any single populated column keeps the row. +EMPTY_LINE = """ + description = '' + AND quantity = 1 + AND unit_cost IS NULL + AND (received_quantity IS NULL OR received_quantity = 0) + AND job_id IS NULL + AND xero_line_item_id IS NULL + AND item_code IS NULL + AND supplier_item_code IS NULL + AND metal_type IS NULL + AND alloy IS NULL + AND dimensions IS NULL + AND specifics IS NULL + AND location IS NULL + AND price_tbc = false + AND (raw_line_data IS NULL OR raw_line_data::text IN ('null', '{}')) +""" + +MARKER_DESCRIPTION = "Old PO line without a description" + + +class Migration(migrations.Migration): + dependencies = [ + ("purchasing", "0008_text_unset_constraints"), + ] + + operations = [ + migrations.RunSQL( + sql=f"DELETE FROM purchasing_purchaseorderline WHERE {EMPTY_LINE}", + reverse_sql=migrations.RunSQL.noop, + ), + migrations.RunSQL( + sql=( + "UPDATE purchasing_purchaseorderline " + f"SET description = '{MARKER_DESCRIPTION}' WHERE description = ''" + ), + reverse_sql=migrations.RunSQL.noop, + ), + migrations.RunSQL( + sql=( + "UPDATE purchasing_purchaseorder " + "SET status = 'deleted' WHERE status = 'void'" + ), + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/apps/quoting/migrations/0004_clear_invalid_metal_types.py b/apps/quoting/migrations/0004_clear_invalid_metal_types.py new file mode 100644 index 000000000..c53f880d7 --- /dev/null +++ b/apps/quoting/migrations/0004_clear_invalid_metal_types.py @@ -0,0 +1,65 @@ +"""Unset supplier-product metal types that were never valid choices. + +Found by the same full-database validation sweep as +purchasing/0009. Thirteen `ProductParsingMapping` rows hold a +`mapped_metal_type` that has never appeared in `MetalType.choices`: +'unspecified' (ten rows), 'steel' (two) and 'tungsten' (one). The LLM parser +wrote them as free text and nothing validated the result, so these rows have +been unreadable to any consumer that trusts the choices. + +They are set to NULL rather than remapped by hand. NULL is how this schema +spells "unset", and guessing would fabricate data: 'unspecified' means the +parser had no answer, a wire brush described as 'steel' is not a steel +product, and a tungsten TIG electrode has no home in the enum at all. + +`parser_version` is cleared alongside, which is the deliberate re-run lever — +the end-of-run fill selects rows whose parser version is not the current one, +so these thirteen are re-derived properly on the next run instead of being +frozen as unset. + +Rows an operator has hand-validated are excluded: their decision outranks the +parser (and this migration). All thirteen rows are currently unvalidated, so +the guard changes nothing today; it exists because this database keeps taking +writes until cutover and someone may validate one of these rows first. + +Irreversible: reverse cannot recover a value it deliberately discarded as +invalid, so it is a no-op rather than a wrong restore (house pattern: +purchasing/0007_text_unset_is_null). +""" + +from django.db import migrations + +# apps.job.enums.MetalType.values — inlined because a migration must not +# import app code, which is free to change after this migration is frozen. +VALID_METAL_TYPES = ( + "stainless_steel", + "mild_steel", + "aluminium", + "brass", + "copper", + "titanium", + "zinc", + "galvanized", + "other", +) + +_VALUE_LIST = ", ".join(f"'{value}'" for value in VALID_METAL_TYPES) + + +class Migration(migrations.Migration): + dependencies = [ + ("quoting", "0003_text_unset_is_null"), + ] + + operations = [ + migrations.RunSQL( + sql=( + "UPDATE quoting_productparsingmapping " + "SET mapped_metal_type = NULL, parser_version = NULL " + "WHERE mapped_metal_type IS NOT NULL " + f"AND mapped_metal_type NOT IN ({_VALUE_LIST}) " + "AND is_validated = false" + ), + reverse_sql=migrations.RunSQL.noop, + ), + ]