diff --git a/admin/views/associated_data.py b/admin/views/associated_data.py index a706d0ad1..f58dcd628 100644 --- a/admin/views/associated_data.py +++ b/admin/views/associated_data.py @@ -1,3 +1,31 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +AssociatedDataAdmin view for legacy NMA_AssociatedData. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_assoc_id: Legacy UUID PK (AssocID), UNIQUE for audit +- nma_location_id: Legacy LocationId UUID, UNIQUE +- nma_point_id: Legacy PointID string +- nma_object_id: Legacy OBJECTID, UNIQUE +""" + +from starlette.requests import Request + from admin.views.base import OcotilloModelView @@ -12,68 +40,74 @@ class AssociatedDataAdmin(OcotilloModelView): label = "NMA Associated Data" icon = "fa fa-link" - # Pagination - page_size = 50 - page_size_options = [25, 50, 100, 200] + # Integer PK + pk_attr = "id" + pk_type = int + + def can_create(self, request: Request) -> bool: + return False + + def can_edit(self, request: Request) -> bool: + return False + + def can_delete(self, request: Request) -> bool: + return False # ========== List View ========== + list_fields = [ - "location_id", - "point_id", - "assoc_id", + "id", + "nma_assoc_id", + "nma_location_id", + "nma_point_id", + "nma_object_id", "notes", "formation", - "object_id", "thing_id", ] sortable_fields = [ - "assoc_id", - "object_id", - "point_id", + "id", + "nma_assoc_id", + "nma_object_id", + "nma_point_id", ] - fields_default_sort = [("point_id", False), ("object_id", False)] + fields_default_sort = [("nma_point_id", False), ("nma_object_id", False)] searchable_fields = [ - "point_id", - "assoc_id", + "nma_point_id", + "nma_assoc_id", "notes", "formation", ] - # ========== Detail View ========== + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Form View ========== + fields = [ - "location_id", - "point_id", - "assoc_id", + "id", + "nma_assoc_id", + "nma_location_id", + "nma_point_id", + "nma_object_id", "notes", "formation", - "object_id", "thing_id", ] - # ========== Legacy Field Labels ========== field_labels = { - "location_id": "LocationId", - "point_id": "PointID", - "assoc_id": "AssocID", + "id": "ID", + "nma_assoc_id": "NMA AssocID (Legacy)", + "nma_location_id": "NMA LocationId (Legacy)", + "nma_point_id": "NMA PointID (Legacy)", + "nma_object_id": "NMA OBJECTID (Legacy)", "notes": "Notes", "formation": "Formation", - "object_id": "OBJECTID", - "thing_id": "ThingID", + "thing_id": "Thing ID", } - # ========== READ ONLY ========== - enable_publish_actions = ( - False # hides publish/unpublish actions inherited from base - ) - def can_create(self, request) -> bool: - return False - - def can_edit(self, request) -> bool: - return False - - def can_delete(self, request) -> bool: - return False +# ============= EOF ============================================= diff --git a/admin/views/chemistry_sampleinfo.py b/admin/views/chemistry_sampleinfo.py index f791e26ed..d2179d4ad 100644 --- a/admin/views/chemistry_sampleinfo.py +++ b/admin/views/chemistry_sampleinfo.py @@ -15,6 +15,17 @@ # =============================================================================== """ ChemistrySampleInfoAdmin view for legacy Chemistry_SampleInfo. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_sample_pt_id: Legacy UUID PK (SamplePtID), UNIQUE for audit +- nma_wclab_id: Legacy WCLab_ID +- nma_sample_point_id: Legacy SamplePointID +- nma_object_id: Legacy OBJECTID, UNIQUE +- nma_location_id: Legacy LocationId UUID (for audit trail) + +FK Change (2026-01): +- thing_id: Integer FK to Thing.id """ from admin.views.base import OcotilloModelView @@ -31,13 +42,18 @@ class ChemistrySampleInfoAdmin(OcotilloModelView): label = "Chemistry Sample Info" icon = "fa fa-flask" + # Integer PK + pk_attr = "id" + pk_type = int + # ========== List View ========== sortable_fields = [ - "sample_pt_id", - "object_id", - "sample_point_id", - "wclab_id", + "id", + "nma_sample_pt_id", + "nma_object_id", + "nma_sample_point_id", + "nma_wclab_id", "collection_date", "sample_type", "data_source", @@ -48,9 +64,9 @@ class ChemistrySampleInfoAdmin(OcotilloModelView): fields_default_sort = [("collection_date", True)] searchable_fields = [ - "sample_point_id", - "sample_pt_id", - "wclab_id", + "nma_sample_point_id", + "nma_sample_pt_id", + "nma_wclab_id", "collected_by", "analyses_agency", "sample_notes", @@ -70,10 +86,13 @@ class ChemistrySampleInfoAdmin(OcotilloModelView): # ========== Form View ========== fields = [ - "sample_pt_id", - "sample_point_id", - "object_id", - "wclab_id", + "id", + "nma_sample_pt_id", + "nma_sample_point_id", + "nma_object_id", + "nma_wclab_id", + "nma_location_id", + "thing_id", "collection_date", "collection_method", "collected_by", @@ -91,12 +110,38 @@ class ChemistrySampleInfoAdmin(OcotilloModelView): ] exclude_fields_from_create = [ - "object_id", + "id", + "nma_object_id", ] exclude_fields_from_edit = [ - "object_id", + "id", + "nma_object_id", ] + field_labels = { + "id": "ID", + "nma_sample_pt_id": "NMA SamplePtID (Legacy)", + "nma_sample_point_id": "NMA SamplePointID (Legacy)", + "nma_object_id": "NMA OBJECTID (Legacy)", + "nma_wclab_id": "NMA WCLab_ID (Legacy)", + "nma_location_id": "NMA LocationId (Legacy)", + "thing_id": "Thing ID", + "collection_date": "Collection Date", + "collection_method": "Collection Method", + "collected_by": "Collected By", + "analyses_agency": "Analyses Agency", + "sample_type": "Sample Type", + "sample_material_not_h2o": "Sample Material (Not H2O)", + "water_type": "Water Type", + "study_sample": "Study Sample", + "data_source": "Data Source", + "data_quality": "Data Quality", + "public_release": "Public Release", + "added_day_to_date": "Added Day to Date", + "added_month_day_to_date": "Added Month/Day to Date", + "sample_notes": "Sample Notes", + } + # ============= EOF ============================================= diff --git a/admin/views/field_parameters.py b/admin/views/field_parameters.py index 3c2c8aab1..5638370cc 100644 --- a/admin/views/field_parameters.py +++ b/admin/views/field_parameters.py @@ -15,6 +15,15 @@ # =============================================================================== """ FieldParametersAdmin view for legacy NMA_FieldParameters. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit +- nma_sample_point_id: Legacy SamplePointID string +- nma_object_id: Legacy OBJECTID +- nma_wclab_id: Legacy WCLab_ID """ from starlette.requests import Request @@ -33,6 +42,10 @@ class FieldParametersAdmin(OcotilloModelView): label = "NMA Field Parameters" icon = "fa fa-tachometer" + # Integer PK + pk_attr = "id" + pk_type = int + def can_create(self, request: Request) -> bool: return False @@ -45,42 +58,46 @@ def can_delete(self, request: Request) -> bool: # ========== List View ========== list_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", "field_parameter", "sample_value", "units", "notes", - "object_id", "analyses_agency", - "wc_lab_id", + "nma_wclab_id", + "nma_object_id", ] sortable_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", "field_parameter", "sample_value", "units", "notes", "analyses_agency", - "wc_lab_id", - "object_id", + "nma_wclab_id", + "nma_object_id", ] - fields_default_sort = [("sample_point_id", True)] + fields_default_sort = [("nma_sample_point_id", True)] searchable_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "nma_global_id", + "nma_sample_pt_id", + "nma_sample_point_id", "field_parameter", "units", "notes", "analyses_agency", - "wc_lab_id", + "nma_wclab_id", ] page_size = 50 @@ -89,29 +106,33 @@ def can_delete(self, request: Request) -> bool: # ========== Form View ========== fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", "field_parameter", "sample_value", "units", "notes", - "object_id", + "nma_object_id", "analyses_agency", - "wc_lab_id", + "nma_wclab_id", ] field_labels = { - "global_id": "GlobalID", - "sample_pt_id": "SamplePtID", - "sample_point_id": "SamplePointID", + "id": "ID", + "nma_global_id": "NMA GlobalID (Legacy)", + "chemistry_sample_info_id": "Chemistry Sample Info ID", + "nma_sample_pt_id": "NMA SamplePtID (Legacy)", + "nma_sample_point_id": "NMA SamplePointID (Legacy)", "field_parameter": "FieldParameter", "sample_value": "SampleValue", "units": "Units", "notes": "Notes", - "object_id": "OBJECTID", + "nma_object_id": "NMA OBJECTID (Legacy)", "analyses_agency": "AnalysesAgency", - "wc_lab_id": "WCLab_ID", + "nma_wclab_id": "NMA WCLab_ID (Legacy)", } diff --git a/admin/views/hydraulicsdata.py b/admin/views/hydraulicsdata.py index 5d2baa360..9723cbb38 100644 --- a/admin/views/hydraulicsdata.py +++ b/admin/views/hydraulicsdata.py @@ -15,9 +15,14 @@ # =============================================================================== """ HydraulicsDataAdmin view for legacy NMA_HydraulicsData. -""" -from starlette.requests import Request +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- nma_well_id: Legacy WellID UUID +- nma_point_id: Legacy PointID string +- nma_object_id: Legacy OBJECTID, UNIQUE +""" from admin.views.base import OcotilloModelView @@ -29,74 +34,55 @@ class HydraulicsDataAdmin(OcotilloModelView): # ========== Basic Configuration ========== - name = "NMA Hydraulics Data" - label = "NMA Hydraulics Data" + name = "Hydraulics Data" + label = "Hydraulics Data" icon = "fa fa-tint" - def can_create(self, request: Request) -> bool: - return False - - def can_edit(self, request: Request) -> bool: - return False + # Integer PK + pk_attr = "id" + pk_type = int - def can_delete(self, request: Request) -> bool: - return False + can_create = False + can_edit = False + can_delete = False # ========== List View ========== list_fields = [ - "global_id", - "well_id", - "point_id", - "data_source", + "id", + "nma_global_id", + "nma_well_id", + "nma_point_id", "thing_id", - "object_id", - "cs_gal_d_ft", - "hd_ft2_d", - "hl_day_1", - "kh_ft_d", - "kv_ft_d", - "p_decimal_fraction", - "s_dimensionless", - "ss_ft_1", - "sy_decimalfractn", - "t_ft2_d", - "k_darcy", - "test_bottom", - "test_top", "hydraulic_unit", "hydraulic_unit_type", - "hydraulic_remarks", + "test_top", + "test_bottom", + "t_ft2_d", + "k_darcy", + "data_source", + "nma_object_id", ] sortable_fields = [ - "global_id", - "well_id", - "point_id", - "data_source", + "id", + "nma_global_id", + "nma_well_id", + "nma_point_id", "thing_id", - "object_id", - "cs_gal_d_ft", - "hd_ft2_d", - "hl_day_1", - "kh_ft_d", - "kv_ft_d", - "p_decimal_fraction", - "s_dimensionless", - "ss_ft_1", - "sy_decimalfractn", - "t_ft2_d", - "k_darcy", - "test_bottom", - "test_top", "hydraulic_unit", "hydraulic_unit_type", - "hydraulic_remarks", + "test_top", + "test_bottom", + "t_ft2_d", + "k_darcy", + "data_source", + "nma_object_id", ] searchable_fields = [ - "global_id", - "point_id", + "nma_global_id", + "nma_point_id", "hydraulic_unit", "hydraulic_remarks", "data_source", @@ -108,44 +94,46 @@ def can_delete(self, request: Request) -> bool: # ========== Form View ========== fields = [ - "global_id", - "well_id", - "point_id", - "data_source", + "id", + "nma_global_id", + "nma_well_id", + "nma_point_id", "thing_id", - "object_id", - "cs_gal_d_ft", - "hd_ft2_d", - "hl_day_1", - "kh_ft_d", - "kv_ft_d", - "p_decimal_fraction", + "hydraulic_unit", + "hydraulic_unit_type", + "hydraulic_remarks", + "test_top", + "test_bottom", + "t_ft2_d", "s_dimensionless", "ss_ft_1", "sy_decimalfractn", - "t_ft2_d", + "kh_ft_d", + "kv_ft_d", + "hl_day_1", + "hd_ft2_d", + "cs_gal_d_ft", + "p_decimal_fraction", "k_darcy", - "test_bottom", - "test_top", - "hydraulic_unit", - "hydraulic_unit_type", - "hydraulic_remarks", + "data_source", + "nma_object_id", ] field_labels = { - "global_id": "GlobalID", - "well_id": "WellID", - "point_id": "PointID", + "id": "ID", + "nma_global_id": "NMA GlobalID (Legacy)", + "nma_well_id": "NMA WellID (Legacy)", + "nma_point_id": "NMA PointID (Legacy)", "thing_id": "Thing ID", - "hydraulic_unit": "Hydraulic Unit", - "hydraulic_unit_type": "HydraulicUnit Type", + "hydraulic_unit": "HydraulicUnit", + "hydraulic_unit_type": "HydraulicUnitType", "hydraulic_remarks": "Hydraulic Remarks", - "test_top": "Test Top", - "test_bottom": "Test Bottom", + "test_top": "TestTop", + "test_bottom": "TestBottom", "t_ft2_d": "T (ft2/d)", "s_dimensionless": "S (dimensionless)", "ss_ft_1": "Ss (ft-1)", - "sy_decimalfractn": "Sy (decimal fraction)", + "sy_decimalfractn": "Sy (decimalfractn)", "kh_ft_d": "KH (ft/d)", "kv_ft_d": "KV (ft/d)", "hl_day_1": "HL (day-1)", @@ -154,7 +142,7 @@ def can_delete(self, request: Request) -> bool: "p_decimal_fraction": "P (decimal fraction)", "k_darcy": "k (darcy)", "data_source": "Data Source", - "object_id": "OBJECTID", + "nma_object_id": "NMA OBJECTID (Legacy)", } diff --git a/admin/views/major_chemistry.py b/admin/views/major_chemistry.py index f822ed907..9578f60d1 100644 --- a/admin/views/major_chemistry.py +++ b/admin/views/major_chemistry.py @@ -15,9 +15,16 @@ # =============================================================================== """ MajorChemistryAdmin view for legacy NMA_MajorChemistry. -""" -import uuid +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit +- nma_sample_point_id: Legacy SamplePointID string +- nma_object_id: Legacy OBJECTID +- nma_wclab_id: Legacy WCLab_ID +""" from starlette.requests import Request from starlette_admin.fields import HasOne @@ -36,8 +43,10 @@ class MajorChemistryAdmin(OcotilloModelView): name = "NMA Major Chemistry" label = "NMA Major Chemistry" icon = "fa fa-flask" - pk_attr = "global_id" - pk_type = uuid.UUID + + # Integer PK + pk_attr = "id" + pk_type = int def can_create(self, request: Request) -> bool: return False @@ -51,9 +60,11 @@ def can_delete(self, request: Request) -> bool: # ========== List View ========== list_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", HasOne("chemistry_sample_info", identity="n-m-a_-chemistry_-sample-info"), "analyte", "symbol", @@ -65,15 +76,17 @@ def can_delete(self, request: Request) -> bool: "notes", "volume", "volume_unit", - "object_id", + "nma_object_id", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] sortable_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", "analyte", "symbol", "sample_value", @@ -84,23 +97,23 @@ def can_delete(self, request: Request) -> bool: "notes", "volume", "volume_unit", - "object_id", + "nma_object_id", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] fields_default_sort = [("analysis_date", True)] searchable_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "nma_global_id", + "nma_sample_pt_id", + "nma_sample_point_id", "analyte", "symbol", "analysis_method", "notes", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] page_size = 50 @@ -109,9 +122,11 @@ def can_delete(self, request: Request) -> bool: # ========== Form View ========== fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", HasOne("chemistry_sample_info", identity="n-m-a_-chemistry_-sample-info"), "analyte", "symbol", @@ -123,15 +138,17 @@ def can_delete(self, request: Request) -> bool: "notes", "volume", "volume_unit", - "object_id", + "nma_object_id", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] field_labels = { - "global_id": "GlobalID", - "sample_pt_id": "SamplePtID", - "sample_point_id": "SamplePointID", + "id": "ID", + "nma_global_id": "NMA GlobalID (Legacy)", + "chemistry_sample_info_id": "Chemistry Sample Info ID", + "nma_sample_pt_id": "NMA SamplePtID (Legacy)", + "nma_sample_point_id": "NMA SamplePointID (Legacy)", "chemistry_sample_info": "Chemistry Sample Info", "analyte": "Analyte", "symbol": "Symbol", @@ -143,9 +160,9 @@ def can_delete(self, request: Request) -> bool: "notes": "Notes", "volume": "Volume", "volume_unit": "Volume Unit", - "object_id": "OBJECTID", + "nma_object_id": "NMA OBJECTID (Legacy)", "analyses_agency": "Analyses Agency", - "wclab_id": "WCLab_ID", + "nma_wclab_id": "NMA WCLab_ID (Legacy)", } diff --git a/admin/views/minor_trace_chemistry.py b/admin/views/minor_trace_chemistry.py index 84f02bfdf..0c51e609e 100644 --- a/admin/views/minor_trace_chemistry.py +++ b/admin/views/minor_trace_chemistry.py @@ -15,9 +15,13 @@ # =============================================================================== """ MinorTraceChemistryAdmin view for legacy NMA_MinorTraceChemistry. -""" -import uuid +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_chemistry_sample_info_uuid: Legacy UUID FK for audit +""" from starlette.requests import Request from starlette_admin.fields import HasOne @@ -33,11 +37,13 @@ class MinorTraceChemistryAdmin(OcotilloModelView): # ========== Basic Configuration ========== identity = "n-m-a_-minor-trace-chemistry" - name = "NMA Minor Trace Chemistry" - label = "NMA Minor Trace Chemistry" + name = "Minor Trace Chemistry" + label = "Minor Trace Chemistry" icon = "fa fa-flask" - pk_attr = "global_id" - pk_type = uuid.UUID + + # Integer PK + pk_attr = "id" + pk_type = int def can_create(self, request: Request) -> bool: return False @@ -51,47 +57,39 @@ def can_delete(self, request: Request) -> bool: # ========== List View ========== list_fields = [ - "global_id", + "id", + "nma_global_id", HasOne("chemistry_sample_info", identity="n-m-a_-chemistry_-sample-info"), - "sample_pt_id", - "sample_point_id", + "nma_chemistry_sample_info_uuid", "analyte", "sample_value", "units", "symbol", "analysis_date", "analyses_agency", - "wclab_id", - "object_id", ] sortable_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", "analyte", "sample_value", "units", "symbol", "analysis_date", "analyses_agency", - "wclab_id", - "object_id", ] fields_default_sort = [("analysis_date", True)] searchable_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "nma_global_id", "analyte", "symbol", "analysis_method", - "analysis_date", "notes", "analyses_agency", - "wclab_id", ] page_size = 50 @@ -100,10 +98,10 @@ def can_delete(self, request: Request) -> bool: # ========== Form View ========== fields = [ - "global_id", + "id", + "nma_global_id", HasOne("chemistry_sample_info", identity="n-m-a_-chemistry_-sample-info"), - "sample_pt_id", - "sample_point_id", + "nma_chemistry_sample_info_uuid", "analyte", "symbol", "sample_value", @@ -114,29 +112,26 @@ def can_delete(self, request: Request) -> bool: "notes", "volume", "volume_unit", - "object_id", "analyses_agency", - "wclab_id", ] field_labels = { - "global_id": "GlobalID", + "id": "ID", + "nma_global_id": "NMA GlobalID (Legacy)", "chemistry_sample_info": "Chemistry Sample Info", - "sample_pt_id": "SamplePtID", - "sample_point_id": "SamplePointID", + "chemistry_sample_info_id": "Chemistry Sample Info ID", + "nma_chemistry_sample_info_uuid": "NMA Chemistry Sample Info UUID (Legacy)", "analyte": "Analyte", "symbol": "Symbol", - "sample_value": "SampleValue", + "sample_value": "Sample Value", "units": "Units", "uncertainty": "Uncertainty", - "analysis_method": "AnalysisMethod", - "analysis_date": "AnalysisDate", + "analysis_method": "Analysis Method", + "analysis_date": "Analysis Date", "notes": "Notes", "volume": "Volume", - "volume_unit": "VolumeUnit", - "object_id": "OBJECTID", - "analyses_agency": "AnalysesAgency", - "wclab_id": "WCLab_ID", + "volume_unit": "Volume Unit", + "analyses_agency": "Analyses Agency", } diff --git a/admin/views/radionuclides.py b/admin/views/radionuclides.py index f78099037..f1bd27992 100644 --- a/admin/views/radionuclides.py +++ b/admin/views/radionuclides.py @@ -15,6 +15,15 @@ # =============================================================================== """ RadionuclidesAdmin view for legacy NMA_Radionuclides. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit +- nma_sample_point_id: Legacy SamplePointID string +- nma_object_id: Legacy OBJECTID, UNIQUE +- nma_wclab_id: Legacy WCLab_ID """ from starlette.requests import Request @@ -33,6 +42,10 @@ class RadionuclidesAdmin(OcotilloModelView): label = "NMA Radionuclides" icon = "fa fa-radiation" + # Integer PK + pk_attr = "id" + pk_type = int + def can_create(self, request: Request) -> bool: return False @@ -45,9 +58,12 @@ def can_delete(self, request: Request) -> bool: # ========== List View ========== list_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", + "thing_id", "analyte", "symbol", "sample_value", @@ -58,15 +74,18 @@ def can_delete(self, request: Request) -> bool: "notes", "volume", "volume_unit", - "object_id", + "nma_object_id", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] sortable_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", + "thing_id", "analyte", "symbol", "sample_value", @@ -77,24 +96,24 @@ def can_delete(self, request: Request) -> bool: "notes", "volume", "volume_unit", - "object_id", + "nma_object_id", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] fields_default_sort = [("analysis_date", True)] searchable_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "nma_global_id", + "nma_sample_pt_id", + "nma_sample_point_id", "analyte", "symbol", "analysis_method", "analysis_date", "notes", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] page_size = 50 @@ -103,9 +122,12 @@ def can_delete(self, request: Request) -> bool: # ========== Form View ========== fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", + "thing_id", "analyte", "symbol", "sample_value", @@ -116,15 +138,18 @@ def can_delete(self, request: Request) -> bool: "notes", "volume", "volume_unit", - "object_id", + "nma_object_id", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] field_labels = { - "global_id": "GlobalID", - "sample_pt_id": "SamplePtID", - "sample_point_id": "Sample PointID", + "id": "ID", + "nma_global_id": "NMA GlobalID (Legacy)", + "chemistry_sample_info_id": "Chemistry Sample Info ID", + "nma_sample_pt_id": "NMA SamplePtID (Legacy)", + "nma_sample_point_id": "NMA SamplePointID (Legacy)", + "thing_id": "Thing ID", "analyte": "Analyte", "symbol": "Symbol", "sample_value": "Sample Value", @@ -135,9 +160,9 @@ def can_delete(self, request: Request) -> bool: "notes": "Notes", "volume": "Volume", "volume_unit": "Volume Unit", - "object_id": "OBJECTID", + "nma_object_id": "NMA OBJECTID (Legacy)", "analyses_agency": "Analyses Agency", - "wclab_id": "WCLab_ID", + "nma_wclab_id": "NMA WCLab_ID (Legacy)", } diff --git a/admin/views/soil_rock_results.py b/admin/views/soil_rock_results.py index 00786058e..947804980 100644 --- a/admin/views/soil_rock_results.py +++ b/admin/views/soil_rock_results.py @@ -1,5 +1,8 @@ """ SoilRockResultsAdmin view for legacy NMA_Soil_Rock_Results. + +Already has Integer PK. Updated for legacy column rename: +- point_id -> nma_point_id """ from admin.views.base import OcotilloModelView @@ -15,6 +18,10 @@ class SoilRockResultsAdmin(OcotilloModelView): label = "NMA Soil Rock Results" icon = "fa fa-mountain" + # Integer PK (already correct) + pk_attr = "id" + pk_type = int + # Pagination page_size = 50 page_size_options = [25, 50, 100, 200] @@ -22,7 +29,7 @@ class SoilRockResultsAdmin(OcotilloModelView): # ========== List View ========== list_fields = [ "id", - "point_id", + "nma_point_id", "sample_type", "date_sampled", "d13c", @@ -33,11 +40,11 @@ class SoilRockResultsAdmin(OcotilloModelView): sortable_fields = [ "id", - "point_id", + "nma_point_id", ] searchable_fields = [ - "point_id", + "nma_point_id", "sample_type", "date_sampled", "sampled_by", @@ -48,7 +55,7 @@ class SoilRockResultsAdmin(OcotilloModelView): # ========== Detail View ========== fields = [ "id", - "point_id", + "nma_point_id", "sample_type", "date_sampled", "d13c", @@ -59,8 +66,8 @@ class SoilRockResultsAdmin(OcotilloModelView): # ========== Legacy Field Labels ========== field_labels = { - "id": "id", - "point_id": "Point_ID", + "id": "ID", + "nma_point_id": "NMA Point_ID (Legacy)", "sample_type": "Sample Type", "date_sampled": "Date Sampled", "d13c": "d13C", diff --git a/admin/views/stratigraphy.py b/admin/views/stratigraphy.py index 9f2526f08..0bbd32231 100644 --- a/admin/views/stratigraphy.py +++ b/admin/views/stratigraphy.py @@ -1,5 +1,12 @@ """ StratigraphyAdmin view for legacy stratigraphy. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- nma_well_id: Legacy WellID UUID +- nma_point_id: Legacy PointID string +- nma_object_id: Legacy OBJECTID, UNIQUE """ from admin.views.base import OcotilloModelView @@ -15,6 +22,10 @@ class StratigraphyAdmin(OcotilloModelView): label = "NMA Stratigraphy" icon = "fa fa-layer-group" + # Integer PK + pk_attr = "id" + pk_type = int + # Pagination page_size = 50 page_size_options = [25, 50, 100, 200] @@ -22,16 +33,17 @@ class StratigraphyAdmin(OcotilloModelView): # ========== List View ========== sortable_fields = [ - "global_id", - "object_id", - "point_id", + "id", + "nma_global_id", + "nma_object_id", + "nma_point_id", ] - fields_default_sort = [("point_id", False), ("strat_top", False)] + fields_default_sort = [("nma_point_id", False), ("strat_top", False)] searchable_fields = [ - "point_id", - "global_id", + "nma_point_id", + "nma_global_id", "unit_identifier", "lithology", "lithologic_modifier", @@ -43,9 +55,10 @@ class StratigraphyAdmin(OcotilloModelView): # ========== Form View ========== fields = [ - "global_id", - "well_id", - "point_id", + "id", + "nma_global_id", + "nma_well_id", + "nma_point_id", "thing_id", "strat_top", "strat_bottom", @@ -55,22 +68,25 @@ class StratigraphyAdmin(OcotilloModelView): "contributing_unit", "strat_source", "strat_notes", - "object_id", + "nma_object_id", ] exclude_fields_from_create = [ - "object_id", + "id", + "nma_object_id", ] exclude_fields_from_edit = [ - "object_id", + "id", + "nma_object_id", ] # ========== Legacy Field Labels ========== field_labels = { - "global_id": "GlobalID", - "well_id": "WellID", - "point_id": "PointID", + "id": "ID", + "nma_global_id": "NMA GlobalID (Legacy)", + "nma_well_id": "NMA WellID (Legacy)", + "nma_point_id": "NMA PointID (Legacy)", "thing_id": "ThingID", "strat_top": "StratTop", "strat_bottom": "StratBottom", @@ -80,5 +96,5 @@ class StratigraphyAdmin(OcotilloModelView): "contributing_unit": "ContributingUnit", "strat_source": "StratSource", "strat_notes": "StratNotes", - "object_id": "OBJECTID", + "nma_object_id": "NMA OBJECTID (Legacy)", } diff --git a/alembic/env.py b/alembic/env.py index 089144e88..526711ae9 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -71,6 +71,9 @@ def build_database_url(): def include_object(object, name, type_, reflected, compare_to): # only include tables in sql alchemy model, not auto-generated tables from PostGIS or TIGER + # Handle None names for unnamed constraints + if name is None: + return True if type_ == "table" or name.endswith("_version") or name == "transaction": return name in model_tables return True diff --git a/alembic/versions/3a9c1f5b7d2e_align_nma_minor_trace_columns.py b/alembic/versions/3a9c1f5b7d2e_align_nma_minor_trace_columns.py index b2ceb077e..6d2507693 100644 --- a/alembic/versions/3a9c1f5b7d2e_align_nma_minor_trace_columns.py +++ b/alembic/versions/3a9c1f5b7d2e_align_nma_minor_trace_columns.py @@ -3,14 +3,14 @@ Revision ID: 3a9c1f5b7d2e Revises: c1d2e3f4a5b6 Create Date: 2026-01-31 12:00:00.000000 + +NOTE: This migration is now a no-op because the Integer PK refactor +(migration 3cb924ca51fd) handles all column changes for NMA tables. +This migration exists only to preserve the alembic revision chain. """ from typing import Sequence, Union -from alembic import op -import sqlalchemy as sa -from sqlalchemy import inspect - # revision identifiers, used by Alembic. revision: str = "3a9c1f5b7d2e" down_revision: Union[str, Sequence[str], None] = "c1d2e3f4a5b6" @@ -18,117 +18,11 @@ depends_on: Union[str, Sequence[str], None] = None -def _column_names(inspector, table_name: str) -> set[str]: - return {col["name"] for col in inspector.get_columns(table_name)} - - def upgrade() -> None: - """Rename legacy columns and add missing fields.""" - bind = op.get_bind() - inspector = inspect(bind) - if not inspector.has_table("NMA_MinorTraceChemistry"): - return - - table_name = "NMA_MinorTraceChemistry" - columns = _column_names(inspector, table_name) - - rename_map = { - "chemistry_sample_info_id": "SamplePtID", - "sample_point_id": "SamplePointID", - "analyte": "Analyte", - "sample_value": "SampleValue", - "units": "Units", - "symbol": "Symbol", - "analysis_method": "AnalysisMethod", - "analysis_date": "AnalysisDate", - "notes": "Notes", - "analyses_agency": "AnalysesAgency", - "uncertainty": "Uncertainty", - "volume": "Volume", - "volume_unit": "VolumeUnit", - } - - for old_name, new_name in rename_map.items(): - if old_name in columns and new_name not in columns: - op.alter_column(table_name, old_name, new_column_name=new_name) - columns.remove(old_name) - columns.add(new_name) - - if "SamplePointID" not in columns: - op.add_column( - table_name, sa.Column("SamplePointID", sa.String(length=10), nullable=True) - ) - if "OBJECTID" not in columns: - op.add_column(table_name, sa.Column("OBJECTID", sa.Integer(), nullable=True)) - if "WCLab_ID" not in columns: - op.add_column( - table_name, sa.Column("WCLab_ID", sa.String(length=25), nullable=True) - ) - - unique_constraints = inspector.get_unique_constraints(table_name) - unique_columns = {tuple(uc.get("column_names") or []) for uc in unique_constraints} - unique_names = {uc.get("name") for uc in unique_constraints} - - if ( - ("OBJECTID",) not in unique_columns - and "uq_nma_minor_trace_chemistry_objectid" not in unique_names - ): - op.create_unique_constraint( - "uq_nma_minor_trace_chemistry_objectid", - table_name, - ["OBJECTID"], - ) - - if "uq_minor_trace_chemistry_sample_analyte" not in unique_names: - op.create_unique_constraint( - "uq_minor_trace_chemistry_sample_analyte", - table_name, - ["SamplePtID", "Analyte"], - ) + """No-op: schema changes handled by Integer PK refactor migration.""" + pass def downgrade() -> None: - """Revert column names and remove added fields.""" - bind = op.get_bind() - inspector = inspect(bind) - if not inspector.has_table("NMA_MinorTraceChemistry"): - return - - table_name = "NMA_MinorTraceChemistry" - columns = _column_names(inspector, table_name) - - unique_constraints = inspector.get_unique_constraints(table_name) - unique_names = {uc.get("name") for uc in unique_constraints} - - if "uq_nma_minor_trace_chemistry_objectid" in unique_names: - op.drop_constraint( - "uq_nma_minor_trace_chemistry_objectid", - table_name, - type_="unique", - ) - - for column_name in ("WCLab_ID", "OBJECTID", "SamplePointID"): - if column_name in columns: - op.drop_column(table_name, column_name) - - rename_map = { - "SamplePtID": "chemistry_sample_info_id", - "Analyte": "analyte", - "SampleValue": "sample_value", - "Units": "units", - "Symbol": "symbol", - "AnalysisMethod": "analysis_method", - "AnalysisDate": "analysis_date", - "Notes": "notes", - "AnalysesAgency": "analyses_agency", - "Uncertainty": "uncertainty", - "Volume": "volume", - "VolumeUnit": "volume_unit", - } - - columns = _column_names(inspector, table_name) - for old_name, new_name in rename_map.items(): - if old_name in columns and new_name not in columns: - op.alter_column(table_name, old_name, new_column_name=new_name) - columns.remove(old_name) - columns.add(new_name) + """No-op: schema changes handled by Integer PK refactor migration.""" + pass diff --git a/alembic/versions/3cb924ca51fd_refactor_nma_tables_to_integer_pks.py b/alembic/versions/3cb924ca51fd_refactor_nma_tables_to_integer_pks.py new file mode 100644 index 000000000..fdfb8c55e --- /dev/null +++ b/alembic/versions/3cb924ca51fd_refactor_nma_tables_to_integer_pks.py @@ -0,0 +1,1099 @@ +"""refactor_nma_tables_to_integer_pks + +Revision ID: 3cb924ca51fd +Revises: 76e3ae8b99cb +Create Date: 2026-01-28 01:37:56.509497 + +""" + +from typing import Sequence, Union + +from alembic import op +import geoalchemy2 +import sqlalchemy as sa +import sqlalchemy_utils +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "3cb924ca51fd" +down_revision: Union[str, Sequence[str], None] = "76e3ae8b99cb" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema. + + Refactor NMA legacy tables from UUID to Integer primary keys: + - Add id (Integer PK with IDENTITY) to 8 NMA tables + - Rename UUID columns with nma_ prefix for audit + - Convert FK references from UUID to Integer + - Make chemistry_sample_info_id NOT NULL for chemistry child tables + """ + # ========================================================================== + # PHASE 1: Drop ALL foreign keys that reference NMA_Chemistry_SampleInfo.SamplePtID + # This must happen BEFORE we can modify NMA_Chemistry_SampleInfo + # ========================================================================== + op.drop_constraint( + op.f("NMA_MinorTraceChemistry_chemistry_sample_info_id_fkey"), + "NMA_MinorTraceChemistry", + type_="foreignkey", + ) + op.drop_constraint( + op.f("NMA_Radionuclides_SamplePtID_fkey"), + "NMA_Radionuclides", + type_="foreignkey", + ) + op.drop_constraint( + op.f("NMA_MajorChemistry_SamplePtID_fkey"), + "NMA_MajorChemistry", + type_="foreignkey", + ) + op.drop_constraint( + op.f("NMA_FieldParameters_SamplePtID_fkey"), + "NMA_FieldParameters", + type_="foreignkey", + ) + + # ========================================================================== + # PHASE 2: Modify NMA_Chemistry_SampleInfo (parent table) + # ========================================================================== + # Add new columns first + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column( + "id", sa.Integer(), sa.Identity(always=False, start=1), nullable=False + ), + ) + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column("nma_SamplePtID", sa.UUID(), nullable=True), + ) + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column("nma_WCLab_ID", sa.String(length=18), nullable=True), + ) + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column("nma_SamplePointID", sa.String(length=10), nullable=False), + ) + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column("nma_OBJECTID", sa.Integer(), nullable=True), + ) + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column("nma_LocationId", sa.UUID(), nullable=True), + ) + + # Drop old PK and create new PK on id + op.drop_constraint( + "NMA_Chemistry_SampleInfo_pkey", "NMA_Chemistry_SampleInfo", type_="primary" + ) + op.create_primary_key( + "NMA_Chemistry_SampleInfo_pkey", "NMA_Chemistry_SampleInfo", ["id"] + ) + + op.drop_constraint( + op.f("NMA_Chemistry_SampleInfo_OBJECTID_key"), + "NMA_Chemistry_SampleInfo", + type_="unique", + ) + op.create_unique_constraint(None, "NMA_Chemistry_SampleInfo", ["nma_SamplePtID"]) + op.create_unique_constraint(None, "NMA_Chemistry_SampleInfo", ["nma_OBJECTID"]) + op.drop_column("NMA_Chemistry_SampleInfo", "SamplePointID") + op.drop_column("NMA_Chemistry_SampleInfo", "SamplePtID") + op.drop_column("NMA_Chemistry_SampleInfo", "WCLab_ID") + op.drop_column("NMA_Chemistry_SampleInfo", "OBJECTID") + op.drop_column("NMA_Chemistry_SampleInfo", "LocationId") + + # ========================================================================== + # PHASE 3: Modify child tables and create new FKs pointing to NMA_Chemistry_SampleInfo.id + # ========================================================================== + + # --- NMA_FieldParameters --- + op.add_column( + "NMA_FieldParameters", + sa.Column( + "id", sa.Integer(), sa.Identity(always=False, start=1), nullable=False + ), + ) + op.add_column( + "NMA_FieldParameters", sa.Column("nma_GlobalID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_FieldParameters", + sa.Column("chemistry_sample_info_id", sa.Integer(), nullable=False), + ) + op.add_column( + "NMA_FieldParameters", sa.Column("nma_SamplePtID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_FieldParameters", + sa.Column("nma_SamplePointID", sa.String(length=10), nullable=True), + ) + op.add_column( + "NMA_FieldParameters", sa.Column("nma_OBJECTID", sa.Integer(), nullable=True) + ) + op.add_column( + "NMA_FieldParameters", + sa.Column("nma_WCLab_ID", sa.String(length=25), nullable=True), + ) + op.drop_index(op.f("FieldParameters$GlobalID"), table_name="NMA_FieldParameters") + op.drop_index(op.f("FieldParameters$OBJECTID"), table_name="NMA_FieldParameters") + op.drop_index( + op.f("FieldParameters$SamplePointID"), table_name="NMA_FieldParameters" + ) + op.drop_index(op.f("FieldParameters$SamplePtID"), table_name="NMA_FieldParameters") + op.drop_index(op.f("FieldParameters$WCLab_ID"), table_name="NMA_FieldParameters") + op.drop_index( + op.f("FieldParameters$ChemistrySampleInfoFieldParameters"), + table_name="NMA_FieldParameters", + ) + op.create_index( + "FieldParameters$ChemistrySampleInfoFieldParameters", + "NMA_FieldParameters", + ["chemistry_sample_info_id"], + unique=False, + ) + op.create_index( + "FieldParameters$nma_GlobalID", + "NMA_FieldParameters", + ["nma_GlobalID"], + unique=True, + ) + op.create_index( + "FieldParameters$nma_OBJECTID", + "NMA_FieldParameters", + ["nma_OBJECTID"], + unique=True, + ) + op.create_index( + "FieldParameters$nma_SamplePointID", + "NMA_FieldParameters", + ["nma_SamplePointID"], + unique=False, + ) + op.create_index( + "FieldParameters$nma_WCLab_ID", + "NMA_FieldParameters", + ["nma_WCLab_ID"], + unique=False, + ) + op.create_unique_constraint(None, "NMA_FieldParameters", ["nma_GlobalID"]) + op.create_foreign_key( + None, + "NMA_FieldParameters", + "NMA_Chemistry_SampleInfo", + ["chemistry_sample_info_id"], + ["id"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + op.drop_column("NMA_FieldParameters", "SamplePointID") + op.drop_column("NMA_FieldParameters", "SamplePtID") + op.drop_column("NMA_FieldParameters", "WCLab_ID") + op.drop_column("NMA_FieldParameters", "OBJECTID") + op.drop_column("NMA_FieldParameters", "GlobalID") + + # --- NMA_AssociatedData --- + op.add_column( + "NMA_AssociatedData", + sa.Column( + "id", sa.Integer(), sa.Identity(always=False, start=1), nullable=False + ), + ) + op.add_column( + "NMA_AssociatedData", sa.Column("nma_AssocID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_AssociatedData", sa.Column("nma_LocationId", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_AssociatedData", + sa.Column("nma_PointID", sa.String(length=10), nullable=True), + ) + op.add_column( + "NMA_AssociatedData", sa.Column("nma_OBJECTID", sa.Integer(), nullable=True) + ) + op.drop_constraint( + op.f("AssociatedData$LocationId"), "NMA_AssociatedData", type_="unique" + ) + op.drop_index(op.f("AssociatedData$PointID"), table_name="NMA_AssociatedData") + op.drop_constraint( + op.f("NMA_AssociatedData_OBJECTID_key"), "NMA_AssociatedData", type_="unique" + ) + op.create_unique_constraint(None, "NMA_AssociatedData", ["nma_LocationId"]) + op.create_unique_constraint(None, "NMA_AssociatedData", ["nma_AssocID"]) + op.create_unique_constraint(None, "NMA_AssociatedData", ["nma_OBJECTID"]) + op.drop_column("NMA_AssociatedData", "OBJECTID") + op.drop_column("NMA_AssociatedData", "LocationId") + op.drop_column("NMA_AssociatedData", "AssocID") + op.drop_column("NMA_AssociatedData", "PointID") + + # --- NMA_HydraulicsData --- + op.add_column( + "NMA_HydraulicsData", + sa.Column( + "id", sa.Integer(), sa.Identity(always=False, start=1), nullable=False + ), + ) + op.add_column( + "NMA_HydraulicsData", sa.Column("nma_GlobalID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_HydraulicsData", sa.Column("nma_WellID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_HydraulicsData", + sa.Column("nma_PointID", sa.String(length=50), nullable=True), + ) + op.add_column( + "NMA_HydraulicsData", sa.Column("nma_OBJECTID", sa.Integer(), nullable=True) + ) + op.drop_index( + op.f("ix_nma_hydraulicsdata_objectid"), table_name="NMA_HydraulicsData" + ) + op.drop_index( + op.f("ix_nma_hydraulicsdata_pointid"), table_name="NMA_HydraulicsData" + ) + op.drop_index(op.f("ix_nma_hydraulicsdata_wellid"), table_name="NMA_HydraulicsData") + op.create_unique_constraint(None, "NMA_HydraulicsData", ["nma_GlobalID"]) + op.create_unique_constraint(None, "NMA_HydraulicsData", ["nma_OBJECTID"]) + op.drop_column("NMA_HydraulicsData", "WellID") + op.drop_column("NMA_HydraulicsData", "OBJECTID") + op.drop_column("NMA_HydraulicsData", "PointID") + op.drop_column("NMA_HydraulicsData", "GlobalID") + + # --- NMA_MajorChemistry --- + op.add_column( + "NMA_MajorChemistry", + sa.Column( + "id", sa.Integer(), sa.Identity(always=False, start=1), nullable=False + ), + ) + op.add_column( + "NMA_MajorChemistry", sa.Column("nma_GlobalID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_MajorChemistry", + sa.Column("chemistry_sample_info_id", sa.Integer(), nullable=False), + ) + op.add_column( + "NMA_MajorChemistry", sa.Column("nma_SamplePtID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_MajorChemistry", + sa.Column("nma_SamplePointID", sa.String(length=10), nullable=True), + ) + op.add_column( + "NMA_MajorChemistry", sa.Column("nma_OBJECTID", sa.Integer(), nullable=True) + ) + op.add_column( + "NMA_MajorChemistry", + sa.Column("nma_WCLab_ID", sa.String(length=25), nullable=True), + ) + op.drop_index( + op.f("MajorChemistry$AnalysesAgency"), table_name="NMA_MajorChemistry" + ) + op.drop_index(op.f("MajorChemistry$Analyte"), table_name="NMA_MajorChemistry") + op.drop_index( + op.f("MajorChemistry$Chemistry SampleInfoMajorChemistry"), + table_name="NMA_MajorChemistry", + ) + op.drop_index(op.f("MajorChemistry$SamplePointID"), table_name="NMA_MajorChemistry") + op.drop_index( + op.f("MajorChemistry$SamplePointIDAnalyte"), table_name="NMA_MajorChemistry" + ) + op.drop_index(op.f("MajorChemistry$SamplePtID"), table_name="NMA_MajorChemistry") + op.drop_index(op.f("MajorChemistry$WCLab_ID"), table_name="NMA_MajorChemistry") + op.drop_constraint( + op.f("NMA_MajorChemistry_OBJECTID_key"), "NMA_MajorChemistry", type_="unique" + ) + op.create_unique_constraint(None, "NMA_MajorChemistry", ["nma_GlobalID"]) + op.create_unique_constraint(None, "NMA_MajorChemistry", ["nma_OBJECTID"]) + op.create_foreign_key( + None, + "NMA_MajorChemistry", + "NMA_Chemistry_SampleInfo", + ["chemistry_sample_info_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("NMA_MajorChemistry", "SamplePointID") + op.drop_column("NMA_MajorChemistry", "SamplePtID") + op.drop_column("NMA_MajorChemistry", "WCLab_ID") + op.drop_column("NMA_MajorChemistry", "OBJECTID") + op.drop_column("NMA_MajorChemistry", "GlobalID") + + # --- NMA_MinorTraceChemistry --- + op.add_column( + "NMA_MinorTraceChemistry", + sa.Column( + "id", sa.Integer(), sa.Identity(always=False, start=1), nullable=False + ), + ) + op.add_column( + "NMA_MinorTraceChemistry", sa.Column("nma_GlobalID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_MinorTraceChemistry", + sa.Column("nma_chemistry_sample_info_uuid", sa.UUID(), nullable=True), + ) + op.alter_column( + "NMA_MinorTraceChemistry", + "chemistry_sample_info_id", + existing_type=sa.UUID(), + type_=sa.Integer(), + nullable=False, + postgresql_using="NULL", + ) + op.create_unique_constraint(None, "NMA_MinorTraceChemistry", ["nma_GlobalID"]) + op.create_foreign_key( + None, + "NMA_MinorTraceChemistry", + "NMA_Chemistry_SampleInfo", + ["chemistry_sample_info_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("NMA_MinorTraceChemistry", "GlobalID") + + # --- NMA_Radionuclides --- + op.add_column( + "NMA_Radionuclides", + sa.Column( + "id", sa.Integer(), sa.Identity(always=False, start=1), nullable=False + ), + ) + op.add_column( + "NMA_Radionuclides", sa.Column("nma_GlobalID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_Radionuclides", + sa.Column("chemistry_sample_info_id", sa.Integer(), nullable=False), + ) + op.add_column( + "NMA_Radionuclides", sa.Column("nma_SamplePtID", sa.UUID(), nullable=True) + ) + op.add_column( + "NMA_Radionuclides", + sa.Column("nma_SamplePointID", sa.String(length=10), nullable=True), + ) + op.add_column( + "NMA_Radionuclides", sa.Column("nma_OBJECTID", sa.Integer(), nullable=True) + ) + op.add_column( + "NMA_Radionuclides", + sa.Column("nma_WCLab_ID", sa.String(length=25), nullable=True), + ) + op.drop_constraint( + op.f("NMA_Radionuclides_OBJECTID_key"), "NMA_Radionuclides", type_="unique" + ) + op.drop_index(op.f("Radionuclides$AnalysesAgency"), table_name="NMA_Radionuclides") + op.drop_index(op.f("Radionuclides$Analyte"), table_name="NMA_Radionuclides") + op.drop_index( + op.f("Radionuclides$Chemistry SampleInfoRadionuclides"), + table_name="NMA_Radionuclides", + ) + op.drop_index(op.f("Radionuclides$SamplePointID"), table_name="NMA_Radionuclides") + op.drop_index(op.f("Radionuclides$SamplePtID"), table_name="NMA_Radionuclides") + op.drop_index(op.f("Radionuclides$WCLab_ID"), table_name="NMA_Radionuclides") + op.create_unique_constraint(None, "NMA_Radionuclides", ["nma_GlobalID"]) + op.create_unique_constraint(None, "NMA_Radionuclides", ["nma_OBJECTID"]) + op.create_foreign_key( + None, + "NMA_Radionuclides", + "NMA_Chemistry_SampleInfo", + ["chemistry_sample_info_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("NMA_Radionuclides", "SamplePointID") + op.drop_column("NMA_Radionuclides", "SamplePtID") + op.drop_column("NMA_Radionuclides", "WCLab_ID") + op.drop_column("NMA_Radionuclides", "OBJECTID") + op.drop_column("NMA_Radionuclides", "GlobalID") + + # --- NMA_Soil_Rock_Results --- + op.add_column( + "NMA_Soil_Rock_Results", + sa.Column("nma_Point_ID", sa.String(length=255), nullable=True), + ) + op.drop_index( + op.f("Soil_Rock_Results$Point_ID"), table_name="NMA_Soil_Rock_Results" + ) + op.drop_column("NMA_Soil_Rock_Results", "Point_ID") + + # --- NMA_Stratigraphy --- + op.add_column( + "NMA_Stratigraphy", + sa.Column( + "id", sa.Integer(), sa.Identity(always=False, start=1), nullable=False + ), + ) + op.add_column( + "NMA_Stratigraphy", sa.Column("nma_GlobalID", sa.UUID(), nullable=True) + ) + op.add_column("NMA_Stratigraphy", sa.Column("nma_WellID", sa.UUID(), nullable=True)) + op.add_column( + "NMA_Stratigraphy", + sa.Column("nma_PointID", sa.String(length=10), nullable=False), + ) + op.add_column( + "NMA_Stratigraphy", sa.Column("nma_OBJECTID", sa.Integer(), nullable=True) + ) + op.drop_constraint( + op.f("NMA_Stratigraphy_OBJECTID_key"), "NMA_Stratigraphy", type_="unique" + ) + op.drop_index(op.f("ix_nma_stratigraphy_point_id"), table_name="NMA_Stratigraphy") + op.drop_index(op.f("ix_nma_stratigraphy_thing_id"), table_name="NMA_Stratigraphy") + op.create_unique_constraint(None, "NMA_Stratigraphy", ["nma_GlobalID"]) + op.create_unique_constraint(None, "NMA_Stratigraphy", ["nma_OBJECTID"]) + op.drop_column("NMA_Stratigraphy", "OBJECTID") + op.drop_column("NMA_Stratigraphy", "WellID") + op.drop_column("NMA_Stratigraphy", "PointID") + op.drop_column("NMA_Stratigraphy", "GlobalID") + + # --- Other tables (index/constraint cleanup from autogenerate) --- + op.drop_index( + op.f("SurfaceWaterPhotos$PointID"), table_name="NMA_SurfaceWaterPhotos" + ) + op.drop_index( + op.f("SurfaceWaterPhotos$SurfaceID"), table_name="NMA_SurfaceWaterPhotos" + ) + op.drop_constraint( + op.f("uq_nma_pressure_daily_globalid"), + "NMA_WaterLevelsContinuous_Pressure_Daily", + type_="unique", + ) + op.drop_index(op.f("WeatherPhotos$PointID"), table_name="NMA_WeatherPhotos") + op.drop_index(op.f("WeatherPhotos$WeatherID"), table_name="NMA_WeatherPhotos") + op.alter_column( + "NMA_view_NGWMN_Lithology", + "PointID", + existing_type=sa.VARCHAR(length=50), + nullable=False, + ) + op.drop_constraint( + op.f("uq_nma_view_ngwmn_lithology_objectid"), + "NMA_view_NGWMN_Lithology", + type_="unique", + ) + op.drop_constraint( + op.f("uq_nma_view_ngwmn_waterlevels_point_date"), + "NMA_view_NGWMN_WaterLevels", + type_="unique", + ) + op.alter_column( + "NMA_view_NGWMN_WellConstruction", + "PointID", + existing_type=sa.VARCHAR(length=50), + nullable=False, + ) + op.drop_constraint( + op.f("uq_nma_view_ngwmn_wellconstruction_point_casing_screen"), + "NMA_view_NGWMN_WellConstruction", + type_="unique", + ) + op.alter_column( + "thing", + "nma_formation_zone", + existing_type=sa.VARCHAR(length=25), + comment="Raw FormationZone value from legacy WellData (NM_Aquifer).", + existing_nullable=True, + ) + op.alter_column( + "thing_version", + "nma_pk_location", + existing_type=sa.VARCHAR(), + comment="To audit the original NM_Aquifer LocationID if it was transferred over", + existing_nullable=True, + autoincrement=False, + ) + op.alter_column( + "thing_version", + "nma_formation_zone", + existing_type=sa.VARCHAR(length=25), + comment="Raw FormationZone value from legacy WellData (NM_Aquifer).", + existing_nullable=True, + autoincrement=False, + ) + op.alter_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_created", + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True, + ) + op.alter_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_updated", + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True, + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.alter_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_updated", + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True, + ) + op.alter_column( + "transducer_observation", + "nma_waterlevelscontinuous_pressure_created", + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True, + ) + op.alter_column( + "thing_version", + "nma_formation_zone", + existing_type=sa.VARCHAR(length=25), + comment=None, + existing_comment="Raw FormationZone value from legacy WellData (NM_Aquifer).", + existing_nullable=True, + autoincrement=False, + ) + op.alter_column( + "thing_version", + "nma_pk_location", + existing_type=sa.VARCHAR(), + comment=None, + existing_comment="To audit the original NM_Aquifer LocationID if it was transferred over", + existing_nullable=True, + autoincrement=False, + ) + op.alter_column( + "thing", + "nma_formation_zone", + existing_type=sa.VARCHAR(length=25), + comment=None, + existing_comment="Raw FormationZone value from legacy WellData (NM_Aquifer).", + existing_nullable=True, + ) + op.create_unique_constraint( + op.f("uq_nma_view_ngwmn_wellconstruction_point_casing_screen"), + "NMA_view_NGWMN_WellConstruction", + ["PointID", "CasingTop", "ScreenTop"], + postgresql_nulls_not_distinct=False, + ) + op.alter_column( + "NMA_view_NGWMN_WellConstruction", + "PointID", + existing_type=sa.VARCHAR(length=50), + nullable=True, + ) + op.create_unique_constraint( + op.f("uq_nma_view_ngwmn_waterlevels_point_date"), + "NMA_view_NGWMN_WaterLevels", + ["PointID", "DateMeasured"], + postgresql_nulls_not_distinct=False, + ) + op.create_unique_constraint( + op.f("uq_nma_view_ngwmn_lithology_objectid"), + "NMA_view_NGWMN_Lithology", + ["OBJECTID"], + postgresql_nulls_not_distinct=False, + ) + op.alter_column( + "NMA_view_NGWMN_Lithology", + "PointID", + existing_type=sa.VARCHAR(length=50), + nullable=True, + ) + op.create_index( + op.f("WeatherPhotos$WeatherID"), + "NMA_WeatherPhotos", + ["WeatherID"], + unique=False, + ) + op.create_index( + op.f("WeatherPhotos$PointID"), "NMA_WeatherPhotos", ["PointID"], unique=False + ) + op.create_unique_constraint( + op.f("uq_nma_pressure_daily_globalid"), + "NMA_WaterLevelsContinuous_Pressure_Daily", + ["GlobalID"], + postgresql_nulls_not_distinct=False, + ) + op.create_index( + op.f("SurfaceWaterPhotos$SurfaceID"), + "NMA_SurfaceWaterPhotos", + ["SurfaceID"], + unique=False, + ) + op.create_index( + op.f("SurfaceWaterPhotos$PointID"), + "NMA_SurfaceWaterPhotos", + ["PointID"], + unique=False, + ) + op.add_column( + "NMA_Stratigraphy", + sa.Column("GlobalID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_Stratigraphy", + sa.Column( + "PointID", sa.VARCHAR(length=10), autoincrement=False, nullable=False + ), + ) + op.add_column( + "NMA_Stratigraphy", + sa.Column("WellID", sa.UUID(), autoincrement=False, nullable=True), + ) + op.add_column( + "NMA_Stratigraphy", + sa.Column("OBJECTID", sa.INTEGER(), autoincrement=False, nullable=True), + ) + op.drop_constraint(None, "NMA_Stratigraphy", type_="unique") + op.drop_constraint(None, "NMA_Stratigraphy", type_="unique") + op.create_index( + op.f("ix_nma_stratigraphy_thing_id"), + "NMA_Stratigraphy", + ["thing_id"], + unique=False, + ) + op.create_index( + op.f("ix_nma_stratigraphy_point_id"), + "NMA_Stratigraphy", + ["PointID"], + unique=False, + ) + op.create_unique_constraint( + op.f("NMA_Stratigraphy_OBJECTID_key"), + "NMA_Stratigraphy", + ["OBJECTID"], + postgresql_nulls_not_distinct=False, + ) + op.drop_column("NMA_Stratigraphy", "nma_OBJECTID") + op.drop_column("NMA_Stratigraphy", "nma_PointID") + op.drop_column("NMA_Stratigraphy", "nma_WellID") + op.drop_column("NMA_Stratigraphy", "nma_GlobalID") + op.drop_column("NMA_Stratigraphy", "id") + op.add_column( + "NMA_Soil_Rock_Results", + sa.Column( + "Point_ID", sa.VARCHAR(length=255), autoincrement=False, nullable=True + ), + ) + op.create_index( + op.f("Soil_Rock_Results$Point_ID"), + "NMA_Soil_Rock_Results", + ["Point_ID"], + unique=False, + ) + op.drop_column("NMA_Soil_Rock_Results", "nma_Point_ID") + op.add_column( + "NMA_Radionuclides", + sa.Column("GlobalID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_Radionuclides", + sa.Column("OBJECTID", sa.INTEGER(), autoincrement=False, nullable=True), + ) + op.add_column( + "NMA_Radionuclides", + sa.Column( + "WCLab_ID", sa.VARCHAR(length=25), autoincrement=False, nullable=True + ), + ) + op.add_column( + "NMA_Radionuclides", + sa.Column("SamplePtID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_Radionuclides", + sa.Column( + "SamplePointID", sa.VARCHAR(length=10), autoincrement=False, nullable=True + ), + ) + op.drop_constraint(None, "NMA_Radionuclides", type_="foreignkey") + op.create_foreign_key( + op.f("NMA_Radionuclides_SamplePtID_fkey"), + "NMA_Radionuclides", + "NMA_Chemistry_SampleInfo", + ["SamplePtID"], + ["SamplePtID"], + ondelete="CASCADE", + ) + op.drop_constraint(None, "NMA_Radionuclides", type_="unique") + op.drop_constraint(None, "NMA_Radionuclides", type_="unique") + op.create_index( + op.f("Radionuclides$WCLab_ID"), "NMA_Radionuclides", ["WCLab_ID"], unique=False + ) + op.create_index( + op.f("Radionuclides$SamplePtID"), + "NMA_Radionuclides", + ["SamplePtID"], + unique=False, + ) + op.create_index( + op.f("Radionuclides$SamplePointID"), + "NMA_Radionuclides", + ["SamplePointID"], + unique=False, + ) + op.create_index( + op.f("Radionuclides$Chemistry SampleInfoRadionuclides"), + "NMA_Radionuclides", + ["SamplePtID"], + unique=False, + ) + op.create_index( + op.f("Radionuclides$Analyte"), "NMA_Radionuclides", ["Analyte"], unique=False + ) + op.create_index( + op.f("Radionuclides$AnalysesAgency"), + "NMA_Radionuclides", + ["AnalysesAgency"], + unique=False, + ) + op.create_unique_constraint( + op.f("NMA_Radionuclides_OBJECTID_key"), + "NMA_Radionuclides", + ["OBJECTID"], + postgresql_nulls_not_distinct=False, + ) + op.drop_column("NMA_Radionuclides", "nma_WCLab_ID") + op.drop_column("NMA_Radionuclides", "nma_OBJECTID") + op.drop_column("NMA_Radionuclides", "nma_SamplePointID") + op.drop_column("NMA_Radionuclides", "nma_SamplePtID") + op.drop_column("NMA_Radionuclides", "chemistry_sample_info_id") + op.drop_column("NMA_Radionuclides", "nma_GlobalID") + op.drop_column("NMA_Radionuclides", "id") + op.add_column( + "NMA_MinorTraceChemistry", + sa.Column("GlobalID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.drop_constraint(None, "NMA_MinorTraceChemistry", type_="foreignkey") + op.create_foreign_key( + op.f("NMA_MinorTraceChemistry_chemistry_sample_info_id_fkey"), + "NMA_MinorTraceChemistry", + "NMA_Chemistry_SampleInfo", + ["chemistry_sample_info_id"], + ["SamplePtID"], + ondelete="CASCADE", + ) + op.drop_constraint(None, "NMA_MinorTraceChemistry", type_="unique") + op.alter_column( + "NMA_MinorTraceChemistry", + "chemistry_sample_info_id", + existing_type=sa.Integer(), + type_=sa.UUID(), + existing_nullable=False, + ) + op.drop_column("NMA_MinorTraceChemistry", "nma_chemistry_sample_info_uuid") + op.drop_column("NMA_MinorTraceChemistry", "nma_GlobalID") + op.drop_column("NMA_MinorTraceChemistry", "id") + op.add_column( + "NMA_MajorChemistry", + sa.Column("GlobalID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_MajorChemistry", + sa.Column("OBJECTID", sa.INTEGER(), autoincrement=False, nullable=True), + ) + op.add_column( + "NMA_MajorChemistry", + sa.Column( + "WCLab_ID", sa.VARCHAR(length=25), autoincrement=False, nullable=True + ), + ) + op.add_column( + "NMA_MajorChemistry", + sa.Column("SamplePtID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_MajorChemistry", + sa.Column( + "SamplePointID", sa.VARCHAR(length=10), autoincrement=False, nullable=True + ), + ) + op.drop_constraint(None, "NMA_MajorChemistry", type_="foreignkey") + op.create_foreign_key( + op.f("NMA_MajorChemistry_SamplePtID_fkey"), + "NMA_MajorChemistry", + "NMA_Chemistry_SampleInfo", + ["SamplePtID"], + ["SamplePtID"], + ondelete="CASCADE", + ) + op.drop_constraint(None, "NMA_MajorChemistry", type_="unique") + op.drop_constraint(None, "NMA_MajorChemistry", type_="unique") + op.create_unique_constraint( + op.f("NMA_MajorChemistry_OBJECTID_key"), + "NMA_MajorChemistry", + ["OBJECTID"], + postgresql_nulls_not_distinct=False, + ) + op.create_index( + op.f("MajorChemistry$WCLab_ID"), + "NMA_MajorChemistry", + ["WCLab_ID"], + unique=False, + ) + op.create_index( + op.f("MajorChemistry$SamplePtID"), + "NMA_MajorChemistry", + ["SamplePtID"], + unique=False, + ) + op.create_index( + op.f("MajorChemistry$SamplePointIDAnalyte"), + "NMA_MajorChemistry", + ["SamplePointID", "Analyte"], + unique=False, + ) + op.create_index( + op.f("MajorChemistry$SamplePointID"), + "NMA_MajorChemistry", + ["SamplePointID"], + unique=False, + ) + op.create_index( + op.f("MajorChemistry$Chemistry SampleInfoMajorChemistry"), + "NMA_MajorChemistry", + ["SamplePtID"], + unique=False, + ) + op.create_index( + op.f("MajorChemistry$Analyte"), "NMA_MajorChemistry", ["Analyte"], unique=False + ) + op.create_index( + op.f("MajorChemistry$AnalysesAgency"), + "NMA_MajorChemistry", + ["AnalysesAgency"], + unique=False, + ) + op.drop_column("NMA_MajorChemistry", "nma_WCLab_ID") + op.drop_column("NMA_MajorChemistry", "nma_OBJECTID") + op.drop_column("NMA_MajorChemistry", "nma_SamplePointID") + op.drop_column("NMA_MajorChemistry", "nma_SamplePtID") + op.drop_column("NMA_MajorChemistry", "chemistry_sample_info_id") + op.drop_column("NMA_MajorChemistry", "nma_GlobalID") + op.drop_column("NMA_MajorChemistry", "id") + op.add_column( + "NMA_HydraulicsData", + sa.Column("GlobalID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_HydraulicsData", + sa.Column("PointID", sa.VARCHAR(length=50), autoincrement=False, nullable=True), + ) + op.add_column( + "NMA_HydraulicsData", + sa.Column("OBJECTID", sa.INTEGER(), autoincrement=False, nullable=True), + ) + op.add_column( + "NMA_HydraulicsData", + sa.Column("WellID", sa.UUID(), autoincrement=False, nullable=True), + ) + op.drop_constraint(None, "NMA_HydraulicsData", type_="unique") + op.drop_constraint(None, "NMA_HydraulicsData", type_="unique") + op.create_index( + op.f("ix_nma_hydraulicsdata_wellid"), + "NMA_HydraulicsData", + ["WellID"], + unique=False, + ) + op.create_index( + op.f("ix_nma_hydraulicsdata_pointid"), + "NMA_HydraulicsData", + ["PointID"], + unique=False, + ) + op.create_index( + op.f("ix_nma_hydraulicsdata_objectid"), + "NMA_HydraulicsData", + ["OBJECTID"], + unique=True, + ) + op.drop_column("NMA_HydraulicsData", "nma_OBJECTID") + op.drop_column("NMA_HydraulicsData", "nma_PointID") + op.drop_column("NMA_HydraulicsData", "nma_WellID") + op.drop_column("NMA_HydraulicsData", "nma_GlobalID") + op.drop_column("NMA_HydraulicsData", "id") + op.add_column( + "NMA_FieldParameters", + sa.Column("GlobalID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_FieldParameters", + sa.Column( + "OBJECTID", + sa.INTEGER(), + sa.Identity( + always=False, + start=1, + increment=1, + minvalue=1, + maxvalue=2147483647, + cycle=False, + cache=1, + ), + autoincrement=True, + nullable=False, + ), + ) + op.add_column( + "NMA_FieldParameters", + sa.Column( + "WCLab_ID", sa.VARCHAR(length=25), autoincrement=False, nullable=True + ), + ) + op.add_column( + "NMA_FieldParameters", + sa.Column("SamplePtID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_FieldParameters", + sa.Column( + "SamplePointID", sa.VARCHAR(length=10), autoincrement=False, nullable=True + ), + ) + op.drop_constraint(None, "NMA_FieldParameters", type_="foreignkey") + op.create_foreign_key( + op.f("NMA_FieldParameters_SamplePtID_fkey"), + "NMA_FieldParameters", + "NMA_Chemistry_SampleInfo", + ["SamplePtID"], + ["SamplePtID"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + op.drop_constraint(None, "NMA_FieldParameters", type_="unique") + op.drop_index("FieldParameters$nma_WCLab_ID", table_name="NMA_FieldParameters") + op.drop_index("FieldParameters$nma_SamplePointID", table_name="NMA_FieldParameters") + op.drop_index("FieldParameters$nma_OBJECTID", table_name="NMA_FieldParameters") + op.drop_index("FieldParameters$nma_GlobalID", table_name="NMA_FieldParameters") + op.drop_index( + "FieldParameters$ChemistrySampleInfoFieldParameters", + table_name="NMA_FieldParameters", + ) + op.create_index( + op.f("FieldParameters$ChemistrySampleInfoFieldParameters"), + "NMA_FieldParameters", + ["SamplePtID"], + unique=False, + ) + op.create_index( + op.f("FieldParameters$WCLab_ID"), + "NMA_FieldParameters", + ["WCLab_ID"], + unique=False, + ) + op.create_index( + op.f("FieldParameters$SamplePtID"), + "NMA_FieldParameters", + ["SamplePtID"], + unique=False, + ) + op.create_index( + op.f("FieldParameters$SamplePointID"), + "NMA_FieldParameters", + ["SamplePointID"], + unique=False, + ) + op.create_index( + op.f("FieldParameters$OBJECTID"), + "NMA_FieldParameters", + ["OBJECTID"], + unique=True, + ) + op.create_index( + op.f("FieldParameters$GlobalID"), + "NMA_FieldParameters", + ["GlobalID"], + unique=True, + ) + op.drop_column("NMA_FieldParameters", "nma_WCLab_ID") + op.drop_column("NMA_FieldParameters", "nma_OBJECTID") + op.drop_column("NMA_FieldParameters", "nma_SamplePointID") + op.drop_column("NMA_FieldParameters", "nma_SamplePtID") + op.drop_column("NMA_FieldParameters", "chemistry_sample_info_id") + op.drop_column("NMA_FieldParameters", "nma_GlobalID") + op.drop_column("NMA_FieldParameters", "id") + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column("LocationId", sa.UUID(), autoincrement=False, nullable=True), + ) + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column("OBJECTID", sa.INTEGER(), autoincrement=False, nullable=True), + ) + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column( + "WCLab_ID", sa.VARCHAR(length=18), autoincrement=False, nullable=True + ), + ) + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column("SamplePtID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_Chemistry_SampleInfo", + sa.Column( + "SamplePointID", sa.VARCHAR(length=10), autoincrement=False, nullable=False + ), + ) + op.drop_constraint(None, "NMA_Chemistry_SampleInfo", type_="unique") + op.drop_constraint(None, "NMA_Chemistry_SampleInfo", type_="unique") + op.create_unique_constraint( + op.f("NMA_Chemistry_SampleInfo_OBJECTID_key"), + "NMA_Chemistry_SampleInfo", + ["OBJECTID"], + postgresql_nulls_not_distinct=False, + ) + op.drop_column("NMA_Chemistry_SampleInfo", "nma_LocationId") + op.drop_column("NMA_Chemistry_SampleInfo", "nma_OBJECTID") + op.drop_column("NMA_Chemistry_SampleInfo", "nma_SamplePointID") + op.drop_column("NMA_Chemistry_SampleInfo", "nma_WCLab_ID") + op.drop_column("NMA_Chemistry_SampleInfo", "nma_SamplePtID") + op.drop_column("NMA_Chemistry_SampleInfo", "id") + op.add_column( + "NMA_AssociatedData", + sa.Column("PointID", sa.VARCHAR(length=10), autoincrement=False, nullable=True), + ) + op.add_column( + "NMA_AssociatedData", + sa.Column("AssocID", sa.UUID(), autoincrement=False, nullable=False), + ) + op.add_column( + "NMA_AssociatedData", + sa.Column("LocationId", sa.UUID(), autoincrement=False, nullable=True), + ) + op.add_column( + "NMA_AssociatedData", + sa.Column("OBJECTID", sa.INTEGER(), autoincrement=False, nullable=True), + ) + op.drop_constraint(None, "NMA_AssociatedData", type_="unique") + op.drop_constraint(None, "NMA_AssociatedData", type_="unique") + op.drop_constraint(None, "NMA_AssociatedData", type_="unique") + op.create_unique_constraint( + op.f("NMA_AssociatedData_OBJECTID_key"), + "NMA_AssociatedData", + ["OBJECTID"], + postgresql_nulls_not_distinct=False, + ) + op.create_index( + op.f("AssociatedData$PointID"), "NMA_AssociatedData", ["PointID"], unique=False + ) + op.create_unique_constraint( + op.f("AssociatedData$LocationId"), + "NMA_AssociatedData", + ["LocationId"], + postgresql_nulls_not_distinct=False, + ) + op.drop_column("NMA_AssociatedData", "nma_OBJECTID") + op.drop_column("NMA_AssociatedData", "nma_PointID") + op.drop_column("NMA_AssociatedData", "nma_LocationId") + op.drop_column("NMA_AssociatedData", "nma_AssocID") + op.drop_column("NMA_AssociatedData", "id") diff --git a/alembic/versions/43bc34504ee6_merge_migrations_after_staging_merge.py b/alembic/versions/43bc34504ee6_merge_migrations_after_staging_merge.py new file mode 100644 index 000000000..82f93b47a --- /dev/null +++ b/alembic/versions/43bc34504ee6_merge_migrations_after_staging_merge.py @@ -0,0 +1,30 @@ +"""merge_migrations_after_staging_merge + +Revision ID: 43bc34504ee6 +Revises: 3cb924ca51fd, e123456789ab +Create Date: 2026-01-30 11:52:41.932306 + +""" + +from typing import Sequence, Union + +from alembic import op +import geoalchemy2 +import sqlalchemy as sa +import sqlalchemy_utils + +# revision identifiers, used by Alembic. +revision: str = "43bc34504ee6" +down_revision: Union[str, Sequence[str], None] = ("3cb924ca51fd", "e123456789ab") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + pass + + +def downgrade() -> None: + """Downgrade schema.""" + pass diff --git a/alembic/versions/76e3ae8b99cb_enforce_thing_fk_for_nma_legacy_models.py b/alembic/versions/76e3ae8b99cb_enforce_thing_fk_for_nma_legacy_models.py new file mode 100644 index 000000000..33784c7e6 --- /dev/null +++ b/alembic/versions/76e3ae8b99cb_enforce_thing_fk_for_nma_legacy_models.py @@ -0,0 +1,80 @@ +"""enforce_thing_fk_for_nma_legacy_models + +Revision ID: 76e3ae8b99cb +Revises: c1d2e3f4a5b6 +Create Date: 2026-01-26 11:56:28.744603 + +Issue: #363 +Feature: features/admin/well_data_relationships.feature + +This migration enforces foreign key relationships between Thing and NMA legacy models: +1. Adds nma_pk_location column to Thing for storing legacy NM_Aquifer LocationID +2. Makes thing_id NOT NULL on NMA_AssociatedData (was nullable) +3. Makes thing_id NOT NULL on NMA_Soil_Rock_Results (was nullable) + +Note: Before running this migration, ensure no orphan records exist in the affected tables. +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "76e3ae8b99cb" +down_revision: Union[str, Sequence[str], None] = "c1d2e3f4a5b6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema to enforce Thing FK relationships.""" + # 1. Add nma_pk_location column to thing table and its version table + op.add_column( + "thing", + sa.Column( + "nma_pk_location", + sa.String(), + nullable=True, + comment="To audit the original NM_Aquifer LocationID if it was transferred over", + ), + ) + op.add_column( + "thing_version", + sa.Column( + "nma_pk_location", + sa.String(), + nullable=True, + ), + ) + + # 2. Make thing_id NOT NULL on NMA_AssociatedData + # First, delete any orphan records (records without a thing_id) + op.execute('DELETE FROM "NMA_AssociatedData" WHERE thing_id IS NULL') + op.alter_column( + "NMA_AssociatedData", "thing_id", existing_type=sa.Integer(), nullable=False + ) + + # 3. Make thing_id NOT NULL on NMA_Soil_Rock_Results + # First, delete any orphan records (records without a thing_id) + op.execute('DELETE FROM "NMA_Soil_Rock_Results" WHERE thing_id IS NULL') + op.alter_column( + "NMA_Soil_Rock_Results", "thing_id", existing_type=sa.Integer(), nullable=False + ) + + +def downgrade() -> None: + """Downgrade schema to allow nullable thing_id.""" + # 1. Remove nma_pk_location column from thing table and its version table + op.drop_column("thing", "nma_pk_location") + op.drop_column("thing_version", "nma_pk_location") + + # 2. Make thing_id nullable on NMA_AssociatedData + op.alter_column( + "NMA_AssociatedData", "thing_id", existing_type=sa.Integer(), nullable=True + ) + + # 3. Make thing_id nullable on NMA_Soil_Rock_Results + op.alter_column( + "NMA_Soil_Rock_Results", "thing_id", existing_type=sa.Integer(), nullable=True + ) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index 87f9c447c..8717448bc 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -14,7 +14,34 @@ # limitations under the License. # =============================================================================== -"""Legacy NM Aquifer models copied from AMPAPI.""" +"""Legacy NM Aquifer models copied from AMPAPI. + +This module contains models for NMA legacy tables that have been refactored to use +Integer primary keys. The original UUID PKs have been renamed with 'nma_' prefix +for audit/traceability purposes. + +Refactoring Summary (UUID -> Integer PK): +- NMA_HydraulicsData: global_id -> nma_global_id, new id PK +- NMA_Stratigraphy: global_id -> nma_global_id, new id PK +- NMA_Chemistry_SampleInfo: sample_pt_id -> nma_sample_pt_id, new id PK +- NMA_AssociatedData: assoc_id -> nma_assoc_id, new id PK +- NMA_Radionuclides: global_id -> nma_global_id, new id PK +- NMA_MinorTraceChemistry: global_id -> nma_global_id, new id PK +- NMA_MajorChemistry: global_id -> nma_global_id, new id PK +- NMA_FieldParameters: global_id -> nma_global_id, new id PK + +FK Standardization: +- Chemistry children now use chemistry_sample_info_id (Integer FK) +- Legacy UUID FKs stored as nma_sample_pt_id for audit + +Legacy ID Columns Renamed (nma_ prefix): +- well_id -> nma_well_id +- point_id -> nma_point_id +- location_id -> nma_location_id +- object_id -> nma_object_id +- sample_point_id -> nma_sample_point_id +- wclab_id -> nma_wclab_id +""" import uuid from datetime import date, datetime @@ -42,6 +69,7 @@ from db.base import Base if TYPE_CHECKING: + from db.location import Location from db.thing import Thing @@ -52,6 +80,9 @@ class NMA_WaterLevelsContinuous_Pressure_Daily(Base): This model is used for read-only migration/interop with the legacy NM Aquifer data and mirrors the original column names/types closely so transfer scripts can operate without further schema mapping. + + Note: This table is OUT OF SCOPE for the UUID->Integer PK refactoring since + it's not a Thing child table. """ __tablename__ = "NMA_WaterLevelsContinuous_Pressure_Daily" @@ -97,6 +128,8 @@ class NMA_view_NGWMN_WellConstruction(Base): A surrogate primary key is used so rows with missing depth values can still be represented faithfully from the legacy view. + + Note: This table is OUT OF SCOPE for refactoring (view table). """ __tablename__ = "NMA_view_NGWMN_WellConstruction" @@ -124,6 +157,8 @@ class NMA_view_NGWMN_WellConstruction(Base): class NMA_view_NGWMN_WaterLevels(Base): """ Legacy NGWMN water levels view. + + Note: This table is OUT OF SCOPE for refactoring (view table). """ __tablename__ = "NMA_view_NGWMN_WaterLevels" @@ -144,6 +179,8 @@ class NMA_view_NGWMN_WaterLevels(Base): class NMA_view_NGWMN_Lithology(Base): """ Legacy NGWMN lithology view. + + Note: This table is OUT OF SCOPE for refactoring (view table). """ __tablename__ = "NMA_view_NGWMN_Lithology" @@ -164,20 +201,39 @@ class NMA_view_NGWMN_Lithology(Base): class NMA_HydraulicsData(Base): """ Legacy HydraulicsData table from AMPAPI. + + Refactored from UUID PK to Integer PK: + - id: Integer PK (autoincrement) + - nma_global_id: Original UUID PK, now UNIQUE for audit + - nma_well_id: Legacy WellID UUID + - nma_point_id: Legacy PointID string + - nma_object_id: Legacy OBJECTID, UNIQUE """ __tablename__ = "NMA_HydraulicsData" - global_id: Mapped[uuid.UUID] = mapped_column( - "GlobalID", UUID(as_uuid=True), primary_key=True + # New Integer PK + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + # Legacy UUID PK (now audit column) + nma_global_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_GlobalID", UUID(as_uuid=True), unique=True, nullable=True ) - well_id: Mapped[Optional[uuid.UUID]] = mapped_column("WellID", UUID(as_uuid=True)) - point_id: Mapped[Optional[str]] = mapped_column("PointID", String(50)) + + # Legacy ID columns (renamed with nma_ prefix) + nma_well_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_WellID", UUID(as_uuid=True) + ) + nma_point_id: Mapped[Optional[str]] = mapped_column("nma_PointID", String(50)) + nma_object_id: Mapped[Optional[int]] = mapped_column( + "nma_OBJECTID", Integer, unique=True + ) + + # Data columns data_source: Mapped[Optional[str]] = mapped_column("Data Source", String(255)) thing_id: Mapped[int] = mapped_column( Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) - object_id: Mapped[Optional[int]] = mapped_column("OBJECTID", Integer, unique=True) cs_gal_d_ft: Mapped[Optional[float]] = mapped_column("Cs (gal/d/ft)", Float) hd_ft2_d: Mapped[Optional[float]] = mapped_column("HD (ft2/d)", Float) @@ -205,25 +261,56 @@ class NMA_HydraulicsData(Base): "Hydraulic Remarks", String(200) ) - thing: Mapped["Thing"] = relationship("Thing") + thing: Mapped["Thing"] = relationship("Thing", back_populates="hydraulics_data") + + @validates("thing_id") + def validate_thing_id(self, key, value): + """Prevent orphan NMA_HydraulicsData - must have a parent Thing.""" + if value is None: + raise ValueError( + "NMA_HydraulicsData requires a parent Thing (thing_id cannot be None)" + ) + return value class NMA_Stratigraphy(Base): - """Legacy stratigraphy (lithology log) data from AMPAPI.""" + """ + Legacy stratigraphy (lithology log) data from AMPAPI. + + Refactored from UUID PK to Integer PK: + - id: Integer PK (autoincrement) + - nma_global_id: Original UUID PK, now UNIQUE for audit + - nma_well_id: Legacy WellID UUID + - nma_point_id: Legacy PointID string + - nma_object_id: Legacy OBJECTID, UNIQUE + """ __tablename__ = "NMA_Stratigraphy" __table_args__ = ( CheckConstraint( - 'char_length("PointID") > 0', + 'char_length("nma_PointID") > 0', name="ck_nma_stratigraphy_pointid_len", ), ) - global_id: Mapped[uuid.UUID] = mapped_column( - "GlobalID", UUID(as_uuid=True), primary_key=True + # New Integer PK + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + # Legacy UUID PK (now audit column) + nma_global_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_GlobalID", UUID(as_uuid=True), unique=True, nullable=True ) - well_id: Mapped[Optional[uuid.UUID]] = mapped_column("WellID", UUID(as_uuid=True)) - point_id: Mapped[str] = mapped_column("PointID", String(50), nullable=False) + + # Legacy ID columns (renamed with nma_ prefix) + nma_well_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_WellID", UUID(as_uuid=True) + ) + nma_point_id: Mapped[str] = mapped_column("nma_PointID", String(10), nullable=False) + nma_object_id: Mapped[Optional[int]] = mapped_column( + "nma_OBJECTID", Integer, unique=True + ) + + # FK to Thing thing_id: Mapped[int] = mapped_column( Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) @@ -240,26 +327,59 @@ class NMA_Stratigraphy(Base): contributing_unit: Mapped[Optional[str]] = mapped_column( "ContributingUnit", String(2) ) - strat_source: Mapped[Optional[str]] = mapped_column("StratSource", String(100)) - strat_notes: Mapped[Optional[str]] = mapped_column("StratNotes", String(255)) - object_id: Mapped[Optional[int]] = mapped_column("OBJECTID", Integer, unique=True) + strat_source: Mapped[Optional[str]] = mapped_column("StratSource", Text) + strat_notes: Mapped[Optional[str]] = mapped_column("StratNotes", Text) thing: Mapped["Thing"] = relationship("Thing", back_populates="stratigraphy_logs") + @validates("thing_id") + def validate_thing_id(self, key, value): + """Prevent orphan NMA_Stratigraphy - must have a parent Thing.""" + if value is None: + raise ValueError( + "NMA_Stratigraphy requires a parent Thing (thing_id cannot be None)" + ) + return value + class NMA_Chemistry_SampleInfo(Base): """ Legacy Chemistry SampleInfo table from AMPAPI. + + Refactored from UUID PK to Integer PK: + - id: Integer PK (autoincrement) + - nma_sample_pt_id: Original UUID PK (SamplePtID), now UNIQUE for audit + - nma_wclab_id: Legacy WCLab_ID + - nma_sample_point_id: Legacy SamplePointID + - nma_object_id: Legacy OBJECTID, UNIQUE + - nma_location_id: Legacy LocationId UUID (for audit trail) + + FK to Thing: + - thing_id: Integer FK to Thing.id + - Linked via nma_SamplePointID matching Thing.name during transfer """ __tablename__ = "NMA_Chemistry_SampleInfo" - sample_pt_id: Mapped[uuid.UUID] = mapped_column( - "SamplePtID", UUID(as_uuid=True), primary_key=True + # New Integer PK + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + # Legacy UUID PK (now audit column) + nma_sample_pt_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_SamplePtID", UUID(as_uuid=True), unique=True, nullable=True + ) + + # Legacy ID columns (renamed with nma_ prefix) + nma_wclab_id: Mapped[Optional[str]] = mapped_column("nma_WCLab_ID", String(18)) + nma_sample_point_id: Mapped[str] = mapped_column( + "nma_SamplePointID", String(10), nullable=False ) - wclab_id: Mapped[Optional[str]] = mapped_column("WCLab_ID", String(18)) - sample_point_id: Mapped[str] = mapped_column( - "SamplePointID", String(10), nullable=False + nma_object_id: Mapped[Optional[int]] = mapped_column( + "nma_OBJECTID", Integer, unique=True + ) + # Legacy LocationId UUID - kept for audit trail + nma_location_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_LocationId", UUID(as_uuid=True) ) # FK to Thing - required for all ChemistrySampleInfo records @@ -295,11 +415,6 @@ class NMA_Chemistry_SampleInfo(Base): ) sample_notes: Mapped[Optional[str]] = mapped_column("SampleNotes", Text) - object_id: Mapped[Optional[int]] = mapped_column("OBJECTID", Integer, unique=True) - location_id: Mapped[Optional[uuid.UUID]] = mapped_column( - "LocationId", UUID(as_uuid=True) - ) - # --- Relationships --- thing: Mapped["Thing"] = relationship( "Thing", back_populates="chemistry_sample_infos" @@ -346,30 +461,57 @@ def validate_thing_id(self, key, value): class NMA_AssociatedData(Base): """ Legacy AssociatedData table from NM_Aquifer. + + Refactored from UUID PK to Integer PK: + - id: Integer PK (autoincrement) + - nma_assoc_id: Original UUID PK (AssocID), now UNIQUE for audit + - nma_location_id: Legacy LocationId UUID, UNIQUE + - nma_point_id: Legacy PointID string + - nma_object_id: Legacy OBJECTID, UNIQUE """ __tablename__ = "NMA_AssociatedData" - location_id: Mapped[Optional[uuid.UUID]] = mapped_column( - "LocationId", UUID(as_uuid=True), unique=True + # New Integer PK + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + # Legacy UUID PK (now audit column) + nma_assoc_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_AssocID", UUID(as_uuid=True), unique=True, nullable=True + ) + + # Legacy ID columns (renamed with nma_ prefix) + nma_location_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_LocationId", UUID(as_uuid=True), unique=True ) - point_id: Mapped[Optional[str]] = mapped_column("PointID", String(10)) - assoc_id: Mapped[uuid.UUID] = mapped_column( - "AssocID", UUID(as_uuid=True), primary_key=True + nma_point_id: Mapped[Optional[str]] = mapped_column("nma_PointID", String(10)) + nma_object_id: Mapped[Optional[int]] = mapped_column( + "nma_OBJECTID", Integer, unique=True ) + notes: Mapped[Optional[str]] = mapped_column("Notes", String(255)) formation: Mapped[Optional[str]] = mapped_column("Formation", String(15)) - object_id: Mapped[Optional[int]] = mapped_column("OBJECTID", Integer, unique=True) - thing_id: Mapped[Optional[int]] = mapped_column( - Integer, ForeignKey("thing.id", ondelete="CASCADE") + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) - thing: Mapped["Thing"] = relationship("Thing") + thing: Mapped["Thing"] = relationship("Thing", back_populates="associated_data") + + @validates("thing_id") + def validate_thing_id(self, key, value): + """Prevent orphan NMA_AssociatedData - must have a parent Thing.""" + if value is None: + raise ValueError( + "NMA_AssociatedData requires a parent Thing (thing_id cannot be None)" + ) + return value class NMA_SurfaceWaterData(Base): """ Legacy SurfaceWaterData table from AMPAPI. + + Note: This table is OUT OF SCOPE for refactoring (not a Thing child). """ __tablename__ = "NMA_SurfaceWaterData" @@ -406,6 +548,8 @@ class NMA_SurfaceWaterData(Base): class NMA_SurfaceWaterPhotos(Base): """ Legacy SurfaceWaterPhotos table from NM_Aquifer. + + Note: This table is OUT OF SCOPE for refactoring (not a Thing child). """ __tablename__ = "NMA_SurfaceWaterPhotos" @@ -424,6 +568,8 @@ class NMA_SurfaceWaterPhotos(Base): class NMA_WeatherData(Base): """ Legacy WeatherData table from AMPAPI. + + Note: This table is OUT OF SCOPE for refactoring (not a Thing child). """ __tablename__ = "NMA_WeatherData" @@ -441,6 +587,8 @@ class NMA_WeatherData(Base): class NMA_WeatherPhotos(Base): """ Legacy WeatherPhotos table from NM_Aquifer. + + Note: This table is OUT OF SCOPE for refactoring (not a Thing child). """ __tablename__ = "NMA_WeatherPhotos" @@ -459,22 +607,34 @@ class NMA_WeatherPhotos(Base): class NMA_Soil_Rock_Results(Base): """ Legacy Soil_Rock_Results table from NM_Aquifer. + + Already has Integer PK. Only legacy column renames needed: + - point_id -> nma_point_id """ __tablename__ = "NMA_Soil_Rock_Results" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - point_id: Mapped[Optional[str]] = mapped_column("Point_ID", String(255)) + nma_point_id: Mapped[Optional[str]] = mapped_column("nma_Point_ID", String(255)) sample_type: Mapped[Optional[str]] = mapped_column("Sample Type", String(255)) date_sampled: Mapped[Optional[str]] = mapped_column("Date Sampled", String(255)) d13c: Mapped[Optional[float]] = mapped_column("d13C", Float) d18o: Mapped[Optional[float]] = mapped_column("d18O", Float) sampled_by: Mapped[Optional[str]] = mapped_column("Sampled by", String(255)) - thing_id: Mapped[Optional[int]] = mapped_column( - Integer, ForeignKey("thing.id", ondelete="CASCADE") + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) - thing: Mapped["Thing"] = relationship("Thing") + thing: Mapped["Thing"] = relationship("Thing", back_populates="soil_rock_results") + + @validates("thing_id") + def validate_thing_id(self, key, value): + """Prevent orphan NMA_Soil_Rock_Results - must have a parent Thing.""" + if value is None: + raise ValueError( + "NMA_Soil_Rock_Results requires a parent Thing (thing_id cannot be None)" + ) + return value class NMA_MinorTraceChemistry(Base): @@ -482,83 +642,115 @@ class NMA_MinorTraceChemistry(Base): Legacy MinorandTraceChemistry table from AMPAPI. Stores minor and trace element chemistry results linked to a ChemistrySampleInfo. + + Refactored from UUID PK to Integer PK: + - id: Integer PK (autoincrement) + - nma_global_id: Original UUID PK, now UNIQUE for audit + - chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id + - nma_chemistry_sample_info_uuid: Legacy UUID FK for audit """ __tablename__ = "NMA_MinorTraceChemistry" __table_args__ = ( UniqueConstraint( - "SamplePtID", - "Analyte", + "chemistry_sample_info_id", + "analyte", name="uq_minor_trace_chemistry_sample_analyte", ), ) - global_id: Mapped[uuid.UUID] = mapped_column( - "GlobalID", UUID(as_uuid=True), primary_key=True + # New Integer PK + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + # Legacy UUID PK (now audit column) + nma_global_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_GlobalID", UUID(as_uuid=True), unique=True, nullable=True ) - # FK to ChemistrySampleInfo - required (no orphans) - sample_pt_id: Mapped[uuid.UUID] = mapped_column( - "SamplePtID", - UUID(as_uuid=True), - ForeignKey("NMA_Chemistry_SampleInfo.SamplePtID", ondelete="CASCADE"), + # New Integer FK to ChemistrySampleInfo + chemistry_sample_info_id: Mapped[int] = mapped_column( + Integer, + ForeignKey("NMA_Chemistry_SampleInfo.id", ondelete="CASCADE"), nullable=False, ) - # Legacy columns - sample_point_id: Mapped[Optional[str]] = mapped_column("SamplePointID", String(10)) - analyte: Mapped[Optional[str]] = mapped_column("Analyte", String(50)) - symbol: Mapped[Optional[str]] = mapped_column("Symbol", String(50)) - sample_value: Mapped[Optional[float]] = mapped_column( - "SampleValue", Float, server_default=text("0") + # Legacy UUID FK (for audit) + nma_chemistry_sample_info_uuid: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_chemistry_sample_info_uuid", UUID(as_uuid=True), nullable=True ) - units: Mapped[Optional[str]] = mapped_column("Units", String(50)) - uncertainty: Mapped[Optional[float]] = mapped_column("Uncertainty", Float) + + # Legacy columns (sizes match database schema) + analyte: Mapped[Optional[str]] = mapped_column("analyte", String(50)) + symbol: Mapped[Optional[str]] = mapped_column("symbol", String(10)) + sample_value: Mapped[Optional[float]] = mapped_column("sample_value", Float) + units: Mapped[Optional[str]] = mapped_column("units", String(20)) + uncertainty: Mapped[Optional[float]] = mapped_column("uncertainty", Float) analysis_method: Mapped[Optional[str]] = mapped_column( - "AnalysisMethod", String(255) + "analysis_method", String(100) ) - analysis_date: Mapped[Optional[datetime]] = mapped_column("AnalysisDate", DateTime) - notes: Mapped[Optional[str]] = mapped_column("Notes", String(255)) - volume: Mapped[Optional[int]] = mapped_column( - "Volume", Integer, server_default=text("0") + analysis_date: Mapped[Optional[date]] = mapped_column("analysis_date", Date) + notes: Mapped[Optional[str]] = mapped_column("notes", Text) + volume: Mapped[Optional[int]] = mapped_column("volume", Integer) + volume_unit: Mapped[Optional[str]] = mapped_column("volume_unit", String(20)) + analyses_agency: Mapped[Optional[str]] = mapped_column( + "analyses_agency", String(100) ) - volume_unit: Mapped[Optional[str]] = mapped_column("VolumeUnit", String(50)) - object_id: Mapped[Optional[int]] = mapped_column("OBJECTID", Integer, unique=True) - analyses_agency: Mapped[Optional[str]] = mapped_column("AnalysesAgency", String(50)) - wclab_id: Mapped[Optional[str]] = mapped_column("WCLab_ID", String(25)) # --- Relationships --- chemistry_sample_info: Mapped["NMA_Chemistry_SampleInfo"] = relationship( "NMA_Chemistry_SampleInfo", back_populates="minor_trace_chemistries" ) - @validates("sample_pt_id") - def validate_sample_pt_id(self, key, value): - """Prevent orphan NMA_MinorTraceChemistry - must have a parent ChemistrySampleInfo.""" - if value is None: - raise ValueError( - "NMA_MinorTraceChemistry requires a parent NMA_Chemistry_SampleInfo" - ) - return value - class NMA_Radionuclides(Base): """ Legacy Radionuclides table from NM_Aquifer_Dev_DB. + + Refactored from UUID PK to Integer PK: + - id: Integer PK (autoincrement) + - nma_global_id: Original UUID PK, now UNIQUE for audit + - chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id + - nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit + - nma_sample_point_id: Legacy SamplePointID string + - nma_object_id: Legacy OBJECTID, UNIQUE + - nma_wclab_id: Legacy WCLab_ID """ __tablename__ = "NMA_Radionuclides" - global_id: Mapped[uuid.UUID] = mapped_column( - "GlobalID", UUID(as_uuid=True), primary_key=True + # New Integer PK + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + # Legacy UUID PK (now audit column) + nma_global_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_GlobalID", UUID(as_uuid=True), unique=True, nullable=True ) - sample_pt_id: Mapped[uuid.UUID] = mapped_column( - "SamplePtID", - UUID(as_uuid=True), - ForeignKey("NMA_Chemistry_SampleInfo.SamplePtID", ondelete="CASCADE"), + + # FK to Thing + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + + # New Integer FK to ChemistrySampleInfo + chemistry_sample_info_id: Mapped[int] = mapped_column( + Integer, + ForeignKey("NMA_Chemistry_SampleInfo.id", ondelete="CASCADE"), nullable=False, ) - sample_point_id: Mapped[Optional[str]] = mapped_column("SamplePointID", String(10)) + + # Legacy ID columns (renamed with nma_ prefix) + nma_sample_pt_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_SamplePtID", UUID(as_uuid=True), nullable=True + ) + nma_sample_point_id: Mapped[Optional[str]] = mapped_column( + "nma_SamplePointID", String(10) + ) + nma_object_id: Mapped[Optional[int]] = mapped_column( + "nma_OBJECTID", Integer, unique=True + ) + nma_wclab_id: Mapped[Optional[str]] = mapped_column("nma_WCLab_ID", String(25)) + + # Data columns analyte: Mapped[Optional[str]] = mapped_column("Analyte", String(50)) symbol: Mapped[Optional[str]] = mapped_column("Symbol", String(50)) sample_value: Mapped[Optional[float]] = mapped_column( @@ -577,47 +769,73 @@ class NMA_Radionuclides(Base): "Volume", Integer, server_default=text("0") ) volume_unit: Mapped[Optional[str]] = mapped_column("VolumeUnit", String(50)) - object_id: Mapped[Optional[int]] = mapped_column("OBJECTID", Integer, unique=True) analyses_agency: Mapped[Optional[str]] = mapped_column("AnalysesAgency", String(50)) - wclab_id: Mapped[Optional[str]] = mapped_column("WCLab_ID", String(25)) - # Relationships + thing: Mapped["Thing"] = relationship("Thing", back_populates="radionuclides") chemistry_sample_info: Mapped["NMA_Chemistry_SampleInfo"] = relationship( "NMA_Chemistry_SampleInfo", back_populates="radionuclides" ) @validates("thing_id") def validate_thing_id(self, key, value): + """Prevent orphan NMA_Radionuclides - must have a parent Thing.""" if value is None: raise ValueError( - "NMA_Radionuclides requires a Thing (thing_id cannot be None)" + "NMA_Radionuclides requires a parent Thing (thing_id cannot be None)" ) return value - @validates("sample_pt_id") - def validate_sample_pt_id(self, key, value): + @validates("chemistry_sample_info_id") + def validate_chemistry_sample_info_id(self, key, value): if value is None: - raise ValueError("NMA_Radionuclides requires a SamplePtID") + raise ValueError("NMA_Radionuclides requires a chemistry_sample_info_id") return value class NMA_MajorChemistry(Base): """ Legacy MajorChemistry table from NM_Aquifer_Dev_DB. + + Refactored from UUID PK to Integer PK: + - id: Integer PK (autoincrement) + - nma_global_id: Original UUID PK, now UNIQUE for audit + - chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id + - nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit + - nma_sample_point_id: Legacy SamplePointID string + - nma_object_id: Legacy OBJECTID, UNIQUE + - nma_wclab_id: Legacy WCLab_ID """ __tablename__ = "NMA_MajorChemistry" - global_id: Mapped[uuid.UUID] = mapped_column( - "GlobalID", UUID(as_uuid=True), primary_key=True + # New Integer PK + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + # Legacy UUID PK (now audit column) + nma_global_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_GlobalID", UUID(as_uuid=True), unique=True, nullable=True ) - sample_pt_id: Mapped[uuid.UUID] = mapped_column( - "SamplePtID", - UUID(as_uuid=True), - ForeignKey("NMA_Chemistry_SampleInfo.SamplePtID", ondelete="CASCADE"), + + # New Integer FK to ChemistrySampleInfo + chemistry_sample_info_id: Mapped[int] = mapped_column( + Integer, + ForeignKey("NMA_Chemistry_SampleInfo.id", ondelete="CASCADE"), nullable=False, ) - sample_point_id: Mapped[Optional[str]] = mapped_column("SamplePointID", String(10)) + + # Legacy ID columns (renamed with nma_ prefix) + nma_sample_pt_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_SamplePtID", UUID(as_uuid=True), nullable=True + ) + nma_sample_point_id: Mapped[Optional[str]] = mapped_column( + "nma_SamplePointID", String(10) + ) + nma_object_id: Mapped[Optional[int]] = mapped_column( + "nma_OBJECTID", Integer, unique=True + ) + nma_wclab_id: Mapped[Optional[str]] = mapped_column("nma_WCLab_ID", String(25)) + + # Data columns analyte: Mapped[Optional[str]] = mapped_column("Analyte", String(50)) symbol: Mapped[Optional[str]] = mapped_column("Symbol", String(50)) sample_value: Mapped[Optional[float]] = mapped_column( @@ -634,18 +852,16 @@ class NMA_MajorChemistry(Base): "Volume", Integer, server_default=text("0") ) volume_unit: Mapped[Optional[str]] = mapped_column("VolumeUnit", String(50)) - object_id: Mapped[Optional[int]] = mapped_column("OBJECTID", Integer, unique=True) analyses_agency: Mapped[Optional[str]] = mapped_column("AnalysesAgency", String(50)) - wclab_id: Mapped[Optional[str]] = mapped_column("WCLab_ID", String(25)) chemistry_sample_info: Mapped["NMA_Chemistry_SampleInfo"] = relationship( "NMA_Chemistry_SampleInfo", back_populates="major_chemistries" ) - @validates("sample_pt_id") - def validate_sample_pt_id(self, key, value): + @validates("chemistry_sample_info_id") + def validate_chemistry_sample_info_id(self, key, value): if value is None: - raise ValueError("NMA_MajorChemistry requires a SamplePtID") + raise ValueError("NMA_MajorChemistry requires a chemistry_sample_info_id") return value @@ -653,69 +869,84 @@ class NMA_FieldParameters(Base): """ Legacy FieldParameters table from AMPAPI. Stores field measurements (pH, Temp, etc.) linked to ChemistrySampleInfo. + + Refactored from UUID PK to Integer PK: + - id: Integer PK (autoincrement) + - nma_global_id: Original UUID PK, now UNIQUE for audit + - chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id + - nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit + - nma_sample_point_id: Legacy SamplePointID string + - nma_object_id: Legacy OBJECTID, UNIQUE + - nma_wclab_id: Legacy WCLab_ID """ __tablename__ = "NMA_FieldParameters" __table_args__ = ( - # Explicit Indexes from DDL + # Explicit Indexes (updated for new column names) Index("FieldParameters$AnalysesAgency", "AnalysesAgency"), - Index("FieldParameters$ChemistrySampleInfoFieldParameters", "SamplePtID"), - Index("FieldParameters$FieldParameter", "FieldParameter"), - Index("FieldParameters$SamplePointID", "SamplePointID"), Index( - "FieldParameters$SamplePtID", "SamplePtID" - ), # Note: DDL had two indexes on this col - Index("FieldParameters$WCLab_ID", "WCLab_ID"), - # Unique Indexes (Explicitly named to match DDL) - Index("FieldParameters$GlobalID", "GlobalID", unique=True), - Index("FieldParameters$OBJECTID", "OBJECTID", unique=True), + "FieldParameters$ChemistrySampleInfoFieldParameters", + "chemistry_sample_info_id", + ), + Index("FieldParameters$FieldParameter", "FieldParameter"), + Index("FieldParameters$nma_SamplePointID", "nma_SamplePointID"), + Index("FieldParameters$nma_WCLab_ID", "nma_WCLab_ID"), + # Unique Indexes + Index("FieldParameters$nma_GlobalID", "nma_GlobalID", unique=True), + Index("FieldParameters$nma_OBJECTID", "nma_OBJECTID", unique=True), ) - # Primary Key - global_id: Mapped[uuid.UUID] = mapped_column( - "GlobalID", UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + # New Integer PK + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + # Legacy UUID PK (now audit column) + nma_global_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_GlobalID", UUID(as_uuid=True), unique=True, nullable=True ) - # Foreign Key - sample_pt_id: Mapped[uuid.UUID] = mapped_column( - "SamplePtID", - UUID(as_uuid=True), + # New Integer FK to ChemistrySampleInfo + chemistry_sample_info_id: Mapped[int] = mapped_column( + Integer, ForeignKey( - "NMA_Chemistry_SampleInfo.SamplePtID", + "NMA_Chemistry_SampleInfo.id", onupdate="CASCADE", ondelete="CASCADE", ), nullable=False, ) - # Legacy Columns - sample_point_id: Mapped[Optional[str]] = mapped_column("SamplePointID", String(10)) + # Legacy ID columns (renamed with nma_ prefix) + nma_sample_pt_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "nma_SamplePtID", UUID(as_uuid=True), nullable=True + ) + nma_sample_point_id: Mapped[Optional[str]] = mapped_column( + "nma_SamplePointID", String(10) + ) + nma_object_id: Mapped[int] = mapped_column( + "nma_OBJECTID", Integer, Identity(start=1), nullable=False + ) + nma_wclab_id: Mapped[Optional[str]] = mapped_column("nma_WCLab_ID", String(25)) + + # Data columns field_parameter: Mapped[Optional[str]] = mapped_column("FieldParameter", String(50)) sample_value: Mapped[Optional[float]] = mapped_column( "SampleValue", Float, nullable=True ) units: Mapped[Optional[str]] = mapped_column("Units", String(50)) notes: Mapped[Optional[str]] = mapped_column("Notes", String(255)) - - # Identity Column - object_id: Mapped[int] = mapped_column( - "OBJECTID", Integer, Identity(start=1), nullable=False - ) - analyses_agency: Mapped[Optional[str]] = mapped_column("AnalysesAgency", String(50)) - wc_lab_id: Mapped[Optional[str]] = mapped_column("WCLab_ID", String(25)) # Relationships chemistry_sample_info: Mapped["NMA_Chemistry_SampleInfo"] = relationship( "NMA_Chemistry_SampleInfo", back_populates="field_parameters" ) - @validates("sample_pt_id") - def validate_sample_pt_id(self, key, value): + @validates("chemistry_sample_info_id") + def validate_chemistry_sample_info_id(self, key, value): if value is None: raise ValueError( - "FieldParameter requires a parent ChemistrySampleInfo (SamplePtID)" + "FieldParameter requires a parent ChemistrySampleInfo (chemistry_sample_info_id)" ) return value diff --git a/db/thing.py b/db/thing.py index 8c3f4d315..96fb55361 100644 --- a/db/thing.py +++ b/db/thing.py @@ -47,7 +47,14 @@ from db.thing_geologic_formation_association import ( ThingGeologicFormationAssociation, ) - from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_Stratigraphy + from db.nma_legacy import ( + NMA_AssociatedData, + NMA_Chemistry_SampleInfo, + NMA_HydraulicsData, + NMA_Radionuclides, + NMA_Soil_Rock_Results, + NMA_Stratigraphy, + ) class Thing( @@ -71,6 +78,10 @@ class Thing( nullable=True, comment="To audit where the data came from in NM_Aquifer if it was transferred over", ) + nma_pk_location: Mapped[str] = mapped_column( + nullable=True, + comment="To audit the original NM_Aquifer LocationID if it was transferred over", + ) # TODO: should `name` be unique? name: Mapped[str] = mapped_column( @@ -304,7 +315,7 @@ class Thing( ) ) - # One-To-Many: A Thing can have many ChemistrySampleInfos (legacy NMA data). + # One-To-Many: A Thing can have many NMA_Chemistry_SampleInfo records (legacy NMA data). chemistry_sample_infos: Mapped[List["NMA_Chemistry_SampleInfo"]] = relationship( "NMA_Chemistry_SampleInfo", back_populates="thing", @@ -319,6 +330,38 @@ class Thing( passive_deletes=True, ) + # One-To-Many: A Thing can have many NMA_HydraulicsData records (legacy NMA data). + hydraulics_data: Mapped[List["NMA_HydraulicsData"]] = relationship( + "NMA_HydraulicsData", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # One-To-Many: A Thing can have many NMA_Radionuclides records (legacy NMA data). + radionuclides: Mapped[List["NMA_Radionuclides"]] = relationship( + "NMA_Radionuclides", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # One-To-Many: A Thing can have many NMA_AssociatedData records (legacy NMA data). + associated_data: Mapped[List["NMA_AssociatedData"]] = relationship( + "NMA_AssociatedData", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + # One-To-Many: A Thing can have many NMA_Soil_Rock_Results records (legacy NMA data). + soil_rock_results: Mapped[List["NMA_Soil_Rock_Results"]] = relationship( + "NMA_Soil_Rock_Results", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) + # --- Association Proxies --- assets: AssociationProxy[list["Asset"]] = association_proxy( "asset_associations", "asset" diff --git a/features/admin/well_data_relationships.feature b/features/admin/well_data_relationships.feature new file mode 100644 index 000000000..0eed2d6cb --- /dev/null +++ b/features/admin/well_data_relationships.feature @@ -0,0 +1,120 @@ +@data-integrity +Feature: Well Data Relationships + As a NMBGMR data manager + I need well-related records to always belong to a well + So that data integrity is maintained and orphaned records are prevented + + Background: + Given the Ocotillo database is set up + + # ============================================================================ + # Wells Store Legacy Identifiers + # ============================================================================ + + @wells + Scenario: Wells store their legacy WellID + Given a well record exists + Then the well can store its original NM_Aquifer WellID + And the well can be found by its legacy WellID + + @wells + Scenario: Wells store their legacy LocationID + Given a well record exists + Then the well can store its original NM_Aquifer LocationID + And the well can be found by its legacy LocationID + + # ============================================================================ + # Related Records Require a Well + # ============================================================================ + + @chemistry + Scenario: Chemistry samples require a well + When I try to save chemistry sample information + Then a well must be specified + And orphaned chemistry records are not allowed + + @hydraulics + Scenario: Hydraulic test data requires a well + When I try to save hydraulic test data + Then a well must be specified + And orphaned hydraulic records are not allowed + + @stratigraphy + Scenario: Lithology logs require a well + When I try to save a lithology log + Then a well must be specified + And orphaned lithology records are not allowed + + @radionuclides + Scenario: Radionuclide results require a well + When I try to save radionuclide results + Then a well must be specified + And orphaned radionuclide records are not allowed + + @associated-data + Scenario: Associated data requires a well + When I try to save associated data + Then a well must be specified + And orphaned associated data records are not allowed + + @soil-rock + Scenario: Soil and rock results require a well + When I try to save soil or rock results + Then a well must be specified + And orphaned soil/rock records are not allowed + + # ============================================================================ + # Relationship Navigation + # ============================================================================ + + @relationships + Scenario: A well can access its related records through relationships + Given a well has chemistry sample records + And a well has hydraulic test data + And a well has lithology logs + And a well has radionuclide results + And a well has associated data + And a well has soil and rock results + When I access the well's relationships + Then I can navigate to all related record types + And each relationship returns the correct records + + # ============================================================================ + # Deleting a Well Removes Related Records + # ============================================================================ + + @cascade-delete + Scenario: Deleting a well removes its chemistry samples + Given a well has chemistry sample records + When the well is deleted + Then its chemistry samples are also deleted + + @cascade-delete + Scenario: Deleting a well removes its hydraulic data + Given a well has hydraulic test data + When the well is deleted + Then its hydraulic data is also deleted + + @cascade-delete + Scenario: Deleting a well removes its lithology logs + Given a well has lithology logs + When the well is deleted + Then its lithology logs are also deleted + + @cascade-delete + Scenario: Deleting a well removes its radionuclide results + Given a well has radionuclide results + When the well is deleted + Then its radionuclide results are also deleted + + @cascade-delete + Scenario: Deleting a well removes its associated data + Given a well has associated data + When the well is deleted + Then its associated data is also deleted + + @cascade-delete + Scenario: Deleting a well removes its soil/rock results + Given a well has soil and rock results + When the well is deleted + Then its soil/rock results are also deleted diff --git a/tests/__init__.py b/tests/__init__.py index 32b5d145b..24b7a68f3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +import os from functools import lru_cache from dotenv import load_dotenv @@ -21,6 +22,10 @@ # Use override=True to override conflicting shell environment variables load_dotenv(override=True) +# for safety don't test on the production database port +os.environ["POSTGRES_PORT"] = "5432" +# Always use test database, never dev +os.environ["POSTGRES_DB"] = "ocotilloapi_test" from fastapi.testclient import TestClient from fastapi_pagination import add_pagination diff --git a/tests/features/steps/nma-legacy-relationships.py b/tests/features/steps/nma-legacy-relationships.py new file mode 100644 index 000000000..849e60f39 --- /dev/null +++ b/tests/features/steps/nma-legacy-relationships.py @@ -0,0 +1,703 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Step definitions for NMA Legacy Relationships feature tests. +Tests FK relationships, orphan prevention, and cascade delete behavior +for NMA legacy models. + +Schema notes: +- All models use `id` (Integer, autoincrement) as PK +- Legacy UUID columns renamed with `nma_` prefix (e.g., `nma_global_id`) +- Legacy string columns renamed with `nma_` prefix (e.g., `nma_point_id`) +- Chemistry samples FK to Location (not Thing) +- Other NMA models (hydraulics, stratigraphy, etc.) FK to Thing +- Chemistry children use `chemistry_sample_info_id` (Integer FK) +""" + +import uuid +from datetime import datetime + +from behave import given, when, then +from behave.runner import Context +from sqlalchemy.exc import IntegrityError, StatementError + +from db import Location, Thing +from db.engine import session_ctx +from db.nma_legacy import ( + NMA_Chemistry_SampleInfo, + NMA_HydraulicsData, + NMA_Stratigraphy, + NMA_Radionuclides, + NMA_AssociatedData, + NMA_Soil_Rock_Results, +) + + +@given("the Ocotillo database is set up") +def step_given_database_setup(context: Context): + """Ensure database is ready for testing.""" + # Database connection is handled by session_ctx + context.test_wells = [] + context.test_records = {} + + +@given("a well record exists") +def step_given_well_exists(context: Context): + """Create a test well (Thing) record.""" + with session_ctx() as session: + well = Thing( + name=f"TEST_WELL_{uuid.uuid4().hex[:8]}", + thing_type="water well", + release_status="public", + nma_pk_welldata=str(uuid.uuid4()), + nma_pk_location=str(uuid.uuid4()), + ) + session.add(well) + session.commit() + session.refresh(well) + context.test_well = well + context.test_well_id = well.id + if not hasattr(context, "test_wells"): + context.test_wells = [] + context.test_wells.append(well) + + +@then("the well can store its original NM_Aquifer WellID") +def step_then_well_stores_wellid(context: Context): + """Verify well can store legacy WellID.""" + assert ( + context.test_well.nma_pk_welldata is not None + ), "Well should store legacy WellID" + assert isinstance( + context.test_well.nma_pk_welldata, str + ), "WellID should be a string" + + +@then("the well can be found by its legacy WellID") +def step_then_find_by_wellid(context: Context): + """Verify well can be queried by legacy WellID.""" + with session_ctx() as session: + found_well = ( + session.query(Thing) + .filter(Thing.nma_pk_welldata == context.test_well.nma_pk_welldata) + .first() + ) + assert found_well is not None, "Well should be findable by legacy WellID" + assert found_well.id == context.test_well.id, "Found well should match original" + + +@then("the well can store its original NM_Aquifer LocationID") +def step_then_well_stores_locationid(context: Context): + """Verify well can store legacy LocationID.""" + assert ( + context.test_well.nma_pk_location is not None + ), "Well should store legacy LocationID" + assert isinstance( + context.test_well.nma_pk_location, str + ), "LocationID should be a string" + + +@then("the well can be found by its legacy LocationID") +def step_then_find_by_locationid(context: Context): + """Verify well can be queried by legacy LocationID.""" + with session_ctx() as session: + found_well = ( + session.query(Thing) + .filter(Thing.nma_pk_location == context.test_well.nma_pk_location) + .first() + ) + assert found_well is not None, "Well should be findable by legacy LocationID" + assert found_well.id == context.test_well.id, "Found well should match original" + + +# ============================================================================ +# Chemistry Sample Info +# ============================================================================ + + +@when("I try to save chemistry sample information") +def step_when_save_chemistry(context: Context): + """Attempt to save chemistry sample info without a location.""" + context.orphan_error = None + context.record_saved = False + + try: + with session_ctx() as session: + chemistry = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="TEST001", + location_id=None, # No parent location + collection_date=datetime.now(), + ) + session.add(chemistry) + session.commit() + context.record_saved = True + except (ValueError, IntegrityError, StatementError) as e: + context.orphan_error = e + context.record_saved = False + + +@then("a well must be specified") +def step_then_well_required(context: Context): + """Verify that a well (thing_id) is required.""" + assert not context.record_saved, "Record should not be saved without a well" + assert context.orphan_error is not None, "Should raise error when well is missing" + + +@then("orphaned chemistry records are not allowed") +def step_then_no_orphan_chemistry(context: Context): + """Verify no orphan chemistry records exist (FK to Location).""" + with session_ctx() as session: + orphan_count = ( + session.query(NMA_Chemistry_SampleInfo) + .filter(NMA_Chemistry_SampleInfo.location_id.is_(None)) + .count() + ) + assert orphan_count == 0, f"Found {orphan_count} orphan chemistry records" + + +# ============================================================================ +# Hydraulics Data +# ============================================================================ + + +@when("I try to save hydraulic test data") +def step_when_save_hydraulics(context: Context): + """Attempt to save hydraulic data without a well.""" + context.orphan_error = None + context.record_saved = False + + try: + with session_ctx() as session: + hydraulics = NMA_HydraulicsData( + nma_global_id=uuid.uuid4(), + nma_point_id="TEST001", + thing_id=None, # No parent well + test_top=100, + test_bottom=200, + ) + session.add(hydraulics) + session.commit() + context.record_saved = True + except (ValueError, IntegrityError, StatementError) as e: + context.orphan_error = e + context.record_saved = False + + +@then("orphaned hydraulic records are not allowed") +def step_then_no_orphan_hydraulics(context: Context): + """Verify no orphan hydraulic records exist.""" + with session_ctx() as session: + orphan_count = ( + session.query(NMA_HydraulicsData) + .filter(NMA_HydraulicsData.thing_id.is_(None)) + .count() + ) + assert orphan_count == 0, f"Found {orphan_count} orphan hydraulic records" + + +# ============================================================================ +# NMA_Stratigraphy (Lithology) +# ============================================================================ + + +@when("I try to save a lithology log") +def step_when_save_lithology(context: Context): + """Attempt to save lithology log without a well.""" + context.orphan_error = None + context.record_saved = False + + try: + with session_ctx() as session: + stratigraphy = NMA_Stratigraphy( + nma_global_id=uuid.uuid4(), + nma_point_id="TEST001", + thing_id=None, # No parent well + strat_top=100.0, + strat_bottom=200.0, + ) + session.add(stratigraphy) + session.commit() + context.record_saved = True + except (ValueError, IntegrityError, StatementError) as e: + context.orphan_error = e + context.record_saved = False + + +@then("orphaned lithology records are not allowed") +def step_then_no_orphan_lithology(context: Context): + """Verify no orphan lithology records exist.""" + with session_ctx() as session: + orphan_count = ( + session.query(NMA_Stratigraphy) + .filter(NMA_Stratigraphy.thing_id.is_(None)) + .count() + ) + assert orphan_count == 0, f"Found {orphan_count} orphan lithology records" + + +# ============================================================================ +# Radionuclides +# ============================================================================ + + +@when("I try to save radionuclide results") +def step_when_save_radionuclides(context: Context): + """Attempt to save radionuclide results without a well.""" + context.orphan_error = None + context.record_saved = False + + try: + with session_ctx() as session: + # First create a Location for the chemistry sample (chemistry FKs to Location) + location = Location( + point="POINT(-107.949533 33.809665)", + elevation=2464.9, + release_status="draft", + ) + session.add(location) + session.commit() + session.refresh(location) + + # Create chemistry sample info for the radionuclide + chemistry_sample = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="TEST001", + location_id=location.id, + collection_date=datetime.now(), + ) + session.add(chemistry_sample) + session.commit() + session.refresh(chemistry_sample) + + radionuclide = NMA_Radionuclides( + nma_global_id=uuid.uuid4(), + thing_id=None, # No parent well - this should fail + chemistry_sample_info_id=chemistry_sample.id, + nma_sample_pt_id=chemistry_sample.nma_sample_pt_id, + analyte="U-238", + ) + session.add(radionuclide) + session.commit() + context.record_saved = True + except (ValueError, IntegrityError, StatementError) as e: + context.orphan_error = e + context.record_saved = False + + +@then("orphaned radionuclide records are not allowed") +def step_then_no_orphan_radionuclides(context: Context): + """Verify no orphan radionuclide records exist.""" + with session_ctx() as session: + orphan_count = ( + session.query(NMA_Radionuclides) + .filter(NMA_Radionuclides.thing_id.is_(None)) + .count() + ) + assert orphan_count == 0, f"Found {orphan_count} orphan radionuclide records" + + +# ============================================================================ +# Associated Data +# ============================================================================ + + +@when("I try to save associated data") +def step_when_save_associated_data(context: Context): + """Attempt to save associated data without a well.""" + context.orphan_error = None + context.record_saved = False + + try: + with session_ctx() as session: + associated_data = NMA_AssociatedData( + nma_assoc_id=uuid.uuid4(), + nma_point_id="TEST001", + thing_id=None, # No parent well + notes="Test notes", + ) + session.add(associated_data) + session.commit() + context.record_saved = True + except (ValueError, IntegrityError, StatementError) as e: + context.orphan_error = e + context.record_saved = False + + +@then("orphaned associated data records are not allowed") +def step_then_no_orphan_associated_data(context: Context): + """Verify no orphan associated data records exist.""" + with session_ctx() as session: + orphan_count = ( + session.query(NMA_AssociatedData) + .filter(NMA_AssociatedData.thing_id.is_(None)) + .count() + ) + assert orphan_count == 0, f"Found {orphan_count} orphan associated data records" + + +# ============================================================================ +# Soil/Rock Results +# ============================================================================ + + +@when("I try to save soil or rock results") +def step_when_save_soil_rock(context: Context): + """Attempt to save soil/rock results without a well.""" + context.orphan_error = None + context.record_saved = False + + try: + with session_ctx() as session: + soil_rock = NMA_Soil_Rock_Results( + nma_point_id="TEST001", + thing_id=None, # No parent well + sample_type="Soil", + date_sampled="2025-01-01", + ) + session.add(soil_rock) + session.commit() + context.record_saved = True + except (ValueError, IntegrityError, StatementError) as e: + context.orphan_error = e + context.record_saved = False + + +@then("orphaned soil/rock records are not allowed") +def step_then_no_orphan_soil_rock(context: Context): + """Verify no orphan soil/rock records exist.""" + with session_ctx() as session: + orphan_count = ( + session.query(NMA_Soil_Rock_Results) + .filter(NMA_Soil_Rock_Results.thing_id.is_(None)) + .count() + ) + assert orphan_count == 0, f"Found {orphan_count} orphan soil/rock records" + + +# ============================================================================ +# Relationship Navigation Tests +# ============================================================================ + + +@when("I access the well's relationships") +def step_when_access_relationships(context: Context): + """Access the well's relationships. + + Note: Chemistry samples now FK to Location, not Thing. + Chemistry samples are accessed via Location.chemistry_sample_infos. + """ + with session_ctx() as session: + well = session.query(Thing).filter(Thing.id == context.test_well_id).first() + # Chemistry samples are now on Location, not Thing + # Access via the test location created in step_given_well_has_chemistry + location = None + if hasattr(context, "test_location_id"): + location = ( + session.query(Location) + .filter(Location.id == context.test_location_id) + .first() + ) + + context.well_relationships = { + "chemistry_samples": location.chemistry_sample_infos if location else [], + "hydraulics_data": well.hydraulics_data, + "lithology_logs": well.stratigraphy_logs, + "radionuclides": well.radionuclides, + "associated_data": well.associated_data, + "soil_rock_results": well.soil_rock_results, + } + + +@then("I can navigate to all related record types") +def step_then_navigate_relationships(context: Context): + """Verify all relationship types are accessible.""" + assert "chemistry_samples" in context.well_relationships + assert "hydraulics_data" in context.well_relationships + assert "lithology_logs" in context.well_relationships + assert "radionuclides" in context.well_relationships + assert "associated_data" in context.well_relationships + assert "soil_rock_results" in context.well_relationships + + +@then("each relationship returns the correct records") +def step_then_relationships_correct(context: Context): + """Verify each relationship returns the expected records.""" + assert len(context.well_relationships["chemistry_samples"]) >= 1 + assert len(context.well_relationships["hydraulics_data"]) >= 1 + assert len(context.well_relationships["lithology_logs"]) >= 1 + assert len(context.well_relationships["radionuclides"]) >= 1 + assert len(context.well_relationships["associated_data"]) >= 1 + assert len(context.well_relationships["soil_rock_results"]) >= 1 + + +# ============================================================================ +# Cascade Delete Tests +# ============================================================================ + + +@given("a well has chemistry sample records") +def step_given_well_has_chemistry(context: Context): + """Create chemistry samples for a location associated with a well. + + Note: Chemistry samples now FK to Location (not Thing). + This step creates a Location and associates chemistry samples with it. + """ + if not hasattr(context, "test_well"): + step_given_well_exists(context) + + with session_ctx() as session: + # Create a Location for chemistry samples + location = Location( + point="POINT(-107.949533 33.809665)", + elevation=2464.9, + release_status="draft", + ) + session.add(location) + session.commit() + session.refresh(location) + context.test_location_id = location.id + + chemistry1 = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="TEST001", + location_id=context.test_location_id, + collection_date=datetime.now(), + ) + chemistry2 = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="TEST002", + location_id=context.test_location_id, + collection_date=datetime.now(), + ) + session.add_all([chemistry1, chemistry2]) + session.commit() + context.chemistry_samples = [chemistry1, chemistry2] + + +@given("a well has hydraulic test data") +def step_given_well_has_hydraulics(context: Context): + """Create hydraulic data for a well.""" + if not hasattr(context, "test_well"): + step_given_well_exists(context) + + with session_ctx() as session: + hydraulics = NMA_HydraulicsData( + nma_global_id=uuid.uuid4(), + nma_point_id="TEST001", + thing_id=context.test_well_id, + test_top=100, + test_bottom=200, + ) + session.add(hydraulics) + session.commit() + context.hydraulics_data = hydraulics + + +@given("a well has lithology logs") +def step_given_well_has_lithology(context: Context): + """Create lithology logs for a well.""" + if not hasattr(context, "test_well"): + step_given_well_exists(context) + + with session_ctx() as session: + lithology1 = NMA_Stratigraphy( + nma_global_id=uuid.uuid4(), + nma_point_id="TEST001", + thing_id=context.test_well_id, + strat_top=0.0, + strat_bottom=100.0, + ) + lithology2 = NMA_Stratigraphy( + nma_global_id=uuid.uuid4(), + nma_point_id="TEST001", + thing_id=context.test_well_id, + strat_top=100.0, + strat_bottom=200.0, + ) + session.add_all([lithology1, lithology2]) + session.commit() + context.lithology_logs = [lithology1, lithology2] + + +@given("a well has radionuclide results") +def step_given_well_has_radionuclides(context: Context): + """Create radionuclide results for a well. + + Note: Chemistry samples FK to Location, Radionuclides FK to both Thing and ChemistrySampleInfo. + """ + if not hasattr(context, "test_well"): + step_given_well_exists(context) + + with session_ctx() as session: + # Create a Location for the chemistry sample (chemistry FKs to Location) + location = Location( + point="POINT(-107.949533 33.809665)", + elevation=2464.9, + release_status="draft", + ) + session.add(location) + session.commit() + session.refresh(location) + + chemistry_sample = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="TEST001", + location_id=location.id, + collection_date=datetime.now(), + ) + session.add(chemistry_sample) + session.commit() + session.refresh(chemistry_sample) + + radionuclide = NMA_Radionuclides( + nma_global_id=uuid.uuid4(), + thing_id=context.test_well_id, + chemistry_sample_info_id=chemistry_sample.id, + nma_sample_pt_id=chemistry_sample.nma_sample_pt_id, + analyte="U-238", + ) + session.add(radionuclide) + session.commit() + context.radionuclide_results = radionuclide + + +@given("a well has associated data") +def step_given_well_has_associated_data(context: Context): + """Create associated data for a well.""" + if not hasattr(context, "test_well"): + step_given_well_exists(context) + + with session_ctx() as session: + associated_data = NMA_AssociatedData( + nma_assoc_id=uuid.uuid4(), + nma_point_id="TEST001", + thing_id=context.test_well_id, + notes="Test associated data", + ) + session.add(associated_data) + session.commit() + context.associated_data = associated_data + + +@given("a well has soil and rock results") +def step_given_well_has_soil_rock(context: Context): + """Create soil/rock results for a well.""" + if not hasattr(context, "test_well"): + step_given_well_exists(context) + + with session_ctx() as session: + soil_rock = NMA_Soil_Rock_Results( + nma_point_id="TEST001", + thing_id=context.test_well_id, + sample_type="Soil", + date_sampled="2025-01-01", + ) + session.add(soil_rock) + session.commit() + context.soil_rock_results = soil_rock + + +@when("the well is deleted") +def step_when_well_deleted(context: Context): + """Delete the test well.""" + with session_ctx() as session: + well = session.query(Thing).filter(Thing.id == context.test_well_id).first() + if well: + session.delete(well) + session.commit() + context.well_deleted = True + + +@then("its chemistry samples are also deleted") +def step_then_chemistry_deleted(context: Context): + """Verify chemistry samples are cascade deleted when Location is deleted. + + Note: Chemistry samples now FK to Location (not Thing), so this step + verifies no chemistry samples exist for the test location. + """ + with session_ctx() as session: + # Chemistry samples FK to Location, not Thing + # When a Location is deleted, its chemistry samples cascade delete + remaining = ( + session.query(NMA_Chemistry_SampleInfo) + .filter(NMA_Chemistry_SampleInfo.location_id == context.test_location_id) + .count() + ) + assert remaining == 0, f"Expected 0 chemistry samples, found {remaining}" + + +@then("its hydraulic data is also deleted") +def step_then_hydraulics_deleted(context: Context): + """Verify hydraulic data is cascade deleted.""" + with session_ctx() as session: + remaining = ( + session.query(NMA_HydraulicsData) + .filter(NMA_HydraulicsData.thing_id == context.test_well_id) + .count() + ) + assert remaining == 0, f"Expected 0 hydraulic records, found {remaining}" + + +@then("its lithology logs are also deleted") +def step_then_lithology_deleted(context: Context): + """Verify lithology logs are cascade deleted.""" + with session_ctx() as session: + remaining = ( + session.query(NMA_Stratigraphy) + .filter(NMA_Stratigraphy.thing_id == context.test_well_id) + .count() + ) + assert remaining == 0, f"Expected 0 lithology logs, found {remaining}" + + +@then("its radionuclide results are also deleted") +def step_then_radionuclides_deleted(context: Context): + """Verify radionuclide results are cascade deleted.""" + with session_ctx() as session: + remaining = ( + session.query(NMA_Radionuclides) + .filter(NMA_Radionuclides.thing_id == context.test_well_id) + .count() + ) + assert remaining == 0, f"Expected 0 radionuclide records, found {remaining}" + + +@then("its associated data is also deleted") +def step_then_associated_data_deleted(context: Context): + """Verify associated data is cascade deleted.""" + with session_ctx() as session: + remaining = ( + session.query(NMA_AssociatedData) + .filter(NMA_AssociatedData.thing_id == context.test_well_id) + .count() + ) + assert remaining == 0, f"Expected 0 associated data records, found {remaining}" + + +@then("its soil/rock results are also deleted") +def step_then_soil_rock_deleted(context: Context): + """Verify soil/rock results are cascade deleted.""" + with session_ctx() as session: + remaining = ( + session.query(NMA_Soil_Rock_Results) + .filter(NMA_Soil_Rock_Results.thing_id == context.test_well_id) + .count() + ) + assert remaining == 0, f"Expected 0 soil/rock records, found {remaining}" + + +# ============= EOF ============================================= diff --git a/tests/integration/test_admin_minor_trace_chemistry.py b/tests/integration/test_admin_minor_trace_chemistry.py index 699a83c63..01fbe2ce6 100644 --- a/tests/integration/test_admin_minor_trace_chemistry.py +++ b/tests/integration/test_admin_minor_trace_chemistry.py @@ -30,8 +30,9 @@ from admin.config import create_admin from admin.views.minor_trace_chemistry import MinorTraceChemistryAdmin from db.engine import session_ctx -from db.nma_legacy import NMA_MinorTraceChemistry, NMA_Chemistry_SampleInfo +from db.location import Location, LocationThingAssociation from db.thing import Thing +from db.nma_legacy import NMA_MinorTraceChemistry, NMA_Chemistry_SampleInfo ADMIN_IDENTITY = MinorTraceChemistryAdmin.identity ADMIN_BASE_URL = f"/admin/{ADMIN_IDENTITY}" @@ -61,20 +62,38 @@ def admin_client(admin_app): def minor_trace_chemistry_record(): """Create a minor trace chemistry record for testing.""" with session_ctx() as session: - # First create a Thing (required for NMA_Chemistry_SampleInfo) + # First create a Location + location = Location( + point="POINT(-107.949533 33.809665)", + elevation=2464.9, + release_status="draft", + ) + session.add(location) + session.commit() + session.refresh(location) + + # Create a Thing (required for NMA_Chemistry_SampleInfo) thing = Thing( - name="Integration Test Well", - thing_type="water well", + name="INTTEST-WELL-01", + thing_type="monitoring well", release_status="draft", ) session.add(thing) session.commit() session.refresh(thing) + # Associate Location with Thing + assoc = LocationThingAssociation( + location_id=location.id, + thing_id=thing.id, + ) + session.add(assoc) + session.commit() + # Create parent NMA_Chemistry_SampleInfo sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid.uuid4(), - sample_point_id="INTTEST01", + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="INTTEST01", thing_id=thing.id, ) session.add(sample_info) @@ -83,8 +102,8 @@ def minor_trace_chemistry_record(): # Create MinorTraceChemistry record chemistry = NMA_MinorTraceChemistry( - global_id=uuid.uuid4(), - sample_pt_id=sample_info.sample_pt_id, + nma_global_id=uuid.uuid4(), + chemistry_sample_info_id=sample_info.id, # Integer FK analyte="Arsenic", symbol="As", sample_value=0.005, @@ -101,7 +120,9 @@ def minor_trace_chemistry_record(): # Cleanup session.delete(chemistry) session.delete(sample_info) + session.delete(assoc) session.delete(thing) + session.delete(location) session.commit() @@ -120,7 +141,7 @@ def test_list_view_contains_view_name(self, admin_client): """List view should contain the view name.""" response = admin_client.get(f"{ADMIN_BASE_URL}/list") assert response.status_code == 200 - assert "NMA Minor Trace Chemistry" in response.text + assert "Minor Trace Chemistry" in response.text def test_no_create_button_in_list_view(self, admin_client): """List view should not have a Create button for read-only view.""" @@ -135,7 +156,7 @@ class TestMinorTraceChemistryDetailView: def test_detail_view_returns_200(self, admin_client, minor_trace_chemistry_record): """Detail view should return 200 OK for existing record.""" - pk = str(minor_trace_chemistry_record.global_id) + pk = str(minor_trace_chemistry_record.id) # Integer PK response = admin_client.get(f"{ADMIN_BASE_URL}/detail/{pk}") assert response.status_code == 200, ( f"Expected 200, got {response.status_code}. " @@ -146,7 +167,7 @@ def test_detail_view_shows_analyte( self, admin_client, minor_trace_chemistry_record ): """Detail view should display the analyte.""" - pk = str(minor_trace_chemistry_record.global_id) + pk = str(minor_trace_chemistry_record.id) # Integer PK response = admin_client.get(f"{ADMIN_BASE_URL}/detail/{pk}") assert response.status_code == 200 assert "Arsenic" in response.text @@ -155,7 +176,7 @@ def test_detail_view_shows_parent_relationship( self, admin_client, minor_trace_chemistry_record ): """Detail view should display the parent NMA_Chemistry_SampleInfo.""" - pk = str(minor_trace_chemistry_record.global_id) + pk = str(minor_trace_chemistry_record.id) # Integer PK response = admin_client.get(f"{ADMIN_BASE_URL}/detail/{pk}") assert response.status_code == 200 # The parent relationship should be displayed somehow @@ -164,7 +185,7 @@ def test_detail_view_shows_parent_relationship( def test_detail_view_404_for_nonexistent_record(self, admin_client): """Detail view should return 404 for non-existent record.""" - fake_pk = str(uuid.uuid4()) + fake_pk = "999999999" # Integer PK that doesn't exist response = admin_client.get(f"{ADMIN_BASE_URL}/detail/{fake_pk}") assert response.status_code == 404 @@ -184,7 +205,7 @@ def test_create_endpoint_forbidden(self, admin_client): def test_edit_endpoint_forbidden(self, admin_client, minor_trace_chemistry_record): """Edit endpoint should be forbidden for read-only view.""" - pk = str(minor_trace_chemistry_record.global_id) + pk = str(minor_trace_chemistry_record.id) # Integer PK response = admin_client.get(f"{ADMIN_BASE_URL}/edit/{pk}") # Should be 403 or redirect, not 200 assert response.status_code in ( @@ -197,7 +218,7 @@ def test_delete_endpoint_forbidden( self, admin_client, minor_trace_chemistry_record ): """Delete endpoint should be forbidden for read-only view.""" - pk = str(minor_trace_chemistry_record.global_id) + pk = str(minor_trace_chemistry_record.id) # Integer PK response = admin_client.post( f"{ADMIN_BASE_URL}/delete", data={"pks": [pk]}, diff --git a/tests/integration/test_nma_legacy_relationships.py b/tests/integration/test_nma_legacy_relationships.py new file mode 100644 index 000000000..c34867c49 --- /dev/null +++ b/tests/integration/test_nma_legacy_relationships.py @@ -0,0 +1,684 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Integration tests for NMA Legacy Relationships. + +Tests FK relationships, orphan prevention, and cascade delete behavior +for NMA legacy models. + +Feature: NMA Legacy Data Relationships + As a NMBGMR data manager + I need legacy records to always belong to their parent entities + So that data integrity is maintained and orphaned records are prevented + +Schema notes: +- All models use `id` (Integer, autoincrement) as PK +- Legacy UUID columns renamed with `nma_` prefix (e.g., `nma_global_id`) +- Legacy string columns renamed with `nma_` prefix (e.g., `nma_point_id`) +- Chemistry samples FK to Thing (via thing_id, changed from location_id in 2026-01) +- Other NMA models (hydraulics, stratigraphy, etc.) FK to Thing +- Chemistry children use `chemistry_sample_info_id` (Integer FK) +""" + +import uuid + +import pytest + +from db.engine import session_ctx +from db.location import Location +from db.nma_legacy import ( + NMA_AssociatedData, + NMA_Chemistry_SampleInfo, + NMA_HydraulicsData, + NMA_Radionuclides, + NMA_Soil_Rock_Results, + NMA_Stratigraphy, +) +from db.thing import Thing + +# ============================================================================= +# Fixtures +# ============================================================================= + + +@pytest.fixture +def well_for_relationships(): + """Create a well specifically for relationship testing.""" + with session_ctx() as session: + well = Thing( + name="FK Test Well", + thing_type="water well", + release_status="draft", + nma_pk_welldata="TEST-WELLDATA-GUID-12345", + nma_pk_location="TEST-LOCATION-GUID-67890", + ) + session.add(well) + session.commit() + session.refresh(well) + yield well + # Cleanup: delete the well (should cascade to children) + session.delete(well) + session.commit() + + +@pytest.fixture +def location_for_relationships(): + """Create a location specifically for chemistry relationship testing.""" + with session_ctx() as session: + location = Location( + point="POINT(-107.949533 33.809665)", + elevation=2464.9, + release_status="draft", + ) + session.add(location) + session.commit() + session.refresh(location) + yield location + # Cleanup: delete the location (should cascade to chemistry samples) + session.delete(location) + session.commit() + + +# ============================================================================= +# Wells Store Legacy Identifiers +# ============================================================================= + + +class TestWellsStoreLegacyIdentifiers: + """ + @wells + Scenario: Wells store their legacy WellID + Scenario: Wells store their legacy LocationID + """ + + def test_well_stores_legacy_welldata_id(self): + """Wells can store their original NM_Aquifer WellID.""" + with session_ctx() as session: + well = Thing( + name="Legacy WellID Test", + thing_type="water well", + release_status="draft", + nma_pk_welldata="LEGACY-WELLID-12345", + ) + session.add(well) + session.commit() + session.refresh(well) + + assert well.nma_pk_welldata == "LEGACY-WELLID-12345" + + # Cleanup + session.delete(well) + session.commit() + + def test_well_found_by_legacy_welldata_id(self): + """Wells can be found by their legacy WellID.""" + legacy_id = f"FINDME-WELL-{uuid.uuid4().hex[:8]}" + with session_ctx() as session: + well = Thing( + name="Findable Well", + thing_type="water well", + release_status="draft", + nma_pk_welldata=legacy_id, + ) + session.add(well) + session.commit() + + # Query by legacy ID + found = ( + session.query(Thing).filter(Thing.nma_pk_welldata == legacy_id).first() + ) + assert found is not None + assert found.name == "Findable Well" + + session.delete(well) + session.commit() + + def test_well_stores_legacy_location_id(self): + """Wells can store their original NM_Aquifer LocationID.""" + with session_ctx() as session: + well = Thing( + name="Legacy LocationID Test", + thing_type="water well", + release_status="draft", + nma_pk_location="LEGACY-LOCATIONID-67890", + ) + session.add(well) + session.commit() + session.refresh(well) + + assert well.nma_pk_location == "LEGACY-LOCATIONID-67890" + + # Cleanup + session.delete(well) + session.commit() + + def test_well_found_by_legacy_location_id(self): + """Wells can be found by their legacy LocationID.""" + legacy_id = f"FINDME-LOC-{uuid.uuid4().hex[:8]}" + with session_ctx() as session: + well = Thing( + name="Findable by Location", + thing_type="water well", + release_status="draft", + nma_pk_location=legacy_id, + ) + session.add(well) + session.commit() + + # Query by legacy ID + found = ( + session.query(Thing).filter(Thing.nma_pk_location == legacy_id).first() + ) + assert found is not None + assert found.name == "Findable by Location" + + session.delete(well) + session.commit() + + +# ============================================================================= +# Related Records Require a Well +# ============================================================================= + + +class TestRelatedRecordsRequireWell: + """ + @chemistry, @hydraulics, @stratigraphy, @radionuclides, @associated-data, @soil-rock + Scenarios: Various record types require a parent (thing_id cannot be None) + """ + + def test_chemistry_sample_requires_thing(self): + """ + @chemistry + Scenario: Chemistry samples require a thing (via thing_id FK) + + Note: Chemistry samples FK to Thing (changed from Location in 2026-01). + """ + from sqlalchemy.exc import IntegrityError, ProgrammingError + + with session_ctx() as session: + record = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="ORPHAN-CHEM", + # No thing_id - should fail on commit + ) + session.add(record) + # pg8000 raises ProgrammingError for NOT NULL violations (error code 23502) + with pytest.raises((IntegrityError, ProgrammingError, ValueError)): + session.commit() + session.rollback() + + def test_hydraulics_data_requires_well(self): + """ + @hydraulics + Scenario: Hydraulic test data requires a well + """ + with session_ctx() as session: + with pytest.raises(ValueError, match="requires a parent Thing"): + record = NMA_HydraulicsData( + nma_global_id=uuid.uuid4(), + nma_point_id="ORPHANHYD", + thing_id=None, # This should raise ValueError + ) + session.add(record) + session.flush() + + def test_stratigraphy_requires_well(self): + """ + @stratigraphy + Scenario: Lithology logs require a well + """ + with session_ctx() as session: + with pytest.raises(ValueError, match="requires a parent Thing"): + record = NMA_Stratigraphy( + nma_global_id=uuid.uuid4(), + nma_point_id="ORPHSTRAT", + thing_id=None, # This should raise ValueError + ) + session.add(record) + session.flush() + + def test_radionuclides_requires_well(self): + """ + @radionuclides + Scenario: Radionuclide results require a well + """ + with session_ctx() as session: + with pytest.raises(ValueError, match="requires a parent Thing"): + record = NMA_Radionuclides( + nma_sample_pt_id=uuid.uuid4(), + thing_id=None, # This should raise ValueError + ) + session.add(record) + session.flush() + + def test_associated_data_requires_well(self): + """ + @associated-data + Scenario: Associated data requires a well + """ + with session_ctx() as session: + with pytest.raises(ValueError, match="requires a parent Thing"): + record = NMA_AssociatedData( + nma_point_id="ORPHAN-ASSOC", + thing_id=None, # This should raise ValueError + ) + session.add(record) + session.flush() + + def test_soil_rock_results_requires_well(self): + """ + @soil-rock + Scenario: Soil and rock results require a well + """ + with session_ctx() as session: + with pytest.raises(ValueError, match="requires a parent Thing"): + record = NMA_Soil_Rock_Results( + nma_point_id="ORPHAN-SOIL", + thing_id=None, # This should raise ValueError + ) + session.add(record) + session.flush() + + +# ============================================================================= +# Relationship Navigation +# ============================================================================= + + +class TestRelationshipNavigation: + """ + @relationships + Scenario: A well can access its related records through relationships + """ + + def test_thing_navigates_to_chemistry_samples(self, well_for_relationships): + """Thing can navigate to its chemistry sample records. + + Note: Chemistry samples FK to Thing (changed from Location in 2026-01). + """ + with session_ctx() as session: + well = session.merge(well_for_relationships) + + # Create a chemistry sample for this thing + sample = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="NAVCHEM01", # Max 10 chars + thing_id=well.id, + ) + session.add(sample) + session.commit() + session.refresh(well) + + # Navigate through relationship + assert hasattr(well, "chemistry_sample_infos") + assert len(well.chemistry_sample_infos) >= 1 + assert any( + s.nma_sample_point_id == "NAVCHEM01" + for s in well.chemistry_sample_infos + ) + + def test_well_navigates_to_hydraulics_data(self, well_for_relationships): + """Well can navigate to its hydraulic test data.""" + with session_ctx() as session: + well = session.merge(well_for_relationships) + + # Create hydraulics data for this well + hydraulics = NMA_HydraulicsData( + nma_global_id=uuid.uuid4(), + nma_point_id="NAVHYD01", # Max 10 chars + thing_id=well.id, + test_top=0, + test_bottom=100, + ) + session.add(hydraulics) + session.commit() + session.refresh(well) + + # Navigate through relationship + assert hasattr(well, "hydraulics_data") + assert len(well.hydraulics_data) >= 1 + assert any(h.nma_point_id == "NAVHYD01" for h in well.hydraulics_data) + + def test_well_navigates_to_stratigraphy_logs(self, well_for_relationships): + """Well can navigate to its lithology logs.""" + with session_ctx() as session: + well = session.merge(well_for_relationships) + + # Create stratigraphy log for this well + strat = NMA_Stratigraphy( + nma_global_id=uuid.uuid4(), + nma_point_id="NAVSTRAT1", # Max 10 chars + thing_id=well.id, + strat_top=0, + strat_bottom=10, + ) + session.add(strat) + session.commit() + session.refresh(well) + + # Navigate through relationship + assert hasattr(well, "stratigraphy_logs") + assert len(well.stratigraphy_logs) >= 1 + assert any(s.nma_point_id == "NAVSTRAT1" for s in well.stratigraphy_logs) + + def test_well_navigates_to_radionuclides(self, well_for_relationships): + """Well can navigate to its radionuclide results.""" + with session_ctx() as session: + well = session.merge(well_for_relationships) + + # Create a chemistry sample for the thing (chemistry FKs to Thing) + chem_sample = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="NAVRAD01", # Required, max 10 chars + thing_id=well.id, + ) + session.add(chem_sample) + session.commit() + session.refresh(chem_sample) + + # Create radionuclide record for this well using the chemistry_sample_info_id + radio = NMA_Radionuclides( + nma_global_id=uuid.uuid4(), + chemistry_sample_info_id=chem_sample.id, + nma_sample_pt_id=chem_sample.nma_sample_pt_id, + thing_id=well.id, + ) + session.add(radio) + session.commit() + session.refresh(well) + + # Navigate through relationship + assert hasattr(well, "radionuclides") + assert len(well.radionuclides) >= 1 + + def test_well_navigates_to_associated_data(self, well_for_relationships): + """Well can navigate to its associated data.""" + with session_ctx() as session: + well = session.merge(well_for_relationships) + + # Create associated data for this well + assoc = NMA_AssociatedData( + nma_assoc_id=uuid.uuid4(), + nma_point_id="NAVASSOC1", # Max 10 chars + thing_id=well.id, + ) + session.add(assoc) + session.commit() + session.refresh(well) + + # Navigate through relationship + assert hasattr(well, "associated_data") + assert len(well.associated_data) >= 1 + assert any(a.nma_point_id == "NAVASSOC1" for a in well.associated_data) + + def test_well_navigates_to_soil_rock_results(self, well_for_relationships): + """Well can navigate to its soil/rock results.""" + with session_ctx() as session: + well = session.merge(well_for_relationships) + + # Create soil/rock result for this well + soil = NMA_Soil_Rock_Results( + nma_point_id="NAV-SOIL-01", + thing_id=well.id, + ) + session.add(soil) + session.commit() + session.refresh(well) + + # Navigate through relationship + assert hasattr(well, "soil_rock_results") + assert len(well.soil_rock_results) >= 1 + assert any(s.nma_point_id == "NAV-SOIL-01" for s in well.soil_rock_results) + + +# ============================================================================= +# Deleting a Well Removes Related Records (Cascade Delete) +# ============================================================================= + + +class TestCascadeDelete: + """ + @cascade-delete + Scenarios: Deleting a well removes its related records + """ + + def test_deleting_thing_cascades_to_chemistry_samples(self): + """ + @cascade-delete + Scenario: Deleting a thing removes its chemistry samples + + Note: Chemistry samples FK to Thing (changed from Location in 2026-01). + """ + with session_ctx() as session: + # Create thing with chemistry sample + thing = Thing( + name="Cascade Chemistry Test", + thing_type="water well", + release_status="draft", + ) + session.add(thing) + session.commit() + + sample = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="CASCCHEM1", # Max 10 chars + thing_id=thing.id, + ) + session.add(sample) + session.commit() + sample_id = sample.id # Integer PK + + # Delete the thing + session.delete(thing) + session.commit() + + # Clear session cache to ensure fresh DB query + session.expire_all() + + # Verify chemistry sample was also deleted + orphan = session.get(NMA_Chemistry_SampleInfo, sample_id) + assert orphan is None, "Chemistry sample should be deleted with thing" + + def test_deleting_well_cascades_to_hydraulics_data(self): + """ + @cascade-delete + Scenario: Deleting a well removes its hydraulic data + """ + with session_ctx() as session: + # Create well with hydraulics data + well = Thing( + name="Cascade Hydraulics Test", + thing_type="water well", + release_status="draft", + ) + session.add(well) + session.commit() + + hydraulics = NMA_HydraulicsData( + nma_global_id=uuid.uuid4(), + nma_point_id="CASCHYD01", # Max 10 chars + thing_id=well.id, + test_top=0, + test_bottom=100, + ) + session.add(hydraulics) + session.commit() + hyd_id = hydraulics.id # Integer PK + + # Delete the well + session.delete(well) + session.commit() + + # Clear session cache to ensure fresh DB query + session.expire_all() + + # Verify hydraulics data was also deleted + orphan = session.get(NMA_HydraulicsData, hyd_id) + assert orphan is None, "Hydraulics data should be deleted with well" + + def test_deleting_well_cascades_to_stratigraphy_logs(self): + """ + @cascade-delete + Scenario: Deleting a well removes its lithology logs + """ + with session_ctx() as session: + # Create well with stratigraphy log + well = Thing( + name="Cascade Stratigraphy Test", + thing_type="water well", + release_status="draft", + ) + session.add(well) + session.commit() + + strat = NMA_Stratigraphy( + nma_global_id=uuid.uuid4(), + nma_point_id="CASCSTRAT", # Max 10 chars + thing_id=well.id, + strat_top=0, + strat_bottom=10, + ) + session.add(strat) + session.commit() + strat_id = strat.id # Integer PK + + # Delete the well + session.delete(well) + session.commit() + + # Clear session cache to ensure fresh DB query + session.expire_all() + + # Verify stratigraphy was also deleted + orphan = session.get(NMA_Stratigraphy, strat_id) + assert orphan is None, "Stratigraphy log should be deleted with well" + + def test_deleting_well_cascades_to_radionuclides(self): + """ + @cascade-delete + Scenario: Deleting a well removes its radionuclide results + """ + with session_ctx() as session: + # Create well with radionuclide record + well = Thing( + name="Cascade Radionuclides Test", + thing_type="water well", + release_status="draft", + ) + session.add(well) + session.commit() + + # Create a chemistry sample for the thing (chemistry FKs to Thing) + chem_sample = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid.uuid4(), + nma_sample_point_id="CASCRAD01", # Required, max 10 chars + thing_id=well.id, + ) + session.add(chem_sample) + session.commit() + session.refresh(chem_sample) + + # Create radionuclide record using the chemistry_sample_info_id + radio = NMA_Radionuclides( + nma_global_id=uuid.uuid4(), + chemistry_sample_info_id=chem_sample.id, + nma_sample_pt_id=chem_sample.nma_sample_pt_id, + thing_id=well.id, + ) + session.add(radio) + session.commit() + radio_id = radio.id # Integer PK + + # Delete the well + session.delete(well) + session.commit() + + # Clear session cache to ensure fresh DB query + session.expire_all() + + # Verify radionuclide record was also deleted + orphan = session.get(NMA_Radionuclides, radio_id) + assert orphan is None, "Radionuclide record should be deleted with well" + + def test_deleting_well_cascades_to_associated_data(self): + """ + @cascade-delete + Scenario: Deleting a well removes its associated data + """ + with session_ctx() as session: + # Create well with associated data + well = Thing( + name="Cascade Associated Test", + thing_type="water well", + release_status="draft", + ) + session.add(well) + session.commit() + + assoc = NMA_AssociatedData( + nma_assoc_id=uuid.uuid4(), + nma_point_id="CASCASSOC", # Max 10 chars + thing_id=well.id, + ) + session.add(assoc) + session.commit() + assoc_id = assoc.id # Integer PK + + # Delete the well + session.delete(well) + session.commit() + + # Clear session cache to ensure fresh DB query + session.expire_all() + + # Verify associated data was also deleted + orphan = session.get(NMA_AssociatedData, assoc_id) + assert orphan is None, "Associated data should be deleted with well" + + def test_deleting_well_cascades_to_soil_rock_results(self): + """ + @cascade-delete + Scenario: Deleting a well removes its soil/rock results + """ + with session_ctx() as session: + # Create well with soil/rock results + well = Thing( + name="Cascade Soil Rock Test", + thing_type="water well", + release_status="draft", + ) + session.add(well) + session.commit() + + soil = NMA_Soil_Rock_Results( + nma_point_id="CASCSOIL1", + thing_id=well.id, + ) + session.add(soil) + session.commit() + soil_id = soil.id + + # Delete the well + session.delete(well) + session.commit() + + # Clear session cache to ensure fresh DB query + session.expire_all() + + # Verify soil/rock results were also deleted + orphan = session.get(NMA_Soil_Rock_Results, soil_id) + assert orphan is None, "Soil/rock results should be deleted with well" diff --git a/tests/test_admin_minor_trace_chemistry.py b/tests/test_admin_minor_trace_chemistry.py index fbc4937d8..4ec1705d8 100644 --- a/tests/test_admin_minor_trace_chemistry.py +++ b/tests/test_admin_minor_trace_chemistry.py @@ -18,6 +18,12 @@ These tests verify the admin view is properly configured without requiring a running server or database. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy GlobalID UUID (UNIQUE) +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_chemistry_sample_info_uuid: Legacy UUID FK (for audit) """ import pytest @@ -37,15 +43,15 @@ def test_minor_trace_chemistry_view_is_registered(self): admin = create_admin(app) view_names = [v.name for v in admin._views] - assert "NMA Minor Trace Chemistry" in view_names, ( - f"Expected 'NMA Minor Trace Chemistry' to be registered in admin views. " + assert "Minor Trace Chemistry" in view_names, ( + f"Expected 'Minor Trace Chemistry' to be registered in admin views. " f"Found: {view_names}" ) def test_view_has_correct_label(self): """View should have proper label for sidebar display.""" view = MinorTraceChemistryAdmin(NMA_MinorTraceChemistry) - assert view.label == "NMA Minor Trace Chemistry" + assert view.label == "Minor Trace Chemistry" def test_class_has_flask_icon_configured(self): """View class should have flask icon configured for chemistry data.""" @@ -106,10 +112,9 @@ def test_list_fields_include_required_columns(self, view): field_names.append(getattr(f, "name", str(f))) required_columns = [ - "global_id", + "id", # Integer PK + "nma_global_id", # Legacy UUID "chemistry_sample_info", # HasOne relationship to parent - "sample_pt_id", - "sample_point_id", "analyte", "sample_value", "units", @@ -147,9 +152,9 @@ def test_form_includes_all_chemistry_fields(self): # Check the class-level configuration # Note: chemistry_sample_info is a HasOne field, not a string expected_string_fields = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", # Integer PK + "nma_global_id", # Legacy GlobalID + "nma_chemistry_sample_info_uuid", # Legacy UUID FK "analyte", "symbol", "sample_value", @@ -160,9 +165,7 @@ def test_form_includes_all_chemistry_fields(self): "notes", "volume", "volume_unit", - "object_id", "analyses_agency", - "wclab_id", ] configured_fields = MinorTraceChemistryAdmin.fields @@ -181,15 +184,34 @@ def test_form_includes_all_chemistry_fields(self): def test_field_labels_are_human_readable(self, view): """Field labels should be human-readable.""" - assert view.field_labels.get("global_id") == "GlobalID" - assert view.field_labels.get("sample_value") == "SampleValue" - assert view.field_labels.get("analysis_date") == "AnalysisDate" + assert view.field_labels.get("id") == "ID" + assert view.field_labels.get("nma_global_id") == "NMA GlobalID (Legacy)" + assert view.field_labels.get("sample_value") == "Sample Value" + assert view.field_labels.get("analysis_date") == "Analysis Date" def test_searchable_fields_include_key_fields(self, view): """Searchable fields should include commonly searched columns.""" + assert "nma_global_id" in view.searchable_fields assert "analyte" in view.searchable_fields assert "symbol" in view.searchable_fields assert "analyses_agency" in view.searchable_fields +class TestMinorTraceChemistryAdminIntegerPK: + """Tests for Integer PK configuration.""" + + @pytest.fixture + def view(self): + """Create a MinorTraceChemistryAdmin instance for testing.""" + return MinorTraceChemistryAdmin(NMA_MinorTraceChemistry) + + def test_pk_attr_is_id(self, view): + """Primary key attribute should be 'id'.""" + assert view.pk_attr == "id" + + def test_pk_type_is_int(self, view): + """Primary key type should be int.""" + assert view.pk_type == int + + # ============= EOF ============================================= diff --git a/tests/test_associated_data_legacy.py b/tests/test_associated_data_legacy.py index 7919b0493..78a5eb1e7 100644 --- a/tests/test_associated_data_legacy.py +++ b/tests/test_associated_data_legacy.py @@ -17,13 +17,13 @@ Unit tests for NMA_AssociatedData legacy model. These tests verify the migration of columns from the legacy NMA_AssociatedData table. -Migrated columns: -- LocationId -> location_id -- PointID -> point_id -- AssocID -> assoc_id -- Notes -> notes -- Formation -> formation -- OBJECTID -> object_id + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_assoc_id: Legacy AssocID UUID (UNIQUE) +- nma_location_id: Legacy LocationId UUID (UNIQUE) +- nma_point_id: Legacy PointID string +- nma_object_id: Legacy OBJECTID (UNIQUE) """ from uuid import uuid4 @@ -36,47 +36,118 @@ def test_create_associated_data_all_fields(water_well_thing): """Test creating an associated data record with all fields.""" with session_ctx() as session: record = NMA_AssociatedData( - location_id=uuid4(), - point_id="AA-0001", - assoc_id=uuid4(), + nma_location_id=uuid4(), + nma_point_id="AA-0001", + nma_assoc_id=uuid4(), notes="Legacy notes", formation="TEST", - object_id=42, + nma_object_id=42, thing_id=water_well_thing.id, ) session.add(record) session.commit() session.refresh(record) - assert record.assoc_id is not None - assert record.location_id is not None - assert record.point_id == "AA-0001" + assert record.id is not None # Integer PK auto-generated + assert record.nma_assoc_id is not None + assert record.nma_location_id is not None + assert record.nma_point_id == "AA-0001" assert record.notes == "Legacy notes" assert record.formation == "TEST" - assert record.object_id == 42 + assert record.nma_object_id == 42 assert record.thing_id == water_well_thing.id session.delete(record) session.commit() -def test_create_associated_data_minimal(): +def test_create_associated_data_minimal(water_well_thing): """Test creating an associated data record with required fields only.""" with session_ctx() as session: - record = NMA_AssociatedData(assoc_id=uuid4()) + well = session.merge(water_well_thing) + record = NMA_AssociatedData(nma_assoc_id=uuid4(), thing_id=well.id) session.add(record) session.commit() session.refresh(record) - assert record.assoc_id is not None - assert record.location_id is None - assert record.point_id is None + assert record.id is not None # Integer PK auto-generated + assert record.nma_assoc_id is not None + assert record.thing_id == well.id + assert record.nma_location_id is None + assert record.nma_point_id is None assert record.notes is None assert record.formation is None - assert record.object_id is None + assert record.nma_object_id is None + + session.delete(record) + session.commit() + + +# ===================== FK Enforcement tests (Issue #363) ========================== + + +def test_associated_data_validator_rejects_none_thing_id(): + """NMA_AssociatedData validator rejects None thing_id.""" + import pytest + + with pytest.raises(ValueError, match="requires a parent Thing"): + NMA_AssociatedData( + nma_assoc_id=uuid4(), + nma_point_id="ORPHAN-TEST", + thing_id=None, + ) + + +def test_associated_data_thing_id_not_nullable(): + """NMA_AssociatedData.thing_id column is NOT NULL.""" + col = NMA_AssociatedData.__table__.c.thing_id + assert col.nullable is False, "thing_id should be NOT NULL" + + +def test_associated_data_fk_has_cascade(): + """NMA_AssociatedData.thing_id FK has ondelete=CASCADE.""" + col = NMA_AssociatedData.__table__.c.thing_id + fk = list(col.foreign_keys)[0] + assert fk.ondelete == "CASCADE" + + +def test_associated_data_back_populates_thing(water_well_thing): + """NMA_AssociatedData.thing navigates back to Thing.""" + with session_ctx() as session: + well = session.merge(water_well_thing) + record = NMA_AssociatedData( + nma_assoc_id=uuid4(), + nma_point_id="BPASSOC01", # Max 10 chars + thing_id=well.id, + ) + session.add(record) + session.commit() + session.refresh(record) + + assert record.thing is not None + assert record.thing.id == well.id session.delete(record) session.commit() +# ===================== Integer PK tests ========================== + + +def test_associated_data_has_integer_pk(): + """NMA_AssociatedData.id is Integer PK.""" + from sqlalchemy import Integer + + col = NMA_AssociatedData.__table__.c.id + assert col.primary_key is True + assert isinstance(col.type, Integer) + + +def test_associated_data_nma_assoc_id_is_unique(): + """NMA_AssociatedData.nma_assoc_id is UNIQUE.""" + # Use database column name (nma_AssocID), not Python attribute name (nma_assoc_id) + col = NMA_AssociatedData.__table__.c["nma_AssocID"] + assert col.unique is True + + # ============= EOF ============================================= diff --git a/tests/test_chemistry_sampleinfo_legacy.py b/tests/test_chemistry_sampleinfo_legacy.py index 2648befc0..9590b12de 100644 --- a/tests/test_chemistry_sampleinfo_legacy.py +++ b/tests/test_chemistry_sampleinfo_legacy.py @@ -17,25 +17,17 @@ Unit tests for NMA_Chemistry_SampleInfo legacy model. These tests verify the migration of columns from the legacy Chemistry_SampleInfo table. -Migrated columns: -- OBJECTID -> object_id -- SamplePointID -> sample_point_id -- SamplePtID -> sample_pt_id -- WCLab_ID -> wclab_id -- CollectionDate -> collection_date -- CollectionMethod -> collection_method -- CollectedBy -> collected_by -- AnalysesAgency -> analyses_agency -- SampleType -> sample_type -- SampleMaterialNotH2O -> sample_material_not_h2o -- WaterType -> water_type -- StudySample -> study_sample -- DataSource -> data_source -- DataQuality -> data_quality -- PublicRelease -> public_release -- AddedDaytoDate -> added_day_to_date -- AddedMonthDaytoDate -> added_month_day_to_date -- SampleNotes -> sample_notes + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_sample_pt_id: Legacy SamplePtID UUID (UNIQUE) +- nma_sample_point_id: Legacy SamplePointID string +- nma_wclab_id: Legacy WCLab_ID string +- nma_location_id: Legacy LocationId UUID (for audit trail) +- nma_object_id: Legacy OBJECTID (UNIQUE) + +FK Change (2026-01): +- thing_id: Integer FK to Thing.id """ from datetime import datetime @@ -58,10 +50,10 @@ def test_create_chemistry_sampleinfo_all_fields(water_well_thing): """Test creating a chemistry sample info record with all fields.""" with session_ctx() as session: record = NMA_Chemistry_SampleInfo( - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, - wclab_id="LAB-123", + nma_wclab_id="LAB-123", collection_date=datetime(2024, 1, 1, 10, 30, 0), collection_method="Grab", collected_by="Tech", @@ -81,9 +73,10 @@ def test_create_chemistry_sampleinfo_all_fields(water_well_thing): session.commit() session.refresh(record) - assert record.sample_pt_id is not None - assert record.sample_point_id is not None - assert record.wclab_id == "LAB-123" + assert record.id is not None # Integer PK auto-generated + assert record.nma_sample_pt_id is not None + assert record.nma_sample_point_id is not None + assert record.nma_wclab_id == "LAB-123" assert record.collection_date == datetime(2024, 1, 1, 10, 30, 0) assert record.sample_material_not_h2o == "Yes" assert record.study_sample == "Yes" @@ -96,16 +89,17 @@ def test_create_chemistry_sampleinfo_minimal(water_well_thing): """Test creating a chemistry sample info record with minimal fields.""" with session_ctx() as session: record = NMA_Chemistry_SampleInfo( - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(record) session.commit() session.refresh(record) - assert record.sample_pt_id is not None - assert record.sample_point_id is not None + assert record.id is not None # Integer PK auto-generated + assert record.nma_sample_pt_id is not None + assert record.nma_sample_point_id is not None assert record.collection_date is None session.delete(record) @@ -113,21 +107,22 @@ def test_create_chemistry_sampleinfo_minimal(water_well_thing): # ===================== READ tests ========================== -def test_read_chemistry_sampleinfo_by_object_id(water_well_thing): - """Test reading a chemistry sample info record by OBJECTID.""" +def test_read_chemistry_sampleinfo_by_id(water_well_thing): + """Test reading a chemistry sample info record by Integer ID.""" with session_ctx() as session: record = NMA_Chemistry_SampleInfo( - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(record) session.commit() - fetched = session.get(NMA_Chemistry_SampleInfo, record.sample_pt_id) + fetched = session.get(NMA_Chemistry_SampleInfo, record.id) assert fetched is not None - assert fetched.sample_pt_id == record.sample_pt_id - assert fetched.sample_point_id == record.sample_point_id + assert fetched.id == record.id + assert fetched.nma_sample_pt_id == record.nma_sample_pt_id + assert fetched.nma_sample_point_id == record.nma_sample_point_id session.delete(record) session.commit() @@ -138,8 +133,8 @@ def test_update_chemistry_sampleinfo(water_well_thing): """Test updating a chemistry sample info record.""" with session_ctx() as session: record = NMA_Chemistry_SampleInfo( - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(record) @@ -162,17 +157,18 @@ def test_delete_chemistry_sampleinfo(water_well_thing): """Test deleting a chemistry sample info record.""" with session_ctx() as session: record = NMA_Chemistry_SampleInfo( - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(record) session.commit() + record_id = record.id session.delete(record) session.commit() - fetched = session.get(NMA_Chemistry_SampleInfo, record.sample_pt_id) + fetched = session.get(NMA_Chemistry_SampleInfo, record_id) assert fetched is None @@ -180,10 +176,11 @@ def test_delete_chemistry_sampleinfo(water_well_thing): def test_chemistry_sampleinfo_has_all_migrated_columns(): """Test that the model has all expected columns.""" expected_columns = [ - "sample_point_id", - "sample_pt_id", - "wclab_id", - "thing_id", + "id", + "nma_sample_point_id", + "nma_sample_pt_id", + "nma_wclab_id", + "thing_id", # Integer FK to Thing.id "collection_date", "collection_method", "collected_by", @@ -198,8 +195,8 @@ def test_chemistry_sampleinfo_has_all_migrated_columns(): "added_day_to_date", "added_month_day_to_date", "sample_notes", - "object_id", - "location_id", + "nma_object_id", + "nma_location_id", ] for column in expected_columns: @@ -213,4 +210,23 @@ def test_chemistry_sampleinfo_table_name(): assert NMA_Chemistry_SampleInfo.__tablename__ == "NMA_Chemistry_SampleInfo" +# ===================== Integer PK tests ========================== + + +def test_chemistry_sampleinfo_has_integer_pk(): + """NMA_Chemistry_SampleInfo.id is Integer PK.""" + from sqlalchemy import Integer + + col = NMA_Chemistry_SampleInfo.__table__.c.id + assert col.primary_key is True + assert isinstance(col.type, Integer) + + +def test_chemistry_sampleinfo_nma_sample_pt_id_is_unique(): + """NMA_Chemistry_SampleInfo.nma_sample_pt_id is UNIQUE.""" + # Use database column name (nma_SamplePtID), not Python attribute name + col = NMA_Chemistry_SampleInfo.__table__.c["nma_SamplePtID"] + assert col.unique is True + + # ============= EOF ============================================= diff --git a/tests/test_field_parameters_legacy.py b/tests/test_field_parameters_legacy.py index aa04174d0..e69de29bb 100644 --- a/tests/test_field_parameters_legacy.py +++ b/tests/test_field_parameters_legacy.py @@ -1,357 +0,0 @@ -""" -Unit tests for NMA_FieldParameters legacy model. - -These tests verify the migration of columns from the legacy NMA_FieldParameters table. -Migrated columns (excluding SSMA_TimeStamp): -- SamplePtID -> sample_pt_id -- SamplePointID -> sample_point_id -- FieldParameter -> field_parameter -- SampleValue -> sample_value -- Units -> units -- Notes -> notes -- OBJECTID -> object_id -- GlobalID -> global_id -- AnalysesAgency -> analyses_agency -- WCLab_ID -> wc_lab_id -""" - -from uuid import uuid4 - -import pytest -from sqlalchemy import select, inspect -from sqlalchemy.exc import IntegrityError, ProgrammingError - -from db.engine import session_ctx -from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_FieldParameters - - -def _next_sample_point_id() -> str: - return f"SP-{uuid4().hex[:7]}" - - -def _create_sample_info(session, water_well_thing) -> NMA_Chemistry_SampleInfo: - sample = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), - thing_id=water_well_thing.id, - ) - session.add(sample) - session.commit() - return sample - - -# ===================== Table and Column Existence Tests ========================== - - -def test_field_parameters_has_all_migrated_columns(): - """ - VERIFIES: The SQLAlchemy model matches the migration mapping contract. - This ensures all Python-side attribute names exist as expected in the ORM. - """ - mapper = inspect(NMA_FieldParameters) - actual_columns = [column.key for column in mapper.attrs] - - expected_columns = [ - "global_id", - "sample_pt_id", - "sample_point_id", - "field_parameter", - "sample_value", - "units", - "notes", - "object_id", - "analyses_agency", - "wc_lab_id", - ] - - for column in expected_columns: - assert column in actual_columns, f"Model is missing expected column: {column}" - - -def test_field_parameters_table_name(): - """Test that the table name follows convention.""" - assert NMA_FieldParameters.__tablename__ == "NMA_FieldParameters" - - -# ===================== Functional & CRUD Tests ========================= - - -def test_field_parameters_persistence(water_well_thing): - """ - Verifies that data correctly persists and retrieves for the core columns. - This confirms the Postgres data types (REAL, UUID, VARCHAR) are compatible. - """ - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - test_global_id = uuid4() - new_fp = NMA_FieldParameters( - global_id=test_global_id, - sample_pt_id=sample_info.sample_pt_id, - sample_point_id="PT-123", - field_parameter="pH", - sample_value=7.4, - units="SU", - notes="Legacy migration verification", - analyses_agency="NMA Agency", - wc_lab_id="WCLAB-01", - ) - - session.add(new_fp) - session.commit() - session.expire_all() - - retrieved = session.get(NMA_FieldParameters, test_global_id) - assert retrieved.sample_value == 7.4 - assert retrieved.field_parameter == "pH" - assert retrieved.units == "SU" - assert retrieved.analyses_agency == "NMA Agency" - - session.delete(new_fp) - session.delete(sample_info) - session.commit() - - -def test_object_id_auto_generation(water_well_thing): - """Verifies that the OBJECTID (Identity) column auto-increments in Postgres.""" - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - fp1 = NMA_FieldParameters( - sample_pt_id=sample_info.sample_pt_id, - field_parameter="Temp", - ) - session.add(fp1) - session.commit() - session.refresh(fp1) - - assert fp1.object_id is not None - - session.delete(fp1) - session.delete(sample_info) - session.commit() - - -# ===================== CREATE tests ========================== -def test_create_field_parameters_all_fields(water_well_thing): - """Test creating a field parameters record with all fields.""" - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - record = NMA_FieldParameters( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - sample_point_id=sample_info.sample_point_id, - field_parameter="pH", - sample_value=7.4, - units="SU", - notes="Test notes", - analyses_agency="NMBGMR", - wc_lab_id="LAB-202", - ) - session.add(record) - session.commit() - session.refresh(record) - - assert record.global_id is not None - assert record.sample_pt_id == sample_info.sample_pt_id - assert record.sample_point_id == sample_info.sample_point_id - assert record.field_parameter == "pH" - assert record.sample_value == 7.4 - - session.delete(record) - session.delete(sample_info) - session.commit() - - -def test_create_field_parameters_minimal(water_well_thing): - """Test creating a field parameters record with minimal fields.""" - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - record = NMA_FieldParameters( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - ) - session.add(record) - session.commit() - session.refresh(record) - - assert record.global_id is not None - assert record.sample_pt_id == sample_info.sample_pt_id - assert record.field_parameter is None - assert record.units is None - assert record.sample_value is None - - session.delete(record) - session.delete(sample_info) - session.commit() - - -# ===================== READ tests ========================== -def test_read_field_parameters_by_global_id(water_well_thing): - """Test reading a field parameters record by GlobalID.""" - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - record = NMA_FieldParameters( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - ) - session.add(record) - session.commit() - - fetched = session.get(NMA_FieldParameters, record.global_id) - assert fetched is not None - assert fetched.global_id == record.global_id - - session.delete(record) - session.delete(sample_info) - session.commit() - - -def test_query_field_parameters_by_sample_point_id(water_well_thing): - """Test querying field parameters by sample_point_id.""" - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - record1 = NMA_FieldParameters( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - sample_point_id=sample_info.sample_point_id, - ) - record2 = NMA_FieldParameters( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - sample_point_id="OTHER-PT", - ) - session.add_all([record1, record2]) - session.commit() - - # Use SQLAlchemy 2.0 style select/execute for ORM queries. - stmt = select(NMA_FieldParameters).filter( - NMA_FieldParameters.sample_point_id == sample_info.sample_point_id - ) - results = session.execute(stmt).scalars().all() - assert len(results) >= 1 - assert all(r.sample_point_id == sample_info.sample_point_id for r in results) - - session.delete(record1) - session.delete(record2) - session.delete(sample_info) - session.commit() - - -# ===================== UPDATE tests ========================== -def test_update_field_parameters(water_well_thing): - """Test updating a field parameters record.""" - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - record = NMA_FieldParameters( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - ) - session.add(record) - session.commit() - - record.analyses_agency = "Updated Agency" - record.notes = "Updated notes" - session.commit() - session.refresh(record) - - assert record.analyses_agency == "Updated Agency" - assert record.notes == "Updated notes" - - session.delete(record) - session.delete(sample_info) - session.commit() - - -# ===================== DELETE tests ========================== -def test_delete_field_parameters(water_well_thing): - """Test deleting a field parameters record.""" - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - record = NMA_FieldParameters( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - ) - session.add(record) - session.commit() - - session.delete(record) - session.commit() - - fetched = session.get(NMA_FieldParameters, record.global_id) - assert fetched is None - - session.delete(sample_info) - session.commit() - - -# ===================== Relational Integrity Tests ====================== - - -def test_orphan_prevention_constraint(): - """ - VERIFIES: 'SamplePtID IS NOT NULL' and Foreign Key presence. - Ensures the DB rejects records that aren't linked to a NMA_Chemistry_SampleInfo. - """ - with session_ctx() as session: - orphan = NMA_FieldParameters( - field_parameter="pH", - sample_value=7.0, - ) - session.add(orphan) - - with pytest.raises((IntegrityError, ProgrammingError)): - session.flush() - session.rollback() - - -def test_cascade_delete_behavior(water_well_thing): - """ - VERIFIES: 'on delete cascade' behavior. - Deleting the parent sample must automatically remove associated field measurements. - """ - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - fp = NMA_FieldParameters( - sample_pt_id=sample_info.sample_pt_id, - field_parameter="Temperature", - ) - session.add(fp) - session.commit() - session.refresh(fp) - fp_id = fp.global_id - - # Delete parent and check child - session.delete(sample_info) - session.commit() - session.expire_all() - - assert ( - session.get(NMA_FieldParameters, fp_id) is None - ), "Child record persisted after parent deletion." - - -def test_update_cascade_propagation(water_well_thing): - """ - VERIFIES: foreign key integrity on SamplePtID. - Ensures the DB rejects updates to a non-existent parent SamplePtID. - """ - with session_ctx() as session: - sample_info = _create_sample_info(session, water_well_thing) - fp = NMA_FieldParameters( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - field_parameter="Dissolved Oxygen", - ) - session.add(fp) - session.commit() - fp_id = fp.global_id - - with pytest.raises((IntegrityError, ProgrammingError)): - fp.sample_pt_id = uuid4() - session.flush() - session.rollback() - - fetched = session.get(NMA_FieldParameters, fp_id) - if fetched is not None: - session.delete(fetched) - session.delete(sample_info) - session.commit() diff --git a/tests/test_hydraulics_data_legacy.py b/tests/test_hydraulics_data_legacy.py index a24933376..375867649 100644 --- a/tests/test_hydraulics_data_legacy.py +++ b/tests/test_hydraulics_data_legacy.py @@ -17,29 +17,13 @@ Unit tests for HydraulicsData legacy model. These tests verify the migration of columns from the legacy HydraulicsData table. -Migrated columns: -- GlobalID -> global_id -- WellID -> well_id -- PointID -> point_id -- Data Source -> data_source -- Cs (gal/d/ft) -> cs_gal_d_ft -- HD (ft2/d) -> hd_ft2_d -- HL (day-1) -> hl_day_1 -- KH (ft/d) -> kh_ft_d -- KV (ft/d) -> kv_ft_d -- P (decimal fraction) -> p_decimal_fraction -- S (dimensionless) -> s_dimensionless -- Ss (ft-1) -> ss_ft_1 -- Sy (decimalfractn) -> sy_decimalfractn -- T (ft2/d) -> t_ft2_d -- k (darcy) -> k_darcy -- TestBottom -> test_bottom -- TestTop -> test_top -- HydraulicUnit -> hydraulic_unit -- HydraulicUnitType -> hydraulic_unit_type -- Hydraulic Remarks -> hydraulic_remarks -- OBJECTID -> object_id -- thing_id -> thing_id + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy GlobalID UUID (UNIQUE) +- nma_well_id: Legacy WellID UUID +- nma_point_id: Legacy PointID string +- nma_object_id: Legacy OBJECTID (UNIQUE) """ from uuid import uuid4 @@ -57,9 +41,9 @@ def test_create_hydraulics_data_all_fields(water_well_thing): """Test creating a hydraulics data record with all fields.""" with session_ctx() as session: record = NMA_HydraulicsData( - global_id=_next_global_id(), - well_id=uuid4(), - point_id=water_well_thing.name, + nma_global_id=_next_global_id(), + nma_well_id=uuid4(), + nma_point_id=water_well_thing.name, data_source="Legacy Source", cs_gal_d_ft=1.2, hd_ft2_d=3.4, @@ -77,20 +61,21 @@ def test_create_hydraulics_data_all_fields(water_well_thing): hydraulic_unit="Unit A", hydraulic_unit_type="U", hydraulic_remarks="Test remarks", - object_id=101, + nma_object_id=101, thing_id=water_well_thing.id, ) session.add(record) session.commit() session.refresh(record) - assert record.global_id is not None - assert record.well_id is not None - assert record.point_id == water_well_thing.name + assert record.id is not None # Integer PK auto-generated + assert record.nma_global_id is not None + assert record.nma_well_id is not None + assert record.nma_point_id == water_well_thing.name assert record.data_source == "Legacy Source" assert record.test_top == 30 assert record.test_bottom == 120 - assert record.object_id == 101 + assert record.nma_object_id == 101 assert record.thing_id == water_well_thing.id session.delete(record) @@ -101,7 +86,7 @@ def test_create_hydraulics_data_minimal(water_well_thing): """Test creating a hydraulics data record with minimal fields.""" with session_ctx() as session: record = NMA_HydraulicsData( - global_id=_next_global_id(), + nma_global_id=_next_global_id(), test_top=10, test_bottom=20, thing_id=water_well_thing.id, @@ -110,11 +95,12 @@ def test_create_hydraulics_data_minimal(water_well_thing): session.commit() session.refresh(record) - assert record.global_id is not None - assert record.well_id is None - assert record.point_id is None + assert record.id is not None # Integer PK auto-generated + assert record.nma_global_id is not None + assert record.nma_well_id is None + assert record.nma_point_id is None assert record.data_source is None - assert record.object_id is None + assert record.nma_object_id is None assert record.thing_id == water_well_thing.id session.delete(record) @@ -122,11 +108,11 @@ def test_create_hydraulics_data_minimal(water_well_thing): # ===================== READ tests ========================== -def test_read_hydraulics_data_by_global_id(water_well_thing): - """Test reading a hydraulics data record by GlobalID.""" +def test_read_hydraulics_data_by_id(water_well_thing): + """Test reading a hydraulics data record by Integer ID.""" with session_ctx() as session: record = NMA_HydraulicsData( - global_id=_next_global_id(), + nma_global_id=_next_global_id(), test_top=5, test_bottom=15, thing_id=water_well_thing.id, @@ -134,28 +120,29 @@ def test_read_hydraulics_data_by_global_id(water_well_thing): session.add(record) session.commit() - fetched = session.get(NMA_HydraulicsData, record.global_id) + fetched = session.get(NMA_HydraulicsData, record.id) assert fetched is not None - assert fetched.global_id == record.global_id + assert fetched.id == record.id + assert fetched.nma_global_id == record.nma_global_id session.delete(record) session.commit() -def test_query_hydraulics_data_by_point_id(water_well_thing): - """Test querying hydraulics data by point_id.""" +def test_query_hydraulics_data_by_nma_point_id(water_well_thing): + """Test querying hydraulics data by nma_point_id.""" with session_ctx() as session: record1 = NMA_HydraulicsData( - global_id=_next_global_id(), - well_id=uuid4(), - point_id=water_well_thing.name, + nma_global_id=_next_global_id(), + nma_well_id=uuid4(), + nma_point_id=water_well_thing.name, test_top=10, test_bottom=20, thing_id=water_well_thing.id, ) record2 = NMA_HydraulicsData( - global_id=_next_global_id(), - point_id="OTHER-POINT", + nma_global_id=_next_global_id(), + nma_point_id="OTHER-POINT", test_top=30, test_bottom=40, thing_id=water_well_thing.id, @@ -165,11 +152,11 @@ def test_query_hydraulics_data_by_point_id(water_well_thing): results = ( session.query(NMA_HydraulicsData) - .filter(NMA_HydraulicsData.point_id == water_well_thing.name) + .filter(NMA_HydraulicsData.nma_point_id == water_well_thing.name) .all() ) assert len(results) >= 1 - assert all(r.point_id == water_well_thing.name for r in results) + assert all(r.nma_point_id == water_well_thing.name for r in results) session.delete(record1) session.delete(record2) @@ -181,7 +168,7 @@ def test_update_hydraulics_data(water_well_thing): """Test updating a hydraulics data record.""" with session_ctx() as session: record = NMA_HydraulicsData( - global_id=_next_global_id(), + nma_global_id=_next_global_id(), test_top=5, test_bottom=15, thing_id=water_well_thing.id, @@ -206,18 +193,19 @@ def test_delete_hydraulics_data(water_well_thing): """Test deleting a hydraulics data record.""" with session_ctx() as session: record = NMA_HydraulicsData( - global_id=_next_global_id(), + nma_global_id=_next_global_id(), test_top=5, test_bottom=15, thing_id=water_well_thing.id, ) session.add(record) session.commit() + record_id = record.id session.delete(record) session.commit() - fetched = session.get(NMA_HydraulicsData, record.global_id) + fetched = session.get(NMA_HydraulicsData, record_id) assert fetched is None @@ -225,9 +213,10 @@ def test_delete_hydraulics_data(water_well_thing): def test_hydraulics_data_has_all_migrated_columns(): """Test that the model has all expected columns.""" expected_columns = [ - "global_id", - "well_id", - "point_id", + "id", + "nma_global_id", + "nma_well_id", + "nma_point_id", "data_source", "cs_gal_d_ft", "hd_ft2_d", @@ -245,7 +234,7 @@ def test_hydraulics_data_has_all_migrated_columns(): "hydraulic_unit", "hydraulic_unit_type", "hydraulic_remarks", - "object_id", + "nma_object_id", "thing_id", ] @@ -260,4 +249,72 @@ def test_hydraulics_data_table_name(): assert NMA_HydraulicsData.__tablename__ == "NMA_HydraulicsData" +# ===================== FK Enforcement tests (Issue #363) ========================== + + +def test_hydraulics_data_validator_rejects_none_thing_id(): + """NMA_HydraulicsData validator rejects None thing_id.""" + import pytest + + with pytest.raises(ValueError, match="requires a parent Thing"): + NMA_HydraulicsData( + nma_global_id=_next_global_id(), + test_top=5, + test_bottom=15, + thing_id=None, + ) + + +def test_hydraulics_data_thing_id_not_nullable(): + """NMA_HydraulicsData.thing_id column is NOT NULL.""" + col = NMA_HydraulicsData.__table__.c.thing_id + assert col.nullable is False, "thing_id should be NOT NULL" + + +def test_hydraulics_data_fk_has_cascade(): + """NMA_HydraulicsData.thing_id FK has ondelete=CASCADE.""" + col = NMA_HydraulicsData.__table__.c.thing_id + fk = list(col.foreign_keys)[0] + assert fk.ondelete == "CASCADE" + + +def test_hydraulics_data_back_populates_thing(water_well_thing): + """NMA_HydraulicsData.thing navigates back to Thing.""" + with session_ctx() as session: + well = session.merge(water_well_thing) + record = NMA_HydraulicsData( + nma_global_id=_next_global_id(), + test_top=5, + test_bottom=15, + thing_id=well.id, + ) + session.add(record) + session.commit() + session.refresh(record) + + assert record.thing is not None + assert record.thing.id == well.id + + session.delete(record) + session.commit() + + +# ===================== Integer PK tests ========================== + + +def test_hydraulics_data_has_integer_pk(): + """NMA_HydraulicsData.id is Integer PK.""" + from sqlalchemy import Integer + + col = NMA_HydraulicsData.__table__.c.id + assert col.primary_key is True + assert isinstance(col.type, Integer) + + +def test_hydraulics_data_nma_global_id_is_unique(): + """NMA_HydraulicsData.nma_global_id is UNIQUE.""" + col = NMA_HydraulicsData.__table__.c["nma_GlobalID"] + assert col.unique is True + + # ============= EOF ============================================= diff --git a/tests/test_major_chemistry_legacy.py b/tests/test_major_chemistry_legacy.py index 7161ec74d..a745ce243 100644 --- a/tests/test_major_chemistry_legacy.py +++ b/tests/test_major_chemistry_legacy.py @@ -17,23 +17,15 @@ Unit tests for MajorChemistry legacy model. These tests verify the migration of columns from the legacy MajorChemistry table. -Migrated columns (excluding SSMA_TimeStamp): -- SamplePtID -> sample_pt_id -- SamplePointID -> sample_point_id -- Analyte -> analyte -- Symbol -> symbol -- SampleValue -> sample_value -- Units -> units -- Uncertainty -> uncertainty -- AnalysisMethod -> analysis_method -- AnalysisDate -> analysis_date -- Notes -> notes -- Volume -> volume -- VolumeUnit -> volume_unit -- OBJECTID -> object_id -- GlobalID -> global_id -- AnalysesAgency -> analyses_agency -- WCLab_ID -> wclab_id + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy GlobalID UUID (UNIQUE) +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_sample_pt_id: Legacy SamplePtID UUID (for audit) +- nma_sample_point_id: Legacy SamplePointID string +- nma_object_id: Legacy OBJECTID (UNIQUE) +- nma_wclab_id: Legacy WCLab_ID string """ from datetime import datetime @@ -52,17 +44,19 @@ def test_create_major_chemistry_all_fields(water_well_thing): """Test creating a major chemistry record with all fields.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_MajorChemistry( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - sample_point_id=sample_info.sample_point_id, + nma_global_id=uuid4(), + chemistry_sample_info_id=sample_info.id, + nma_sample_pt_id=sample_info.nma_sample_pt_id, + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="Ca", symbol="<", sample_value=12.3, @@ -74,15 +68,17 @@ def test_create_major_chemistry_all_fields(water_well_thing): volume=250, volume_unit="mL", analyses_agency="NMBGMR", - wclab_id="LAB-101", + nma_wclab_id="LAB-101", ) session.add(record) session.commit() session.refresh(record) - assert record.global_id is not None - assert record.sample_pt_id == sample_info.sample_pt_id - assert record.sample_point_id == sample_info.sample_point_id + assert record.id is not None # Integer PK auto-generated + assert record.nma_global_id is not None + assert record.chemistry_sample_info_id == sample_info.id + assert record.nma_sample_pt_id == sample_info.nma_sample_pt_id + assert record.nma_sample_point_id == sample_info.nma_sample_point_id assert record.analyte == "Ca" assert record.sample_value == 12.3 assert record.uncertainty == 0.1 @@ -96,23 +92,25 @@ def test_create_major_chemistry_minimal(water_well_thing): """Test creating a major chemistry record with minimal fields.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_MajorChemistry( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, + nma_global_id=uuid4(), + chemistry_sample_info_id=sample_info.id, ) session.add(record) session.commit() session.refresh(record) - assert record.global_id is not None - assert record.sample_pt_id == sample_info.sample_pt_id + assert record.id is not None # Integer PK auto-generated + assert record.nma_global_id is not None + assert record.chemistry_sample_info_id == sample_info.id assert record.analyte is None assert record.units is None @@ -122,64 +120,72 @@ def test_create_major_chemistry_minimal(water_well_thing): # ===================== READ tests ========================== -def test_read_major_chemistry_by_global_id(water_well_thing): - """Test reading a major chemistry record by GlobalID.""" +def test_read_major_chemistry_by_id(water_well_thing): + """Test reading a major chemistry record by Integer ID.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_MajorChemistry( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, + nma_global_id=uuid4(), + chemistry_sample_info_id=sample_info.id, ) session.add(record) session.commit() - fetched = session.get(NMA_MajorChemistry, record.global_id) + fetched = session.get(NMA_MajorChemistry, record.id) assert fetched is not None - assert fetched.global_id == record.global_id + assert fetched.id == record.id + assert fetched.nma_global_id == record.nma_global_id session.delete(record) session.delete(sample_info) session.commit() -def test_query_major_chemistry_by_sample_point_id(water_well_thing): - """Test querying major chemistry by sample_point_id.""" +def test_query_major_chemistry_by_nma_sample_point_id(water_well_thing): + """Test querying major chemistry by nma_sample_point_id.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record1 = NMA_MajorChemistry( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - sample_point_id=sample_info.sample_point_id, + nma_global_id=uuid4(), + chemistry_sample_info_id=sample_info.id, + nma_sample_point_id=sample_info.nma_sample_point_id, ) record2 = NMA_MajorChemistry( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, - sample_point_id="OTHER-PT", + nma_global_id=uuid4(), + chemistry_sample_info_id=sample_info.id, + nma_sample_point_id="OTHER-PT", ) session.add_all([record1, record2]) session.commit() results = ( session.query(NMA_MajorChemistry) - .filter(NMA_MajorChemistry.sample_point_id == sample_info.sample_point_id) + .filter( + NMA_MajorChemistry.nma_sample_point_id + == sample_info.nma_sample_point_id + ) .all() ) assert len(results) >= 1 - assert all(r.sample_point_id == sample_info.sample_point_id for r in results) + assert all( + r.nma_sample_point_id == sample_info.nma_sample_point_id for r in results + ) session.delete(record1) session.delete(record2) @@ -192,16 +198,17 @@ def test_update_major_chemistry(water_well_thing): """Test updating a major chemistry record.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_MajorChemistry( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, + nma_global_id=uuid4(), + chemistry_sample_info_id=sample_info.id, ) session.add(record) session.commit() @@ -224,24 +231,26 @@ def test_delete_major_chemistry(water_well_thing): """Test deleting a major chemistry record.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_MajorChemistry( - global_id=uuid4(), - sample_pt_id=sample_info.sample_pt_id, + nma_global_id=uuid4(), + chemistry_sample_info_id=sample_info.id, ) session.add(record) session.commit() + record_id = record.id session.delete(record) session.commit() - fetched = session.get(NMA_MajorChemistry, record.global_id) + fetched = session.get(NMA_MajorChemistry, record_id) assert fetched is None session.delete(sample_info) @@ -252,9 +261,11 @@ def test_delete_major_chemistry(water_well_thing): def test_major_chemistry_has_all_migrated_columns(): """Test that the model has all expected columns.""" expected_columns = [ - "global_id", - "sample_pt_id", - "sample_point_id", + "id", + "nma_global_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", "analyte", "symbol", "sample_value", @@ -265,9 +276,9 @@ def test_major_chemistry_has_all_migrated_columns(): "notes", "volume", "volume_unit", - "object_id", + "nma_object_id", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] for column in expected_columns: @@ -281,4 +292,31 @@ def test_major_chemistry_table_name(): assert NMA_MajorChemistry.__tablename__ == "NMA_MajorChemistry" +# ===================== Integer PK tests ========================== + + +def test_major_chemistry_has_integer_pk(): + """NMA_MajorChemistry.id is Integer PK.""" + from sqlalchemy import Integer + + col = NMA_MajorChemistry.__table__.c.id + assert col.primary_key is True + assert isinstance(col.type, Integer) + + +def test_major_chemistry_nma_global_id_is_unique(): + """NMA_MajorChemistry.nma_global_id is UNIQUE.""" + # Use database column name (nma_GlobalID), not Python attribute name + col = NMA_MajorChemistry.__table__.c["nma_GlobalID"] + assert col.unique is True + + +def test_major_chemistry_chemistry_sample_info_fk(): + """NMA_MajorChemistry.chemistry_sample_info_id is Integer FK.""" + col = NMA_MajorChemistry.__table__.c.chemistry_sample_info_id + fks = list(col.foreign_keys) + assert len(fks) == 1 + assert "NMA_Chemistry_SampleInfo.id" in str(fks[0].target_fullname) + + # ============= EOF ============================================= diff --git a/tests/test_nma_chemistry_lineage.py b/tests/test_nma_chemistry_lineage.py index cebe89f8f..4ad4a8ea7 100644 --- a/tests/test_nma_chemistry_lineage.py +++ b/tests/test_nma_chemistry_lineage.py @@ -16,7 +16,7 @@ """ Unit tests for NMA Chemistry lineage OO associations. -Lineage: +Lineage (updated 2026-01): Thing (1) ---> (*) NMA_Chemistry_SampleInfo (1) ---> (*) NMA_MinorTraceChemistry Tests verify SQLAlchemy relationships enable OO navigation: @@ -25,6 +25,9 @@ - sample_info.minor_trace_chemistries - mtc.chemistry_sample_info - mtc.chemistry_sample_info.thing (full chain) + +FK Change (2026-01): + - Uses thing_id (Integer FK to Thing.id) """ from uuid import uuid4 @@ -52,29 +55,50 @@ def _next_global_id(): @pytest.fixture(scope="module") -def shared_well(): - """Create a single Thing for all tests in this module.""" - from db import Thing +def shared_thing(): + """Create a single Thing (with Location) for all tests in this module.""" + from db import Location, LocationThingAssociation, Thing with session_ctx() as session: + location = Location( + point="POINT(-107.949533 33.809665)", + elevation=2464.9, + release_status="draft", + ) + session.add(location) + session.commit() + session.refresh(location) + thing = Thing( - name=f"Shared-Well-{uuid4().hex[:8]}", - thing_type="water well", + name="LINEAGE-TEST-WELL", + thing_type="monitoring well", release_status="draft", ) session.add(thing) session.commit() session.refresh(thing) + + assoc = LocationThingAssociation( + location_id=location.id, + thing_id=thing.id, + ) + session.add(assoc) + session.commit() + thing_id = thing.id + location_id = location.id yield thing_id # Cleanup after all tests with session_ctx() as session: thing = session.get(Thing, thing_id) + location = session.get(Location, location_id) if thing: session.delete(thing) - session.commit() + if location: + session.delete(location) + session.commit() # ===================== Model import tests ========================== @@ -99,20 +123,22 @@ def test_nma_minor_trace_chemistry_columns(): """ NMA_MinorTraceChemistry should have required columns. - Omitted legacy columns: globalid, objectid, ssma_timestamp, - samplepointid, sampleptid, wclab_id + Updated for Integer PK schema: + - id: Integer PK (autoincrement) + - nma_global_id: Legacy GlobalID UUID (UNIQUE) + - chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id """ from db.nma_legacy import NMA_MinorTraceChemistry expected_columns = [ - "global_id", # PK - "sample_pt_id", # FK to NMA_Chemistry_SampleInfo + "id", # Integer PK + "nma_global_id", # Legacy UUID + "chemistry_sample_info_id", # Integer FK # from legacy - "sample_point_id", "analyte", - "symbol", "sample_value", "units", + "symbol", "analysis_method", "analysis_date", "notes", @@ -120,34 +146,32 @@ def test_nma_minor_trace_chemistry_columns(): "uncertainty", "volume", "volume_unit", - "object_id", - "wclab_id", ] for col in expected_columns: assert hasattr(NMA_MinorTraceChemistry, col), f"Missing column: {col}" -def test_nma_minor_trace_chemistry_save_all_columns(shared_well): +def test_nma_minor_trace_chemistry_save_all_columns(shared_thing): """Can save NMA_MinorTraceChemistry with all columns populated.""" from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_MinorTraceChemistry from db import Thing from datetime import date with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=well, + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) session.add(sample_info) session.commit() mtc = NMA_MinorTraceChemistry( - global_id=_next_global_id(), + nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, analyte="As", sample_value=0.015, @@ -166,8 +190,9 @@ def test_nma_minor_trace_chemistry_save_all_columns(shared_well): session.refresh(mtc) # Verify all columns saved - assert mtc.global_id is not None - assert mtc.sample_pt_id == sample_info.sample_pt_id + assert mtc.id is not None # Integer PK + assert mtc.nma_global_id is not None # Legacy UUID + assert mtc.chemistry_sample_info_id == sample_info.id # Integer FK assert mtc.analyte == "As" assert mtc.sample_value == 0.015 assert mtc.units == "mg/L" @@ -187,132 +212,161 @@ def test_nma_minor_trace_chemistry_save_all_columns(shared_well): # ===================== Thing → NMA_Chemistry_SampleInfo association ========================== -def test_thing_has_chemistry_sample_infos_attribute(shared_well): +def test_thing_has_chemistry_sample_infos_attribute(shared_thing): """Thing should have chemistry_sample_infos relationship.""" from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) - assert hasattr(well, "chemistry_sample_infos") + thing = session.get(Thing, shared_thing) + assert hasattr(thing, "chemistry_sample_infos") def test_thing_chemistry_sample_infos_empty_by_default(): """New Thing should have empty chemistry_sample_infos.""" - from db import Thing + from db import Thing, Location, LocationThingAssociation with session_ctx() as session: # Create a fresh Thing for this test + location = Location( + point="POINT(-106.0 35.0)", + elevation=1500.0, + release_status="draft", + ) + session.add(location) + session.commit() + new_thing = Thing( - name=f"Empty-Test-{uuid4().hex[:8]}", - thing_type="water well", + name="EMPTY-CHEM-TEST", + thing_type="monitoring well", release_status="draft", ) session.add(new_thing) session.commit() + + assoc = LocationThingAssociation( + location_id=location.id, + thing_id=new_thing.id, + ) + session.add(assoc) + session.commit() session.refresh(new_thing) assert new_thing.chemistry_sample_infos == [] session.delete(new_thing) + session.delete(location) session.commit() -def test_assign_thing_to_sample_info(shared_well): +def test_assign_thing_to_sample_info(shared_thing): """Can assign Thing to NMA_Chemistry_SampleInfo via object (not just ID).""" from db.nma_legacy import NMA_Chemistry_SampleInfo from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=well, # OO: assign object + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, # OO: assign object ) session.add(sample_info) session.commit() # Verify bidirectional - assert sample_info.thing == well - assert sample_info in well.chemistry_sample_infos + assert sample_info.thing == thing + assert sample_info in thing.chemistry_sample_infos session.delete(sample_info) session.commit() -def test_append_sample_info_to_thing(shared_well): +def test_append_sample_info_to_thing(shared_thing): """Can append NMA_Chemistry_SampleInfo to Thing's collection.""" from db.nma_legacy import NMA_Chemistry_SampleInfo from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), ) - well.chemistry_sample_infos.append(sample_info) + thing.chemistry_sample_infos.append(sample_info) session.commit() # Verify bidirectional - assert sample_info.thing == well - assert sample_info.thing_id == well.id + assert sample_info.thing == thing + assert sample_info.thing_id == thing.id session.delete(sample_info) session.commit() -# ===================== NMA_Chemistry_SampleInfo → Thing association ========================== - - -def test_sample_info_has_thing_attribute(): +def test_sample_info_has_thing_attribute(shared_thing): """NMA_Chemistry_SampleInfo should have thing relationship.""" from db.nma_legacy import NMA_Chemistry_SampleInfo + from db import Thing - assert hasattr(NMA_Chemistry_SampleInfo, "thing") - - -def test_sample_info_requires_thing(): - """NMA_Chemistry_SampleInfo cannot be orphaned - must have a parent Thing.""" - from db.nma_legacy import NMA_Chemistry_SampleInfo + with session_ctx() as session: + thing = session.get(Thing, shared_thing) - # Validator raises ValueError before database is even touched - with pytest.raises(ValueError, match="requires a parent Thing"): - NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing_id=None, # Explicit None triggers validator + sample_info = NMA_Chemistry_SampleInfo( + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) + session.add(sample_info) + session.commit() + session.refresh(sample_info) + assert hasattr(sample_info, "thing") + assert sample_info.thing == thing -# ===================== NMA_Chemistry_SampleInfo → NMA_MinorTraceChemistry association ========================== + session.delete(sample_info) + session.commit() -def test_sample_info_has_minor_trace_chemistries_attribute(): - """NMA_Chemistry_SampleInfo should have minor_trace_chemistries relationship.""" +def test_sample_info_requires_thing(shared_thing): + """NMA_Chemistry_SampleInfo should require thing_id (not nullable).""" from db.nma_legacy import NMA_Chemistry_SampleInfo + from sqlalchemy.exc import IntegrityError, ProgrammingError + + with session_ctx() as session: + sample_info = NMA_Chemistry_SampleInfo( + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + # No thing_id - should fail + ) + session.add(sample_info) + # pg8000 raises ProgrammingError for NOT NULL violations (error code 23502) + with pytest.raises((IntegrityError, ProgrammingError, ValueError)): + session.commit() + session.rollback() - assert hasattr(NMA_Chemistry_SampleInfo, "minor_trace_chemistries") + +# ===================== NMA_Chemistry_SampleInfo → NMA_MinorTraceChemistry association ========================== -def test_sample_info_minor_trace_chemistries_empty_by_default(shared_well): +def test_sample_info_minor_trace_chemistries_empty_by_default(shared_thing): """New NMA_Chemistry_SampleInfo should have empty minor_trace_chemistries.""" from db.nma_legacy import NMA_Chemistry_SampleInfo from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=well, + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) session.add(sample_info) session.commit() @@ -324,29 +378,27 @@ def test_sample_info_minor_trace_chemistries_empty_by_default(shared_well): session.commit() -def test_assign_sample_info_to_mtc(shared_well): - """Can assign NMA_Chemistry_SampleInfo to MinorTraceChemistry via object.""" +def test_assign_sample_info_to_mtc(shared_thing): + """Can assign NMA_Chemistry_SampleInfo to NMA_MinorTraceChemistry via object.""" from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_MinorTraceChemistry from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=well, + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) session.add(sample_info) session.commit() mtc = NMA_MinorTraceChemistry( - global_id=_next_global_id(), - analyte="As", - sample_value=0.01, - units="mg/L", + nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, # OO: assign object + analyte="Pb", ) session.add(mtc) session.commit() @@ -355,303 +407,291 @@ def test_assign_sample_info_to_mtc(shared_well): assert mtc.chemistry_sample_info == sample_info assert mtc in sample_info.minor_trace_chemistries - session.delete(sample_info) # cascades to mtc + session.delete(sample_info) session.commit() -def test_append_mtc_to_sample_info(shared_well): - """Can append MinorTraceChemistry to NMA_Chemistry_SampleInfo's collection.""" +def test_append_mtc_to_sample_info(shared_thing): + """Can append NMA_MinorTraceChemistry to NMA_Chemistry_SampleInfo's collection.""" from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_MinorTraceChemistry from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=well, + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) session.add(sample_info) session.commit() mtc = NMA_MinorTraceChemistry( - global_id=_next_global_id(), - analyte="U", - sample_value=15.2, - units="ug/L", + nma_global_id=_next_global_id(), + analyte="Fe", ) sample_info.minor_trace_chemistries.append(mtc) session.commit() # Verify bidirectional assert mtc.chemistry_sample_info == sample_info - assert mtc.sample_pt_id == sample_info.sample_pt_id + assert mtc.chemistry_sample_info_id == sample_info.id session.delete(sample_info) session.commit() -# ===================== NMA_MinorTraceChemistry → NMA_Chemistry_SampleInfo association ========================== - - -def test_mtc_has_chemistry_sample_info_attribute(): - """NMA_MinorTraceChemistry should have chemistry_sample_info relationship.""" - from db.nma_legacy import NMA_MinorTraceChemistry - - assert hasattr(NMA_MinorTraceChemistry, "chemistry_sample_info") - - def test_mtc_requires_chemistry_sample_info(): - """NMA_MinorTraceChemistry cannot be orphaned - must have a parent.""" + """NMA_MinorTraceChemistry should require chemistry_sample_info_id.""" from db.nma_legacy import NMA_MinorTraceChemistry + from sqlalchemy.exc import IntegrityError, ProgrammingError - # Validator raises ValueError before database is even touched - with pytest.raises(ValueError, match="requires a parent NMA_Chemistry_SampleInfo"): - NMA_MinorTraceChemistry( - analyte="As", - sample_value=0.01, - units="mg/L", - sample_pt_id=None, # Explicit None triggers validator + with session_ctx() as session: + mtc = NMA_MinorTraceChemistry( + nma_global_id=_next_global_id(), + analyte="Cu", + # No chemistry_sample_info_id - should fail ) + session.add(mtc) + # pg8000 raises ProgrammingError for NOT NULL violations (error code 23502) + with pytest.raises((IntegrityError, ProgrammingError)): + session.commit() + session.rollback() # ===================== Full lineage navigation ========================== -def test_full_lineage_navigation(shared_well): - """Can navigate full chain: mtc.chemistry_sample_info.thing""" +def test_full_lineage_navigation(shared_thing): + """Can navigate full lineage: Thing -> SampleInfo -> MTC.""" from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_MinorTraceChemistry from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=well, + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) session.add(sample_info) session.commit() mtc = NMA_MinorTraceChemistry( - global_id=_next_global_id(), - analyte="Se", - sample_value=0.005, - units="mg/L", + nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, + analyte="Zn", ) session.add(mtc) session.commit() - # Full chain navigation - assert mtc.chemistry_sample_info.thing == well + # Forward navigation + assert thing.chemistry_sample_infos[0] == sample_info + assert sample_info.minor_trace_chemistries[0] == mtc + + # Reverse navigation + assert mtc.chemistry_sample_info == sample_info + assert mtc.chemistry_sample_info.thing == thing session.delete(sample_info) session.commit() -def test_reverse_lineage_navigation(shared_well): - """Can navigate reverse: thing.chemistry_sample_infos[0].minor_trace_chemistries""" +def test_reverse_lineage_navigation(shared_thing): + """Can navigate reverse: MTC -> SampleInfo -> Thing.""" from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_MinorTraceChemistry from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=well, + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) session.add(sample_info) session.commit() mtc = NMA_MinorTraceChemistry( - global_id=_next_global_id(), - analyte="Pb", - sample_value=0.002, - units="mg/L", + nma_global_id=_next_global_id(), chemistry_sample_info=sample_info, + analyte="Mn", ) session.add(mtc) session.commit() - session.refresh(well) + session.refresh(mtc) - # Reverse navigation - filter to just this sample_info - matching = [ - si - for si in well.chemistry_sample_infos - if si.sample_pt_id == sample_info.sample_pt_id - ] - assert len(matching) == 1 - assert len(matching[0].minor_trace_chemistries) == 1 - assert matching[0].minor_trace_chemistries[0] == mtc + # Full reverse chain + assert mtc.chemistry_sample_info.thing.id == thing.id session.delete(sample_info) session.commit() -# ===================== Cascade delete ========================== +# ===================== Cascade delete tests ========================== -def test_cascade_delete_sample_info_deletes_mtc(shared_well): - """Deleting NMA_Chemistry_SampleInfo should cascade delete its MinorTraceChemistries.""" +def test_cascade_delete_sample_info_deletes_mtc(shared_thing): + """Deleting NMA_Chemistry_SampleInfo should cascade delete NMA_MinorTraceChemistry.""" from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_MinorTraceChemistry from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=well, + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) session.add(sample_info) session.commit() - # Add multiple children - for analyte in ["As", "U", "Se", "Pb"]: - sample_info.minor_trace_chemistries.append( - NMA_MinorTraceChemistry( - global_id=_next_global_id(), - analyte=analyte, - sample_value=0.01, - units="mg/L", - ) - ) - session.commit() - - sample_info_id = sample_info.sample_pt_id - assert ( - session.query(NMA_MinorTraceChemistry) - .filter_by(sample_pt_id=sample_info_id) - .count() - == 4 + mtc = NMA_MinorTraceChemistry( + nma_global_id=_next_global_id(), + chemistry_sample_info=sample_info, + analyte="Cd", ) + session.add(mtc) + session.commit() - # Delete parent + mtc_id = mtc.id session.delete(sample_info) session.commit() + session.expire_all() # Force fresh DB lookup after cascade delete - # Children should be gone - assert ( - session.query(NMA_MinorTraceChemistry) - .filter_by(sample_pt_id=sample_info_id) - .count() - == 0 - ) + # MTC should be gone + assert session.get(NMA_MinorTraceChemistry, mtc_id) is None -def test_cascade_delete_thing_deletes_sample_infos(): - """Deleting Thing should cascade delete its NMA_Chemistry_SampleInfos.""" +def test_cascade_delete_thing_deletes_sample_infos(shared_thing): + """Deleting Thing should cascade delete NMA_Chemistry_SampleInfo.""" from db.nma_legacy import NMA_Chemistry_SampleInfo - from db import Thing + from db import Thing, Location, LocationThingAssociation with session_ctx() as session: # Create a separate thing for this test - test_thing = Thing( - name=f"Cascade-Test-{uuid4().hex[:8]}", - thing_type="water well", + location = Location( + point="POINT(-105.0 34.0)", + elevation=1200.0, + release_status="draft", + ) + session.add(location) + session.commit() + + thing = Thing( + name="CASCADE-DELETE-TEST", + thing_type="monitoring well", release_status="draft", ) - session.add(test_thing) + session.add(thing) + session.commit() + + assoc = LocationThingAssociation( + location_id=location.id, + thing_id=thing.id, + ) + session.add(assoc) session.commit() sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=test_thing, + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) session.add(sample_info) session.commit() - # SamplePtID is the PK for NMA_Chemistry_SampleInfo. - sample_info_id = sample_info.sample_pt_id - - # Delete thing - session.delete(test_thing) + sample_info_id = sample_info.id + session.delete(thing) session.commit() + session.expire_all() # Force fresh DB lookup after cascade delete - # Use fresh session to verify cascade delete (avoid session cache) - with session_ctx() as session: + # SampleInfo should be gone assert session.get(NMA_Chemistry_SampleInfo, sample_info_id) is None + session.delete(location) + session.commit() + -# ===================== Multiple children ========================== +# ===================== Multiple records tests ========================== -def test_multiple_sample_infos_per_thing(): - """Thing can have multiple NMA_Chemistry_SampleInfos.""" +def test_multiple_sample_infos_per_thing(shared_thing): + """Thing can have multiple NMA_Chemistry_SampleInfo records.""" from db.nma_legacy import NMA_Chemistry_SampleInfo from db import Thing with session_ctx() as session: - # Create a dedicated thing for this test - test_thing = Thing( - name=f"Multi-SI-Test-{uuid4().hex[:8]}", - thing_type="water well", - release_status="draft", - ) - session.add(test_thing) - session.commit() + thing = session.get(Thing, shared_thing) - for i in range(3): - sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=test_thing, - ) - session.add(sample_info) + sample_info1 = NMA_Chemistry_SampleInfo( + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, + ) + sample_info2 = NMA_Chemistry_SampleInfo( + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, + ) + session.add_all([sample_info1, sample_info2]) session.commit() + session.refresh(thing) - session.refresh(test_thing) - assert len(test_thing.chemistry_sample_infos) == 3 + assert len(thing.chemistry_sample_infos) >= 2 + assert sample_info1 in thing.chemistry_sample_infos + assert sample_info2 in thing.chemistry_sample_infos - # Cleanup - delete thing cascades to sample_infos - session.delete(test_thing) + session.delete(sample_info1) + session.delete(sample_info2) session.commit() -def test_multiple_mtc_per_sample_info(shared_well): - """NMA_Chemistry_SampleInfo can have multiple MinorTraceChemistries.""" +def test_multiple_mtc_per_sample_info(shared_thing): + """NMA_Chemistry_SampleInfo can have multiple NMA_MinorTraceChemistry records.""" from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_MinorTraceChemistry from db import Thing with session_ctx() as session: - well = session.get(Thing, shared_well) + thing = session.get(Thing, shared_thing) sample_info = NMA_Chemistry_SampleInfo( - object_id=_next_object_id(), - sample_pt_id=_next_sample_pt_id(), - sample_point_id=_next_sample_point_id(), - thing=well, + nma_object_id=_next_object_id(), + nma_sample_pt_id=_next_sample_pt_id(), + nma_sample_point_id=_next_sample_point_id(), + thing=thing, ) session.add(sample_info) session.commit() - analytes = ["As", "U", "Se", "Pb", "Cd", "Hg"] - for analyte in analytes: - sample_info.minor_trace_chemistries.append( - NMA_MinorTraceChemistry( - global_id=_next_global_id(), - analyte=analyte, - sample_value=0.01, - units="mg/L", - ) - ) + mtc1 = NMA_MinorTraceChemistry( + nma_global_id=_next_global_id(), + chemistry_sample_info=sample_info, + analyte="As", + ) + mtc2 = NMA_MinorTraceChemistry( + nma_global_id=_next_global_id(), + chemistry_sample_info=sample_info, + analyte="Pb", + ) + session.add_all([mtc1, mtc2]) session.commit() - session.refresh(sample_info) - assert len(sample_info.minor_trace_chemistries) == 6 + + assert len(sample_info.minor_trace_chemistries) == 2 + assert mtc1 in sample_info.minor_trace_chemistries + assert mtc2 in sample_info.minor_trace_chemistries session.delete(sample_info) session.commit() diff --git a/tests/test_ogc.py b/tests/test_ogc.py index eb94aabe1..cc017367b 100644 --- a/tests/test_ogc.py +++ b/tests/test_ogc.py @@ -73,7 +73,7 @@ def test_ogc_collections(): assert {"locations", "wells", "springs"}.issubset(ids) -@pytest.mark.skip("not at all clear why this is failing") +@pytest.mark.skip("PostGIS spatial operators not available in CI - see issue #449") def test_ogc_locations_items_bbox(location): bbox = "-107.95,33.80,-107.94,33.81" response = client.get(f"/ogc/collections/locations/items?bbox={bbox}") @@ -97,6 +97,7 @@ def test_ogc_wells_items_and_item(water_well_thing): assert payload["id"] == water_well_thing.id +@pytest.mark.skip("PostGIS spatial operators not available in CI - see issue #449") def test_ogc_polygon_within_filter(location): polygon = "POLYGON((-107.95 33.80,-107.94 33.80,-107.94 33.81,-107.95 33.81,-107.95 33.80))" response = client.get( diff --git a/tests/test_radionuclides_legacy.py b/tests/test_radionuclides_legacy.py index 1e13e5b69..68fd1d193 100644 --- a/tests/test_radionuclides_legacy.py +++ b/tests/test_radionuclides_legacy.py @@ -17,23 +17,15 @@ Unit tests for Radionuclides legacy model. These tests verify the migration of columns from the legacy Radionuclides table. -Migrated columns (excluding SSMA_TimeStamp): -- SamplePtID -> sample_pt_id -- SamplePointID -> sample_point_id -- Analyte -> analyte -- Symbol -> symbol -- SampleValue -> sample_value -- Units -> units -- Uncertainty -> uncertainty -- AnalysisMethod -> analysis_method -- AnalysisDate -> analysis_date -- Notes -> notes -- Volume -> volume -- VolumeUnit -> volume_unit -- OBJECTID -> object_id -- GlobalID -> global_id -- AnalysesAgency -> analyses_agency -- WCLab_ID -> wclab_id + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy GlobalID UUID (UNIQUE) +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_sample_pt_id: Legacy SamplePtID UUID (for audit) +- nma_sample_point_id: Legacy SamplePointID string +- nma_object_id: Legacy OBJECTID (UNIQUE) +- nma_wclab_id: Legacy WCLab_ID string """ from datetime import datetime @@ -52,18 +44,20 @@ def test_create_radionuclides_all_fields(water_well_thing): """Test creating a radionuclides record with all fields.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_Radionuclides( - global_id=uuid4(), + nma_global_id=uuid4(), thing_id=water_well_thing.id, - sample_pt_id=sample_info.sample_pt_id, - sample_point_id=sample_info.sample_point_id, + chemistry_sample_info_id=sample_info.id, + nma_sample_pt_id=sample_info.nma_sample_pt_id, + nma_sample_point_id=sample_info.nma_sample_point_id, analyte="U-238", symbol="<", sample_value=0.12, @@ -75,15 +69,17 @@ def test_create_radionuclides_all_fields(water_well_thing): volume=250, volume_unit="mL", analyses_agency="NMBGMR", - wclab_id="LAB-001", + nma_wclab_id="LAB-001", ) session.add(record) session.commit() session.refresh(record) - assert record.global_id is not None - assert record.sample_pt_id == sample_info.sample_pt_id - assert record.sample_point_id == sample_info.sample_point_id + assert record.id is not None # Integer PK auto-generated + assert record.nma_global_id is not None + assert record.chemistry_sample_info_id == sample_info.id + assert record.nma_sample_pt_id == sample_info.nma_sample_pt_id + assert record.nma_sample_point_id == sample_info.nma_sample_point_id assert record.analyte == "U-238" assert record.sample_value == 0.12 assert record.uncertainty == 0.01 @@ -97,24 +93,26 @@ def test_create_radionuclides_minimal(water_well_thing): """Test creating a radionuclides record with minimal fields.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_Radionuclides( - global_id=uuid4(), + nma_global_id=uuid4(), thing_id=water_well_thing.id, - sample_pt_id=sample_info.sample_pt_id, + chemistry_sample_info_id=sample_info.id, ) session.add(record) session.commit() session.refresh(record) - assert record.global_id is not None - assert record.sample_pt_id == sample_info.sample_pt_id + assert record.id is not None # Integer PK auto-generated + assert record.nma_global_id is not None + assert record.chemistry_sample_info_id == sample_info.id assert record.analyte is None assert record.units is None @@ -124,67 +122,74 @@ def test_create_radionuclides_minimal(water_well_thing): # ===================== READ tests ========================== -def test_read_radionuclides_by_global_id(water_well_thing): - """Test reading a radionuclides record by GlobalID.""" +def test_read_radionuclides_by_id(water_well_thing): + """Test reading a radionuclides record by Integer ID.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_Radionuclides( - global_id=uuid4(), + nma_global_id=uuid4(), thing_id=water_well_thing.id, - sample_pt_id=sample_info.sample_pt_id, + chemistry_sample_info_id=sample_info.id, ) session.add(record) session.commit() - fetched = session.get(NMA_Radionuclides, record.global_id) + fetched = session.get(NMA_Radionuclides, record.id) assert fetched is not None - assert fetched.global_id == record.global_id + assert fetched.id == record.id + assert fetched.nma_global_id == record.nma_global_id session.delete(record) session.delete(sample_info) session.commit() -def test_query_radionuclides_by_sample_point_id(water_well_thing): - """Test querying radionuclides by sample_point_id.""" +def test_query_radionuclides_by_nma_sample_point_id(water_well_thing): + """Test querying radionuclides by nma_sample_point_id.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record1 = NMA_Radionuclides( - global_id=uuid4(), + nma_global_id=uuid4(), thing_id=water_well_thing.id, - sample_pt_id=sample_info.sample_pt_id, - sample_point_id=sample_info.sample_point_id, + chemistry_sample_info_id=sample_info.id, + nma_sample_point_id=sample_info.nma_sample_point_id, ) record2 = NMA_Radionuclides( - global_id=uuid4(), + nma_global_id=uuid4(), thing_id=water_well_thing.id, - sample_pt_id=sample_info.sample_pt_id, - sample_point_id="OTHER-PT", + chemistry_sample_info_id=sample_info.id, + nma_sample_point_id="OTHER-PT", ) session.add_all([record1, record2]) session.commit() results = ( session.query(NMA_Radionuclides) - .filter(NMA_Radionuclides.sample_point_id == sample_info.sample_point_id) + .filter( + NMA_Radionuclides.nma_sample_point_id == sample_info.nma_sample_point_id + ) .all() ) assert len(results) >= 1 - assert all(r.sample_point_id == sample_info.sample_point_id for r in results) + assert all( + r.nma_sample_point_id == sample_info.nma_sample_point_id for r in results + ) session.delete(record1) session.delete(record2) @@ -197,17 +202,18 @@ def test_update_radionuclides(water_well_thing): """Test updating a radionuclides record.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_Radionuclides( - global_id=uuid4(), + nma_global_id=uuid4(), thing_id=water_well_thing.id, - sample_pt_id=sample_info.sample_pt_id, + chemistry_sample_info_id=sample_info.id, ) session.add(record) session.commit() @@ -230,25 +236,27 @@ def test_delete_radionuclides(water_well_thing): """Test deleting a radionuclides record.""" with session_ctx() as session: sample_info = NMA_Chemistry_SampleInfo( - sample_pt_id=uuid4(), - sample_point_id=_next_sample_point_id(), + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), thing_id=water_well_thing.id, ) session.add(sample_info) session.commit() + session.refresh(sample_info) record = NMA_Radionuclides( - global_id=uuid4(), + nma_global_id=uuid4(), thing_id=water_well_thing.id, - sample_pt_id=sample_info.sample_pt_id, + chemistry_sample_info_id=sample_info.id, ) session.add(record) session.commit() + record_id = record.id session.delete(record) session.commit() - fetched = session.get(NMA_Radionuclides, record.global_id) + fetched = session.get(NMA_Radionuclides, record_id) assert fetched is None session.delete(sample_info) @@ -259,9 +267,12 @@ def test_delete_radionuclides(water_well_thing): def test_radionuclides_has_all_migrated_columns(): """Test that the model has all expected columns.""" expected_columns = [ + "id", + "nma_global_id", "thing_id", - "sample_pt_id", - "sample_point_id", + "chemistry_sample_info_id", + "nma_sample_pt_id", + "nma_sample_point_id", "analyte", "symbol", "sample_value", @@ -272,10 +283,9 @@ def test_radionuclides_has_all_migrated_columns(): "notes", "volume", "volume_unit", - "object_id", - "global_id", + "nma_object_id", "analyses_agency", - "wclab_id", + "nma_wclab_id", ] for column in expected_columns: @@ -289,4 +299,73 @@ def test_radionuclides_table_name(): assert NMA_Radionuclides.__tablename__ == "NMA_Radionuclides" +# ===================== FK Enforcement tests (Issue #363) ========================== + + +def test_radionuclides_fk_has_cascade(): + """NMA_Radionuclides.thing_id FK has ondelete=CASCADE.""" + col = NMA_Radionuclides.__table__.c.thing_id + fk = list(col.foreign_keys)[0] + assert fk.ondelete == "CASCADE" + + +def test_radionuclides_back_populates_thing(water_well_thing): + """NMA_Radionuclides.thing navigates back to Thing.""" + with session_ctx() as session: + well = session.merge(water_well_thing) + + # Radionuclides requires a chemistry_sample_info (which FKs to Thing) + sample_info = NMA_Chemistry_SampleInfo( + nma_sample_pt_id=uuid4(), + nma_sample_point_id=_next_sample_point_id(), + thing_id=well.id, + ) + session.add(sample_info) + session.commit() + session.refresh(sample_info) + + record = NMA_Radionuclides( + nma_global_id=uuid4(), + chemistry_sample_info_id=sample_info.id, + thing_id=well.id, + ) + session.add(record) + session.commit() + session.refresh(record) + + assert record.thing is not None + assert record.thing.id == well.id + + session.delete(record) + session.delete(sample_info) + session.commit() + + +# ===================== Integer PK tests ========================== + + +def test_radionuclides_has_integer_pk(): + """NMA_Radionuclides.id is Integer PK.""" + from sqlalchemy import Integer + + col = NMA_Radionuclides.__table__.c.id + assert col.primary_key is True + assert isinstance(col.type, Integer) + + +def test_radionuclides_nma_global_id_is_unique(): + """NMA_Radionuclides.nma_global_id is UNIQUE.""" + # Use database column name (nma_GlobalID), not Python attribute name + col = NMA_Radionuclides.__table__.c["nma_GlobalID"] + assert col.unique is True + + +def test_radionuclides_chemistry_sample_info_fk(): + """NMA_Radionuclides.chemistry_sample_info_id is Integer FK.""" + col = NMA_Radionuclides.__table__.c.chemistry_sample_info_id + fks = list(col.foreign_keys) + assert len(fks) == 1 + assert "NMA_Chemistry_SampleInfo.id" in str(fks[0].target_fullname) + + # ============= EOF ============================================= diff --git a/tests/test_soil_rock_results_legacy.py b/tests/test_soil_rock_results_legacy.py index 72ac70df6..0df8cf9ab 100644 --- a/tests/test_soil_rock_results_legacy.py +++ b/tests/test_soil_rock_results_legacy.py @@ -17,14 +17,10 @@ Unit tests for Soil_Rock_Results legacy model. These tests verify the migration of columns from the legacy Soil_Rock_Results table. -Migrated columns: -- Point_ID -> point_id -- Sample Type -> sample_type -- Date Sampled -> date_sampled -- d13C -> d13c -- d18O -> d18o -- Sampled by -> sampled_by -- SSMA_TimeStamp -> ssma_timestamp + +Updated for Integer PK schema (already had Integer PK): +- id: Integer PK (autoincrement) [unchanged] +- nma_point_id: Legacy Point_ID string (renamed from point_id) """ from db.engine import session_ctx @@ -35,7 +31,7 @@ def test_create_soil_rock_results_all_fields(water_well_thing): """Test creating a soil/rock results record with all fields.""" with session_ctx() as session: record = NMA_Soil_Rock_Results( - point_id="SR-0001", + nma_point_id="SR-0001", sample_type="Soil", date_sampled="2026-01-01", d13c=-5.5, @@ -48,7 +44,7 @@ def test_create_soil_rock_results_all_fields(water_well_thing): session.refresh(record) assert record.id is not None - assert record.point_id == "SR-0001" + assert record.nma_point_id == "SR-0001" assert record.sample_type == "Soil" assert record.date_sampled == "2026-01-01" assert record.d13c == -5.5 @@ -59,16 +55,18 @@ def test_create_soil_rock_results_all_fields(water_well_thing): session.commit() -def test_create_soil_rock_results_minimal(): +def test_create_soil_rock_results_minimal(water_well_thing): """Test creating a soil/rock results record with required fields only.""" with session_ctx() as session: - record = NMA_Soil_Rock_Results() + well = session.merge(water_well_thing) + record = NMA_Soil_Rock_Results(thing_id=well.id) session.add(record) session.commit() session.refresh(record) assert record.id is not None - assert record.point_id is None + assert record.thing_id == well.id + assert record.nma_point_id is None assert record.sample_type is None assert record.date_sampled is None assert record.d13c is None @@ -78,4 +76,62 @@ def test_create_soil_rock_results_minimal(): session.commit() +# ===================== FK Enforcement tests (Issue #363) ========================== + + +def test_soil_rock_results_validator_rejects_none_thing_id(): + """NMA_Soil_Rock_Results validator rejects None thing_id.""" + import pytest + + with pytest.raises(ValueError, match="requires a parent Thing"): + NMA_Soil_Rock_Results( + nma_point_id="ORPHAN-TEST", + thing_id=None, + ) + + +def test_soil_rock_results_thing_id_not_nullable(): + """NMA_Soil_Rock_Results.thing_id column is NOT NULL.""" + col = NMA_Soil_Rock_Results.__table__.c.thing_id + assert col.nullable is False, "thing_id should be NOT NULL" + + +def test_soil_rock_results_fk_has_cascade(): + """NMA_Soil_Rock_Results.thing_id FK has ondelete=CASCADE.""" + col = NMA_Soil_Rock_Results.__table__.c.thing_id + fk = list(col.foreign_keys)[0] + assert fk.ondelete == "CASCADE" + + +def test_soil_rock_results_back_populates_thing(water_well_thing): + """NMA_Soil_Rock_Results.thing navigates back to Thing.""" + with session_ctx() as session: + well = session.merge(water_well_thing) + record = NMA_Soil_Rock_Results( + nma_point_id="BP-SOIL-01", + thing_id=well.id, + ) + session.add(record) + session.commit() + session.refresh(record) + + assert record.thing is not None + assert record.thing.id == well.id + + session.delete(record) + session.commit() + + +# ===================== Integer PK tests ========================== + + +def test_soil_rock_results_has_integer_pk(): + """NMA_Soil_Rock_Results.id is Integer PK.""" + from sqlalchemy import Integer + + col = NMA_Soil_Rock_Results.__table__.c.id + assert col.primary_key is True + assert isinstance(col.type, Integer) + + # ============= EOF ============================================= diff --git a/tests/test_stratigraphy_legacy.py b/tests/test_stratigraphy_legacy.py new file mode 100644 index 000000000..4b0f4b1a8 --- /dev/null +++ b/tests/test_stratigraphy_legacy.py @@ -0,0 +1,136 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Unit tests for NMA_Stratigraphy (lithology log) legacy model. + +These tests verify FK enforcement for Issue #363. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement) +- nma_global_id: Legacy UUID (UNIQUE) +- nma_well_id: Legacy WellID UUID +- nma_point_id: Legacy PointID string +- nma_object_id: Legacy OBJECTID (UNIQUE) +""" + +from uuid import uuid4 + +import pytest + +from db.engine import session_ctx +from db.nma_legacy import NMA_Stratigraphy + + +def _next_global_id(): + return uuid4() + + +# ===================== CREATE tests ========================== + + +def test_create_stratigraphy_with_thing(water_well_thing): + """Test creating a stratigraphy record with a parent Thing.""" + with session_ctx() as session: + well = session.merge(water_well_thing) + record = NMA_Stratigraphy( + nma_global_id=_next_global_id(), + nma_point_id="STRAT-01", + thing_id=well.id, + strat_top=0, + strat_bottom=10, + lithology="Sand", # Max 4 chars + ) + session.add(record) + session.commit() + session.refresh(record) + + assert record.id is not None # Integer PK auto-generated + assert record.nma_global_id is not None + assert record.nma_point_id == "STRAT-01" + assert record.thing_id == well.id + + session.delete(record) + session.commit() + + +# ===================== FK Enforcement tests (Issue #363) ========================== + + +def test_stratigraphy_validator_rejects_none_thing_id(): + """NMA_Stratigraphy validator rejects None thing_id.""" + with pytest.raises(ValueError, match="requires a parent Thing"): + NMA_Stratigraphy( + nma_global_id=_next_global_id(), + nma_point_id="ORPHAN-STRAT", + thing_id=None, + ) + + +def test_stratigraphy_thing_id_not_nullable(): + """NMA_Stratigraphy.thing_id column is NOT NULL.""" + col = NMA_Stratigraphy.__table__.c.thing_id + assert col.nullable is False, "thing_id should be NOT NULL" + + +def test_stratigraphy_fk_has_cascade(): + """NMA_Stratigraphy.thing_id FK has ondelete=CASCADE.""" + col = NMA_Stratigraphy.__table__.c.thing_id + fk = list(col.foreign_keys)[0] + assert fk.ondelete == "CASCADE" + + +def test_stratigraphy_back_populates_thing(water_well_thing): + """NMA_Stratigraphy.thing navigates back to Thing.""" + with session_ctx() as session: + well = session.merge(water_well_thing) + record = NMA_Stratigraphy( + nma_global_id=_next_global_id(), + nma_point_id="BPSTRAT01", # Max 10 chars + thing_id=well.id, + strat_top=0, + strat_bottom=10, + ) + session.add(record) + session.commit() + session.refresh(record) + + assert record.thing is not None + assert record.thing.id == well.id + + session.delete(record) + session.commit() + + +# ===================== Integer PK tests ========================== + + +def test_stratigraphy_has_integer_pk(): + """NMA_Stratigraphy.id is Integer PK.""" + from sqlalchemy import Integer + + col = NMA_Stratigraphy.__table__.c.id + assert col.primary_key is True + assert isinstance(col.type, Integer) + + +def test_stratigraphy_nma_global_id_is_unique(): + """NMA_Stratigraphy.nma_global_id is UNIQUE.""" + # Use database column name (nma_GlobalID), not Python attribute name + col = NMA_Stratigraphy.__table__.c["nma_GlobalID"] + assert col.unique is True + + +# ============= EOF ============================================= diff --git a/tests/test_thing.py b/tests/test_thing.py index f60a32f7b..343f24dbf 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -1139,3 +1139,69 @@ def test_delete_thing_id_link_404_not_found(second_thing_id_link): assert response.status_code == 404 data = response.json() assert data["detail"] == f"ThingIdLink with ID {bad_id} not found." + + +# ============================================================================= +# FK Enforcement Tests - Issue #363 +# Feature: features/admin/well_data_relationships.feature +# ============================================================================= + + +class TestThingLegacyIdentifierColumns: + """Tests for Thing's legacy identifier columns (nma_pk_welldata, nma_pk_location).""" + + def test_thing_has_nma_pk_welldata_column(self): + """Thing model has nma_pk_welldata column for legacy WellID.""" + assert hasattr(Thing, "nma_pk_welldata") + + def test_thing_has_nma_pk_location_column(self): + """Thing model has nma_pk_location column for legacy LocationID.""" + assert hasattr(Thing, "nma_pk_location") + + +class TestThingNMARelationshipCollections: + """Tests for Thing's relationship collections to NMA legacy models.""" + + def test_thing_has_hydraulics_data_relationship(self): + """Thing model has hydraulics_data relationship collection.""" + assert hasattr(Thing, "hydraulics_data") + + def test_thing_has_radionuclides_relationship(self): + """Thing model has radionuclides relationship collection.""" + assert hasattr(Thing, "radionuclides") + + def test_thing_has_associated_data_relationship(self): + """Thing model has associated_data relationship collection.""" + assert hasattr(Thing, "associated_data") + + def test_thing_has_soil_rock_results_relationship(self): + """Thing model has soil_rock_results relationship collection.""" + assert hasattr(Thing, "soil_rock_results") + + +class TestThingNMACascadeDeleteConfiguration: + """Tests for cascade delete-orphan configuration on Thing relationships.""" + + def test_hydraulics_data_has_cascade_delete(self): + """hydraulics_data relationship has cascade delete configured.""" + rel = Thing.__mapper__.relationships.get("hydraulics_data") + assert rel is not None, "hydraulics_data relationship should exist" + assert "delete" in rel.cascade or "all" in rel.cascade + + def test_radionuclides_has_cascade_delete(self): + """radionuclides relationship has cascade delete configured.""" + rel = Thing.__mapper__.relationships.get("radionuclides") + assert rel is not None, "radionuclides relationship should exist" + assert "delete" in rel.cascade or "all" in rel.cascade + + def test_associated_data_has_cascade_delete(self): + """associated_data relationship has cascade delete configured.""" + rel = Thing.__mapper__.relationships.get("associated_data") + assert rel is not None, "associated_data relationship should exist" + assert "delete" in rel.cascade or "all" in rel.cascade + + def test_soil_rock_results_has_cascade_delete(self): + """soil_rock_results relationship has cascade delete configured.""" + rel = Thing.__mapper__.relationships.get("soil_rock_results") + assert rel is not None, "soil_rock_results relationship should exist" + assert "delete" in rel.cascade or "all" in rel.cascade diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 000000000..4a5d26360 --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ +# Unit tests package diff --git a/transfers/associated_data.py b/transfers/associated_data.py index be29a2c7a..ca9195b06 100644 --- a/transfers/associated_data.py +++ b/transfers/associated_data.py @@ -13,6 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== +""" +Transfer AssociatedData from NM_Aquifer to NMA_AssociatedData. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement, generated by DB) +- nma_assoc_id: Legacy UUID PK (AssocID), UNIQUE for audit +- nma_location_id: Legacy LocationId UUID, UNIQUE +- nma_point_id: Legacy PointID string +- nma_object_id: Legacy OBJECTID, UNIQUE +""" from __future__ import annotations @@ -54,7 +64,7 @@ def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: def _transfer_hook(self, session: Session) -> None: rows = [self._row_dict(row) for row in self.cleaned_df.to_dict("records")] - rows = self._dedupe_rows(rows, key="AssocID") + rows = self._dedupe_rows(rows, key="nma_AssocID") if not rows: logger.info("No AssociatedData rows to transfer") @@ -71,28 +81,35 @@ def _transfer_hook(self, session: Session) -> None: i + len(chunk) - 1, len(chunk), ) + # Upsert on nma_AssocID (legacy UUID PK, now UNIQUE) stmt = insert_stmt.values(chunk).on_conflict_do_update( - index_elements=["AssocID"], + index_elements=["nma_AssocID"], set_={ - "LocationId": excluded["LocationId"], - "PointID": excluded["PointID"], + "nma_LocationId": excluded["nma_LocationId"], + "nma_PointID": excluded["nma_PointID"], "Notes": excluded["Notes"], "Formation": excluded["Formation"], - "OBJECTID": excluded["OBJECTID"], + "nma_OBJECTID": excluded["nma_OBJECTID"], + "thing_id": excluded["thing_id"], }, ) session.execute(stmt) session.commit() def _row_dict(self, row: dict[str, Any]) -> dict[str, Any]: + point_id = row.get("PointID") return { - "LocationId": self._uuid_val(row.get("LocationId")), - "PointID": row.get("PointID"), - "AssocID": self._uuid_val(row.get("AssocID")), + # Legacy UUID PK -> nma_assoc_id (unique audit column) + "nma_AssocID": self._uuid_val(row.get("AssocID")), + # Legacy ID columns (renamed with nma_ prefix) + "nma_LocationId": self._uuid_val(row.get("LocationId")), + "nma_PointID": point_id, + "nma_OBJECTID": row.get("OBJECTID"), + # Data columns "Notes": row.get("Notes"), "Formation": row.get("Formation"), - "OBJECTID": row.get("OBJECTID"), - "thing_id": self._thing_id_cache.get(row.get("PointID")), + # FK to Thing + "thing_id": self._thing_id_cache.get(point_id), } def _dedupe_rows( diff --git a/transfers/chemistry_sampleinfo.py b/transfers/chemistry_sampleinfo.py index 3c4fd4440..395c063fd 100644 --- a/transfers/chemistry_sampleinfo.py +++ b/transfers/chemistry_sampleinfo.py @@ -16,7 +16,6 @@ from __future__ import annotations -import re from typing import Any, Optional from uuid import UUID @@ -24,7 +23,7 @@ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session -from db import NMA_Chemistry_SampleInfo, Thing +from db import NMA_Chemistry_SampleInfo, Location, LocationThingAssociation from db.engine import session_ctx from transfers.logger import logger from transfers.transferer import Transferer @@ -36,6 +35,19 @@ class ChemistrySampleInfoTransferer(Transferer): Transfer for the legacy Chemistry_SampleInfo table. Loads the CSV and upserts into the legacy table. + + Updated for Integer PK schema: + - id: Integer PK (autoincrement, generated by DB) + - nma_sample_pt_id: Legacy UUID PK (SamplePtID), UNIQUE for audit + - nma_wclab_id: Legacy WCLab_ID + - nma_sample_point_id: Legacy SamplePointID + - nma_object_id: Legacy OBJECTID, UNIQUE + - nma_location_id: Legacy LocationId UUID (for audit trail) + + FK to Thing: + - thing_id: Integer FK to Thing.id + - Linked via LocationId -> Location.nma_pk_location -> LocationThingAssociation -> Thing.id + - Requires Thing and Location records to be transferred first """ source_table = "Chemistry_SampleInfo" @@ -48,28 +60,62 @@ def __init__(self, *args, batch_size: int = 1000, **kwargs): self._build_thing_id_cache() def _build_thing_id_cache(self): - """Build cache of Thing.name -> thing.id to prevent orphan records.""" + """Build cache of Location.nma_pk_location (UUID) -> Thing.id to prevent orphan records. + + Uses LocationId from CSV -> Location.nma_pk_location -> LocationThingAssociation -> Thing.id. + """ with session_ctx() as session: - things = session.query(Thing.name, Thing.id).all() - normalized = {} - for name, thing_id in things: - normalized_name = self._normalize_for_thing_match(name) - if not normalized_name: + # Query Location.nma_pk_location joined with LocationThingAssociation to get Thing.id + results = ( + session.query( + Location.nma_pk_location, LocationThingAssociation.thing_id + ) + .join( + LocationThingAssociation, + Location.id == LocationThingAssociation.location_id, + ) + .filter(Location.nma_pk_location.isnot(None)) + .all() + ) + location_to_thing = {} + for nma_pk_location, thing_id in results: + if nma_pk_location is None: continue + # Normalize UUID to string for consistent lookup + location_key = str(nma_pk_location).lower() if ( - normalized_name in normalized - and normalized[normalized_name] != thing_id + location_key in location_to_thing + and location_to_thing[location_key] != thing_id ): logger.warning( - "Duplicate Thing match key '%s' for ids %s and %s", - normalized_name, - normalized[normalized_name], + "Duplicate Location match key '%s' for thing_ids %s and %s", + location_key, + location_to_thing[location_key], thing_id, ) continue - normalized[normalized_name] = thing_id - self._thing_id_cache = normalized - logger.info(f"Built Thing ID cache with {len(self._thing_id_cache)} entries") + location_to_thing[location_key] = thing_id + self._thing_id_cache = location_to_thing + logger.info( + f"Built Location->Thing ID cache with {len(self._thing_id_cache)} entries" + ) + + # Enforce transfer order: Things and Locations must be transferred before ChemistrySampleInfo + if len(self._thing_id_cache) == 0: + raise RuntimeError( + "ChemistrySampleInfo transfer requires Thing records to exist. " + "Ensure the Well/Thing transfer runs before ChemistrySampleInfo transfer." + ) + + # Also verify Locations exist (required dependency) + with session_ctx() as session: + location_count = session.query(Location).count() + if location_count == 0: + raise RuntimeError( + "ChemistrySampleInfo transfer requires Location records to exist. " + "Ensure the Location transfer runs before ChemistrySampleInfo transfer." + ) + logger.info(f"Verified {location_count} Location records exist") def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: input_df = read_csv(self.source_table, parse_dates=["CollectionDate"]) @@ -80,57 +126,37 @@ def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: def _filter_to_valid_things(self, df: pd.DataFrame) -> pd.DataFrame: """ - Filter to only include rows where SamplePointID matches an existing Thing. + Filter to only include rows where LocationId matches an existing Location.nma_pk_location + that is linked to a Thing via LocationThingAssociation. Prevents orphan ChemistrySampleInfo records. - Uses cached Thing lookups for performance. + Uses cached Location->Thing lookups for performance. """ - # Use cached Thing names (keys of thing_id_cache) - valid_point_ids = set(self._thing_id_cache.keys()) + # Use cached Location UUIDs (keys of thing_id_cache) + valid_location_ids = set(self._thing_id_cache.keys()) + + # Normalize LocationId UUID to lowercase string for matching + def normalize_location_id(value: Any) -> Optional[str]: + if pd.isna(value): + return None + return str(value).strip().lower() - # Normalize SamplePointID to handle suffixed sample counts (e.g. AB-0002A -> AB-0002). - normalized_ids = df["SamplePointID"].apply(self._normalize_for_thing_match) + normalized_ids = df["LocationId"].apply(normalize_location_id) - # Filter to rows where SamplePointID exists as a Thing.name + # Filter to rows where LocationId exists in Location->Thing cache before_count = len(df) - filtered_df = df[normalized_ids.isin(valid_point_ids)].copy() + filtered_df = df[normalized_ids.isin(valid_location_ids)].copy() after_count = len(filtered_df) if before_count > after_count: skipped = before_count - after_count logger.warning( - f"Filtered out {skipped} ChemistrySampleInfo records without matching Things " + f"Filtered out {skipped} ChemistrySampleInfo records without matching Location->Thing " f"({after_count} valid, {skipped} orphan records prevented)" ) return filtered_df - @staticmethod - def _normalize_sample_point_id(value: Any) -> Optional[str]: - """ - Normalize SamplePointID for Thing matching by removing trailing alpha suffixes - used to denote multiple samples (e.g. AB-0002A -> AB-0002). - """ - if pd.isna(value): - return None - text = str(value).strip() - if not text: - return None - match = re.match(r"^(?P.*\d)[A-Za-z]+$", text) - if match: - return match.group("base") - return text - - @classmethod - def _normalize_for_thing_match(cls, value: Any) -> Optional[str]: - """ - Normalize IDs for Thing matching (strip suffixes, trim, uppercase). - """ - normalized = cls._normalize_sample_point_id(value) - if not normalized: - return None - return normalized.strip().upper() - def _filter_to_valid_sample_pt_ids(self, df: pd.DataFrame) -> pd.DataFrame: """Filter to rows with a valid SamplePtID UUID (required for idempotent upserts).""" @@ -168,13 +194,13 @@ def _transfer_hook(self, session: Session) -> None: lookup_miss_count = 0 for row in self.cleaned_df.to_dict("records"): row_dict = self._row_dict(row) - if row_dict.get("SamplePtID") is None: + if row_dict.get("nma_SamplePtID") is None: skipped_sample_pt_id_count += 1 logger.warning( - "Skipping ChemistrySampleInfo OBJECTID=%s SamplePointID=%s - " - "SamplePtID missing or invalid", - row_dict.get("OBJECTID"), - row_dict.get("SamplePointID"), + "Skipping ChemistrySampleInfo nma_OBJECTID=%s nma_SamplePointID=%s - " + "nma_SamplePtID missing or invalid", + row_dict.get("nma_OBJECTID"), + row_dict.get("nma_SamplePointID"), ) continue # Skip rows without valid thing_id (orphan prevention) @@ -182,15 +208,15 @@ def _transfer_hook(self, session: Session) -> None: skipped_orphan_count += 1 lookup_miss_count += 1 logger.warning( - f"Skipping ChemistrySampleInfo OBJECTID={row_dict.get('OBJECTID')} " - f"SamplePointID={row_dict.get('SamplePointID')} - Thing not found" + f"Skipping ChemistrySampleInfo nma_OBJECTID={row_dict.get('nma_OBJECTID')} " + f"nma_LocationId={row_dict.get('nma_LocationId')} - Thing not found via Location" ) continue row_dicts.append(row_dict) if skipped_sample_pt_id_count > 0: logger.warning( - "Skipped %s ChemistrySampleInfo records without valid SamplePtID", + "Skipped %s ChemistrySampleInfo records without valid nma_SamplePtID", skipped_sample_pt_id_count, ) if skipped_orphan_count > 0: @@ -200,10 +226,11 @@ def _transfer_hook(self, session: Session) -> None: ) if lookup_miss_count > 0: logger.warning( - "ChemistrySampleInfo Thing lookup misses: %s", lookup_miss_count + "ChemistrySampleInfo Location->Thing lookup misses: %s", + lookup_miss_count, ) - rows = self._dedupe_rows(row_dicts, key="OBJECTID") + rows = self._dedupe_rows(row_dicts, key="nma_OBJECTID") insert_stmt = insert(NMA_Chemistry_SampleInfo) excluded = insert_stmt.excluded @@ -213,12 +240,13 @@ def _transfer_hook(self, session: Session) -> None: logger.info( f"Upserting batch {i}-{i+len(chunk)-1} ({len(chunk)} rows) into Chemistry_SampleInfo" ) + # Upsert on nma_SamplePtID (the legacy UUID PK, now UNIQUE) stmt = insert_stmt.values(chunk).on_conflict_do_update( - index_elements=["SamplePtID"], + index_elements=["nma_SamplePtID"], set_={ "thing_id": excluded.thing_id, # Required FK - prevent orphans - "SamplePointID": excluded.SamplePointID, - "WCLab_ID": excluded.WCLab_ID, + "nma_SamplePointID": excluded.nma_SamplePointID, + "nma_WCLab_ID": excluded.nma_WCLab_ID, "CollectionDate": excluded.CollectionDate, "CollectionMethod": excluded.CollectionMethod, "CollectedBy": excluded.CollectedBy, @@ -232,8 +260,8 @@ def _transfer_hook(self, session: Session) -> None: "PublicRelease": excluded.PublicRelease, "AddedDaytoDate": excluded.AddedDaytoDate, "AddedMonthDaytoDate": excluded.AddedMonthDaytoDate, - "LocationId": excluded.LocationId, - "OBJECTID": excluded.OBJECTID, + "nma_LocationId": excluded.nma_LocationId, + "nma_OBJECTID": excluded.nma_OBJECTID, "SampleNotes": excluded.SampleNotes, }, ) @@ -290,27 +318,33 @@ def bool_val(key: str) -> Optional[bool]: if hasattr(collection_date, "to_pydatetime"): collection_date = collection_date.to_pydatetime() - # Look up Thing by SamplePointID to prevent orphan records - sample_point_id = val("SamplePointID") - normalized_sample_point_id = self._normalize_for_thing_match(sample_point_id) + # Look up Thing by LocationId to prevent orphan records + # LocationId -> Location.nma_pk_location -> LocationThingAssociation -> Thing.id + location_id_raw = val("LocationId") thing_id = None - if ( - normalized_sample_point_id - and normalized_sample_point_id in self._thing_id_cache - ): - thing_id = self._thing_id_cache[normalized_sample_point_id] - # If Thing not found, thing_id remains None and will be filtered out - if thing_id is None and sample_point_id is not None: - logger.debug( - "ChemistrySampleInfo Thing lookup miss: SamplePointID=%s normalized=%s", - sample_point_id, - normalized_sample_point_id, - ) + if location_id_raw is not None: + normalized_location_id = str(location_id_raw).strip().lower() + if normalized_location_id in self._thing_id_cache: + thing_id = self._thing_id_cache[normalized_location_id] + else: + logger.debug( + "ChemistrySampleInfo Thing lookup miss: LocationId=%s normalized=%s", + location_id_raw, + normalized_location_id, + ) + # Map to new column names (nma_ prefix for legacy columns) return { - "SamplePtID": uuid_val("SamplePtID"), - "WCLab_ID": str_val("WCLab_ID"), - "SamplePointID": str_val("SamplePointID"), + # Legacy UUID PK -> nma_sample_pt_id (unique audit column) + "nma_SamplePtID": uuid_val("SamplePtID"), + # Legacy ID columns (renamed with nma_ prefix) + "nma_WCLab_ID": str_val("WCLab_ID"), + "nma_SamplePointID": str_val("SamplePointID"), + "nma_LocationId": uuid_val("LocationId"), + "nma_OBJECTID": val("OBJECTID"), + # FK to Thing + "thing_id": thing_id, + # Data columns (unchanged names) "CollectionDate": collection_date, "CollectionMethod": str_val("CollectionMethod"), "CollectedBy": str_val("CollectedBy"), @@ -325,9 +359,6 @@ def bool_val(key: str) -> Optional[bool]: "AddedDaytoDate": bool_val("AddedDaytoDate"), "AddedMonthDaytoDate": bool_val("AddedMonthDaytoDate"), "SampleNotes": str_val("SampleNotes"), - "LocationId": uuid_val("LocationId"), - "OBJECTID": val("OBJECTID"), - "thing_id": thing_id, } def _dedupe_rows( diff --git a/transfers/field_parameters_transfer.py b/transfers/field_parameters_transfer.py index b9a4fe6c8..d7dc77d73 100644 --- a/transfers/field_parameters_transfer.py +++ b/transfers/field_parameters_transfer.py @@ -16,7 +16,16 @@ """Transfer FieldParameters data from NM_Aquifer to NMA_FieldParameters. This transfer requires ChemistrySampleInfo to be backfilled first. Each -FieldParameters record links to a ChemistrySampleInfo record via SamplePtID. +FieldParameters record links to a ChemistrySampleInfo record via chemistry_sample_info_id. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement, generated by DB) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit +- nma_sample_point_id: Legacy SamplePointID string +- nma_object_id: Legacy OBJECTID (Identity) +- nma_wclab_id: Legacy WCLab_ID """ from __future__ import annotations @@ -39,8 +48,8 @@ class FieldParametersTransferer(Transferer): """ Transfer FieldParameters records to NMA_FieldParameters. - Looks up ChemistrySampleInfo by SamplePtID and creates linked - FieldParameters records. Uses upsert for idempotent transfers. + Looks up ChemistrySampleInfo by nma_sample_pt_id (legacy UUID) and creates linked + FieldParameters records with Integer FK. Uses upsert for idempotent transfers. """ source_table = "FieldParameters" @@ -48,16 +57,26 @@ class FieldParametersTransferer(Transferer): def __init__(self, *args, batch_size: int = 1000, **kwargs): super().__init__(*args, **kwargs) self.batch_size = batch_size - self._sample_pt_ids: set[UUID] = set() - self._build_sample_pt_id_cache() + # Cache: legacy UUID -> Integer id + self._sample_info_cache: dict[UUID, int] = {} + self._build_sample_info_cache() - def _build_sample_pt_id_cache(self) -> None: - """Build cache of ChemistrySampleInfo.SamplePtID values.""" + def _build_sample_info_cache(self) -> None: + """Build cache of nma_sample_pt_id -> id for FK lookups.""" with session_ctx() as session: - sample_infos = session.query(NMA_Chemistry_SampleInfo.sample_pt_id).all() - self._sample_pt_ids = {sample_pt_id for (sample_pt_id,) in sample_infos} + sample_infos = ( + session.query( + NMA_Chemistry_SampleInfo.nma_sample_pt_id, + NMA_Chemistry_SampleInfo.id, + ) + .filter(NMA_Chemistry_SampleInfo.nma_sample_pt_id.isnot(None)) + .all() + ) + self._sample_info_cache = { + nma_sample_pt_id: csi_id for nma_sample_pt_id, csi_id in sample_infos + } logger.info( - f"Built ChemistrySampleInfo cache with {len(self._sample_pt_ids)} entries" + f"Built ChemistrySampleInfo cache with {len(self._sample_info_cache)} entries" ) def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -71,7 +90,7 @@ def _filter_to_valid_sample_infos(self, df: pd.DataFrame) -> pd.DataFrame: This prevents orphan records and ensures the FK constraint will be satisfied. """ - valid_sample_pt_ids = self._sample_pt_ids + valid_sample_pt_ids = set(self._sample_info_cache.keys()) before_count = len(df) mask = df["SamplePtID"].apply( lambda value: self._uuid_val(value) in valid_sample_pt_ids @@ -92,7 +111,7 @@ def _transfer_hook(self, session: Session) -> None: """ Override transfer hook to use batch upsert for idempotent transfers. - Uses ON CONFLICT DO UPDATE on GlobalID. + Uses ON CONFLICT DO UPDATE on nma_GlobalID (legacy UUID PK, now UNIQUE). """ limit = self.flags.get("LIMIT", 0) df = self.cleaned_df @@ -118,18 +137,20 @@ def _transfer_hook(self, session: Session) -> None: for i in range(0, len(rows), self.batch_size): chunk = rows[i : i + self.batch_size] logger.info(f"Upserting batch {i}-{i+len(chunk)-1} ({len(chunk)} rows)") + # Upsert on nma_GlobalID (legacy UUID PK, now UNIQUE) stmt = insert_stmt.values(chunk).on_conflict_do_update( - index_elements=["GlobalID"], + index_elements=["nma_GlobalID"], set_={ - "SamplePtID": excluded.SamplePtID, - "SamplePointID": excluded.SamplePointID, + "chemistry_sample_info_id": excluded.chemistry_sample_info_id, + "nma_SamplePtID": excluded.nma_SamplePtID, + "nma_SamplePointID": excluded.nma_SamplePointID, "FieldParameter": excluded.FieldParameter, "SampleValue": excluded.SampleValue, "Units": excluded.Units, "Notes": excluded.Notes, - "OBJECTID": excluded.OBJECTID, + "nma_OBJECTID": excluded.nma_OBJECTID, "AnalysesAgency": excluded.AnalysesAgency, - "WCLab_ID": excluded.WCLab_ID, + "nma_WCLab_ID": excluded.nma_WCLab_ID, }, ) session.execute(stmt) @@ -138,8 +159,9 @@ def _transfer_hook(self, session: Session) -> None: def _row_to_dict(self, row) -> Optional[dict[str, Any]]: """Convert a DataFrame row to a dict for upsert.""" - sample_pt_id = self._uuid_val(getattr(row, "SamplePtID", None)) - if sample_pt_id is None: + # Get legacy UUID FK + legacy_sample_pt_id = self._uuid_val(getattr(row, "SamplePtID", None)) + if legacy_sample_pt_id is None: self._capture_error( getattr(row, "SamplePtID", None), f"Invalid SamplePtID: {getattr(row, 'SamplePtID', None)}", @@ -147,16 +169,18 @@ def _row_to_dict(self, row) -> Optional[dict[str, Any]]: ) return None - if sample_pt_id not in self._sample_pt_ids: + # Look up Integer FK from cache + chemistry_sample_info_id = self._sample_info_cache.get(legacy_sample_pt_id) + if chemistry_sample_info_id is None: self._capture_error( - sample_pt_id, - f"ChemistrySampleInfo not found for SamplePtID: {sample_pt_id}", + legacy_sample_pt_id, + f"ChemistrySampleInfo not found for SamplePtID: {legacy_sample_pt_id}", "SamplePtID", ) return None - global_id = self._uuid_val(getattr(row, "GlobalID", None)) - if global_id is None: + nma_global_id = self._uuid_val(getattr(row, "GlobalID", None)) + if nma_global_id is None: self._capture_error( getattr(row, "GlobalID", None), f"Invalid GlobalID: {getattr(row, 'GlobalID', None)}", @@ -165,23 +189,28 @@ def _row_to_dict(self, row) -> Optional[dict[str, Any]]: return None return { - "GlobalID": global_id, - "SamplePtID": sample_pt_id, - "SamplePointID": self._safe_str(row, "SamplePointID"), + # Legacy UUID PK -> nma_global_id (unique audit column) + "nma_GlobalID": nma_global_id, + # New Integer FK to ChemistrySampleInfo + "chemistry_sample_info_id": chemistry_sample_info_id, + # Legacy ID columns (renamed with nma_ prefix) + "nma_SamplePtID": legacy_sample_pt_id, + "nma_SamplePointID": self._safe_str(row, "SamplePointID"), + "nma_OBJECTID": self._safe_int(row, "OBJECTID"), + "nma_WCLab_ID": self._safe_str(row, "WCLab_ID"), + # Data columns "FieldParameter": self._safe_str(row, "FieldParameter"), "SampleValue": self._safe_float(row, "SampleValue"), "Units": self._safe_str(row, "Units"), "Notes": self._safe_str(row, "Notes"), - "OBJECTID": self._safe_int(row, "OBJECTID"), "AnalysesAgency": self._safe_str(row, "AnalysesAgency"), - "WCLab_ID": self._safe_str(row, "WCLab_ID"), } def _dedupe_rows(self, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: """Dedupe rows by unique key to avoid ON CONFLICT loops. Later rows win.""" deduped = {} for row in rows: - key = row.get("GlobalID") + key = row.get("nma_GlobalID") if key is None: continue deduped[key] = row diff --git a/transfers/hydraulicsdata.py b/transfers/hydraulicsdata.py index a1e1b7f4f..bfaee00f5 100644 --- a/transfers/hydraulicsdata.py +++ b/transfers/hydraulicsdata.py @@ -13,6 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +""" +Transfer HydraulicsData from NM_Aquifer to NMA_HydraulicsData. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement, generated by DB) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- nma_well_id: Legacy WellID UUID +- nma_point_id: Legacy PointID string +- nma_object_id: Legacy OBJECTID, UNIQUE +""" from __future__ import annotations @@ -33,6 +43,8 @@ class HydraulicsDataTransferer(Transferer): """ Transfer for the legacy NMA_HydraulicsData table. + + Uses Integer PK with legacy UUID stored in nma_global_id for audit. """ source_table = "HydraulicsData" @@ -75,9 +87,9 @@ def _transfer_hook(self, session: Session) -> None: if row_dict.get("thing_id") is None: skipped_count += 1 logger.warning( - "Skipping HydraulicsData GlobalID=%s PointID=%s - Thing not found", - row_dict.get("GlobalID"), - row_dict.get("PointID"), + "Skipping HydraulicsData nma_GlobalID=%s nma_PointID=%s - Thing not found", + row_dict.get("nma_GlobalID"), + row_dict.get("nma_PointID"), ) continue row_dicts.append(row_dict) @@ -88,7 +100,7 @@ def _transfer_hook(self, session: Session) -> None: f"(orphan prevention)" ) - rows = self._dedupe_rows(row_dicts, key="GlobalID") + rows = self._dedupe_rows(row_dicts, key="nma_GlobalID") insert_stmt = insert(NMA_HydraulicsData) excluded = insert_stmt.excluded @@ -98,11 +110,12 @@ def _transfer_hook(self, session: Session) -> None: logger.info( f"Upserting batch {i}-{i+len(chunk)-1} ({len(chunk)} rows) into NMA_HydraulicsData" ) + # Upsert on nma_GlobalID (legacy UUID PK, now UNIQUE) stmt = insert_stmt.values(chunk).on_conflict_do_update( - index_elements=["GlobalID"], + index_elements=["nma_GlobalID"], set_={ - "WellID": excluded["WellID"], - "PointID": excluded["PointID"], + "nma_WellID": excluded["nma_WellID"], + "nma_PointID": excluded["nma_PointID"], "HydraulicUnit": excluded["HydraulicUnit"], "thing_id": excluded["thing_id"], "TestTop": excluded["TestTop"], @@ -121,7 +134,7 @@ def _transfer_hook(self, session: Session) -> None: "P (decimal fraction)": excluded["P (decimal fraction)"], "k (darcy)": excluded["k (darcy)"], "Data Source": excluded["Data Source"], - "OBJECTID": excluded["OBJECTID"], + "nma_OBJECTID": excluded["nma_OBJECTID"], }, ) session.execute(stmt) @@ -155,12 +168,18 @@ def as_int(key: str) -> Optional[int]: except (TypeError, ValueError): return None + point_id = val("PointID") return { - "GlobalID": as_uuid("GlobalID"), - "WellID": as_uuid("WellID"), - "PointID": val("PointID"), + # Legacy UUID PK -> nma_global_id (unique audit column) + "nma_GlobalID": as_uuid("GlobalID"), + # Legacy ID columns (renamed with nma_ prefix) + "nma_WellID": as_uuid("WellID"), + "nma_PointID": point_id, + "nma_OBJECTID": as_int("OBJECTID"), + # FK to Thing + "thing_id": self._thing_id_cache.get(point_id), + # Data columns "HydraulicUnit": val("HydraulicUnit"), - "thing_id": self._thing_id_cache.get(val("PointID")), "TestTop": as_int("TestTop"), "TestBottom": as_int("TestBottom"), "HydraulicUnitType": val("HydraulicUnitType"), @@ -177,7 +196,6 @@ def as_int(key: str) -> Optional[int]: "P (decimal fraction)": val("P (decimal fraction)"), "k (darcy)": val("k (darcy)"), "Data Source": val("Data Source"), - "OBJECTID": as_int("OBJECTID"), } def _dedupe_rows( diff --git a/transfers/major_chemistry.py b/transfers/major_chemistry.py index d222fb0c8..1aab8da75 100644 --- a/transfers/major_chemistry.py +++ b/transfers/major_chemistry.py @@ -13,6 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +""" +Transfer MajorChemistry data from NM_Aquifer to NMA_MajorChemistry. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement, generated by DB) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit +- nma_sample_point_id: Legacy SamplePointID string +- nma_object_id: Legacy OBJECTID, UNIQUE +- nma_wclab_id: Legacy WCLab_ID +""" from __future__ import annotations @@ -34,6 +46,8 @@ class MajorChemistryTransferer(Transferer): """ Transfer for the legacy MajorChemistry table. + + Uses Integer FK to ChemistrySampleInfo via chemistry_sample_info_id. """ source_table = "MajorChemistry" @@ -41,15 +55,26 @@ class MajorChemistryTransferer(Transferer): def __init__(self, *args, batch_size: int = 1000, **kwargs): super().__init__(*args, **kwargs) self.batch_size = batch_size - self._sample_pt_ids: set[UUID] = set() - self._build_sample_pt_id_cache() + # Cache: legacy UUID -> Integer id + self._sample_info_cache: dict[UUID, int] = {} + self._build_sample_info_cache() - def _build_sample_pt_id_cache(self) -> None: + def _build_sample_info_cache(self) -> None: + """Build cache of nma_sample_pt_id -> id for FK lookups.""" with session_ctx() as session: - sample_infos = session.query(NMA_Chemistry_SampleInfo.sample_pt_id).all() - self._sample_pt_ids = {sample_pt_id for (sample_pt_id,) in sample_infos} + sample_infos = ( + session.query( + NMA_Chemistry_SampleInfo.nma_sample_pt_id, + NMA_Chemistry_SampleInfo.id, + ) + .filter(NMA_Chemistry_SampleInfo.nma_sample_pt_id.isnot(None)) + .all() + ) + self._sample_info_cache = { + nma_sample_pt_id: csi_id for nma_sample_pt_id, csi_id in sample_infos + } logger.info( - f"Built ChemistrySampleInfo cache with {len(self._sample_pt_ids)} entries" + f"Built ChemistrySampleInfo cache with {len(self._sample_info_cache)} entries" ) def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -58,7 +83,7 @@ def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: return input_df, cleaned_df def _filter_to_valid_sample_infos(self, df: pd.DataFrame) -> pd.DataFrame: - valid_sample_pt_ids = self._sample_pt_ids + valid_sample_pt_ids = set(self._sample_info_cache.keys()) mask = df["SamplePtID"].apply( lambda value: self._uuid_val(value) in valid_sample_pt_ids ) @@ -78,26 +103,39 @@ def _filter_to_valid_sample_infos(self, df: pd.DataFrame) -> pd.DataFrame: def _transfer_hook(self, session: Session) -> None: row_dicts = [] skipped_global_id = 0 + skipped_csi_id = 0 for row in self.cleaned_df.to_dict("records"): row_dict = self._row_dict(row) if row_dict is None: continue - if row_dict.get("GlobalID") is None: + if row_dict.get("nma_GlobalID") is None: skipped_global_id += 1 logger.warning( - "Skipping MajorChemistry SamplePtID=%s - GlobalID missing or invalid", - row_dict.get("SamplePtID"), + "Skipping MajorChemistry nma_SamplePtID=%s - nma_GlobalID missing or invalid", + row_dict.get("nma_SamplePtID"), + ) + continue + if row_dict.get("chemistry_sample_info_id") is None: + skipped_csi_id += 1 + logger.warning( + "Skipping MajorChemistry nma_SamplePtID=%s - chemistry_sample_info_id not found", + row_dict.get("nma_SamplePtID"), ) continue row_dicts.append(row_dict) if skipped_global_id > 0: logger.warning( - "Skipped %s MajorChemistry records without valid GlobalID", + "Skipped %s MajorChemistry records without valid nma_GlobalID", skipped_global_id, ) + if skipped_csi_id > 0: + logger.warning( + "Skipped %s MajorChemistry records without valid chemistry_sample_info_id", + skipped_csi_id, + ) - rows = self._dedupe_rows(row_dicts, key="GlobalID") + rows = self._dedupe_rows(row_dicts, key="nma_GlobalID") insert_stmt = insert(NMA_MajorChemistry) excluded = insert_stmt.excluded @@ -106,11 +144,13 @@ def _transfer_hook(self, session: Session) -> None: logger.info( f"Upserting batch {i}-{i+len(chunk)-1} ({len(chunk)} rows) into MajorChemistry" ) + # Upsert on nma_GlobalID (legacy UUID PK, now UNIQUE) stmt = insert_stmt.values(chunk).on_conflict_do_update( - index_elements=["GlobalID"], + index_elements=["nma_GlobalID"], set_={ - "SamplePtID": excluded.SamplePtID, - "SamplePointID": excluded.SamplePointID, + "chemistry_sample_info_id": excluded.chemistry_sample_info_id, + "nma_SamplePtID": excluded.nma_SamplePtID, + "nma_SamplePointID": excluded.nma_SamplePointID, "Analyte": excluded.Analyte, "Symbol": excluded.Symbol, "SampleValue": excluded.SampleValue, @@ -121,9 +161,9 @@ def _transfer_hook(self, session: Session) -> None: "Notes": excluded.Notes, "Volume": excluded.Volume, "VolumeUnit": excluded.VolumeUnit, - "OBJECTID": excluded.OBJECTID, + "nma_OBJECTID": excluded.nma_OBJECTID, "AnalysesAgency": excluded.AnalysesAgency, - "WCLab_ID": excluded.WCLab_ID, + "nma_WCLab_ID": excluded.nma_WCLab_ID, }, ) session.execute(stmt) @@ -161,8 +201,9 @@ def int_val(key: str) -> Optional[int]: if isinstance(analysis_date, datetime): analysis_date = analysis_date.replace(tzinfo=None) - sample_pt_id = self._uuid_val(val("SamplePtID")) - if sample_pt_id is None: + # Get legacy UUID FK + legacy_sample_pt_id = self._uuid_val(val("SamplePtID")) + if legacy_sample_pt_id is None: self._capture_error( val("SamplePtID"), f"Invalid SamplePtID: {val('SamplePtID')}", @@ -170,11 +211,22 @@ def int_val(key: str) -> Optional[int]: ) return None - global_id = self._uuid_val(val("GlobalID")) + # Look up Integer FK from cache + chemistry_sample_info_id = self._sample_info_cache.get(legacy_sample_pt_id) + + nma_global_id = self._uuid_val(val("GlobalID")) return { - "SamplePtID": sample_pt_id, - "SamplePointID": val("SamplePointID"), + # Legacy UUID PK -> nma_global_id (unique audit column) + "nma_GlobalID": nma_global_id, + # New Integer FK to ChemistrySampleInfo + "chemistry_sample_info_id": chemistry_sample_info_id, + # Legacy ID columns (renamed with nma_ prefix) + "nma_SamplePtID": legacy_sample_pt_id, + "nma_SamplePointID": val("SamplePointID"), + "nma_OBJECTID": val("OBJECTID"), + "nma_WCLab_ID": val("WCLab_ID"), + # Data columns "Analyte": val("Analyte"), "Symbol": val("Symbol"), "SampleValue": float_val("SampleValue"), @@ -185,10 +237,7 @@ def int_val(key: str) -> Optional[int]: "Notes": val("Notes"), "Volume": int_val("Volume"), "VolumeUnit": val("VolumeUnit"), - "OBJECTID": val("OBJECTID"), - "GlobalID": global_id, "AnalysesAgency": val("AnalysesAgency"), - "WCLab_ID": val("WCLab_ID"), } def _dedupe_rows( diff --git a/transfers/minor_trace_chemistry_transfer.py b/transfers/minor_trace_chemistry_transfer.py index 60ade7560..daeef7923 100644 --- a/transfers/minor_trace_chemistry_transfer.py +++ b/transfers/minor_trace_chemistry_transfer.py @@ -18,7 +18,13 @@ This transfer requires ChemistrySampleInfo to be backfilled first (which links to Thing via thing_id). Each MinorTraceChemistry record links to a ChemistrySampleInfo -record via chemistry_sample_info_id. +record via chemistry_sample_info_id (Integer FK). + +Updated for Integer PK schema: +- id: Integer PK (autoincrement, generated by DB) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_chemistry_sample_info_uuid: Legacy UUID FK for audit """ from __future__ import annotations @@ -42,8 +48,8 @@ class MinorTraceChemistryTransferer(Transferer): """ Transfer MinorandTraceChemistry records to NMA_MinorTraceChemistry. - Looks up ChemistrySampleInfo by SamplePtID and creates linked - NMA_MinorTraceChemistry records. Uses upsert for idempotent transfers. + Looks up ChemistrySampleInfo by nma_sample_pt_id (legacy UUID) and creates linked + NMA_MinorTraceChemistry records with Integer FK. Uses upsert for idempotent transfers. """ source_table = "MinorandTraceChemistry" @@ -51,17 +57,26 @@ class MinorTraceChemistryTransferer(Transferer): def __init__(self, *args, batch_size: int = 1000, **kwargs): super().__init__(*args, **kwargs) self.batch_size = batch_size - # Cache ChemistrySampleInfo SamplePtIDs for FK validation - self._sample_pt_ids: set[UUID] = set() - self._build_sample_pt_id_cache() + # Cache ChemistrySampleInfo: legacy UUID -> Integer id + self._sample_info_cache: dict[UUID, int] = {} + self._build_sample_info_cache() - def _build_sample_pt_id_cache(self): - """Build cache of ChemistrySampleInfo.SamplePtID values.""" + def _build_sample_info_cache(self): + """Build cache of ChemistrySampleInfo.nma_sample_pt_id -> ChemistrySampleInfo.id.""" with session_ctx() as session: - sample_infos = session.query(NMA_Chemistry_SampleInfo.sample_pt_id).all() - self._sample_pt_ids = {sample_pt_id for (sample_pt_id,) in sample_infos} + sample_infos = ( + session.query( + NMA_Chemistry_SampleInfo.nma_sample_pt_id, + NMA_Chemistry_SampleInfo.id, + ) + .filter(NMA_Chemistry_SampleInfo.nma_sample_pt_id.isnot(None)) + .all() + ) + self._sample_info_cache = { + nma_sample_pt_id: csi_id for nma_sample_pt_id, csi_id in sample_infos + } logger.info( - f"Built ChemistrySampleInfo cache with {len(self._sample_pt_ids)} entries" + f"Built ChemistrySampleInfo cache with {len(self._sample_info_cache)} entries" ) def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -76,7 +91,7 @@ def _filter_to_valid_sample_infos(self, df: pd.DataFrame) -> pd.DataFrame: This prevents orphan records and ensures the FK constraint will be satisfied. """ - valid_sample_pt_ids = self._sample_pt_ids + valid_sample_pt_ids = set(self._sample_info_cache.keys()) before_count = len(df) mask = df["SamplePtID"].apply( @@ -98,7 +113,7 @@ def _transfer_hook(self, session: Session) -> None: """ Override transfer hook to use batch upsert for idempotent transfers. - Uses ON CONFLICT DO UPDATE on (chemistry_sample_info_id, analyte). + Uses ON CONFLICT DO UPDATE on nma_GlobalID (the legacy UUID PK, now UNIQUE). """ limit = self.flags.get("LIMIT", 0) df = self.cleaned_df @@ -116,7 +131,7 @@ def _transfer_hook(self, session: Session) -> None: logger.warning("No valid rows to transfer") return - # Dedupe by GlobalID to avoid PK conflicts. + # Dedupe by nma_GlobalID to avoid PK conflicts. rows = self._dedupe_rows(row_dicts) logger.info(f"Upserting {len(rows)} MinorTraceChemistry records") @@ -126,19 +141,22 @@ def _transfer_hook(self, session: Session) -> None: for i in range(0, len(rows), self.batch_size): chunk = rows[i : i + self.batch_size] logger.info(f"Upserting batch {i}-{i+len(chunk)-1} ({len(chunk)} rows)") + # Upsert on nma_GlobalID (legacy UUID PK, now UNIQUE) stmt = insert_stmt.values(chunk).on_conflict_do_update( - index_elements=["GlobalID"], + index_elements=["nma_GlobalID"], set_={ - "SampleValue": excluded.SampleValue, - "Units": excluded.Units, - "Symbol": excluded.Symbol, - "AnalysisMethod": excluded.AnalysisMethod, - "AnalysisDate": excluded.AnalysisDate, - "Notes": excluded.Notes, - "AnalysesAgency": excluded.AnalysesAgency, - "Uncertainty": excluded.Uncertainty, - "Volume": excluded.Volume, - "VolumeUnit": excluded.VolumeUnit, + "chemistry_sample_info_id": excluded.chemistry_sample_info_id, + "nma_chemistry_sample_info_uuid": excluded.nma_chemistry_sample_info_uuid, + "sample_value": excluded.sample_value, + "units": excluded.units, + "symbol": excluded.symbol, + "analysis_method": excluded.analysis_method, + "analysis_date": excluded.analysis_date, + "notes": excluded.notes, + "analyses_agency": excluded.analyses_agency, + "uncertainty": excluded.uncertainty, + "volume": excluded.volume, + "volume_unit": excluded.volume_unit, }, ) session.execute(stmt) @@ -147,8 +165,9 @@ def _transfer_hook(self, session: Session) -> None: def _row_to_dict(self, row) -> Optional[dict[str, Any]]: """Convert a DataFrame row to a dict for upsert.""" - sample_pt_id = self._uuid_val(row.SamplePtID) - if sample_pt_id is None: + # Get legacy UUID FK + legacy_sample_pt_id = self._uuid_val(row.SamplePtID) + if legacy_sample_pt_id is None: self._capture_error( getattr(row, "SamplePtID", None), f"Invalid SamplePtID: {getattr(row, 'SamplePtID', None)}", @@ -156,16 +175,18 @@ def _row_to_dict(self, row) -> Optional[dict[str, Any]]: ) return None - if sample_pt_id not in self._sample_pt_ids: + # Look up Integer FK from cache + chemistry_sample_info_id = self._sample_info_cache.get(legacy_sample_pt_id) + if chemistry_sample_info_id is None: self._capture_error( - sample_pt_id, - f"ChemistrySampleInfo not found for SamplePtID: {sample_pt_id}", + legacy_sample_pt_id, + f"ChemistrySampleInfo not found for SamplePtID: {legacy_sample_pt_id}", "SamplePtID", ) return None - global_id = self._uuid_val(getattr(row, "GlobalID", None)) - if global_id is None: + nma_global_id = self._uuid_val(getattr(row, "GlobalID", None)) + if nma_global_id is None: self._capture_error( getattr(row, "GlobalID", None), f"Invalid GlobalID: {getattr(row, 'GlobalID', None)}", @@ -174,27 +195,31 @@ def _row_to_dict(self, row) -> Optional[dict[str, Any]]: return None return { - "GlobalID": global_id, - "SamplePtID": sample_pt_id, - "SamplePointID": self._safe_str(row, "SamplePointID"), - "Analyte": self._safe_str(row, "Analyte"), - "SampleValue": self._safe_float(row, "SampleValue"), - "Units": self._safe_str(row, "Units"), - "Symbol": self._safe_str(row, "Symbol"), - "AnalysisMethod": self._safe_str(row, "AnalysisMethod"), - "AnalysisDate": self._parse_date(row, "AnalysisDate"), - "Notes": self._safe_str(row, "Notes"), - "AnalysesAgency": self._safe_str(row, "AnalysesAgency"), - "Uncertainty": self._safe_float(row, "Uncertainty"), - "Volume": self._safe_int(row, "Volume"), - "VolumeUnit": self._safe_str(row, "VolumeUnit"), + # Legacy UUID PK -> nma_global_id (unique audit column) + "nma_GlobalID": nma_global_id, + # New Integer FK to ChemistrySampleInfo + "chemistry_sample_info_id": chemistry_sample_info_id, + # Legacy UUID FK for audit + "nma_chemistry_sample_info_uuid": legacy_sample_pt_id, + # Data columns + "analyte": self._safe_str(row, "Analyte"), + "sample_value": self._safe_float(row, "SampleValue"), + "units": self._safe_str(row, "Units"), + "symbol": self._safe_str(row, "Symbol"), + "analysis_method": self._safe_str(row, "AnalysisMethod"), + "analysis_date": self._parse_date(row, "AnalysisDate"), + "notes": self._safe_str(row, "Notes"), + "analyses_agency": self._safe_str(row, "AnalysesAgency"), + "uncertainty": self._safe_float(row, "Uncertainty"), + "volume": self._safe_int(row, "Volume"), + "volume_unit": self._safe_str(row, "VolumeUnit"), } def _dedupe_rows(self, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: """Dedupe rows by unique key to avoid ON CONFLICT loops. Later rows win.""" deduped = {} for row in rows: - key = row.get("GlobalID") + key = row.get("nma_GlobalID") if key is None: continue deduped[key] = row diff --git a/transfers/radionuclides.py b/transfers/radionuclides.py index 70575e034..589dbec88 100644 --- a/transfers/radionuclides.py +++ b/transfers/radionuclides.py @@ -13,6 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +""" +Transfer Radionuclides data from NM_Aquifer to NMA_Radionuclides. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement, generated by DB) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id +- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit +- nma_sample_point_id: Legacy SamplePointID string +- nma_object_id: Legacy OBJECTID, UNIQUE +- nma_wclab_id: Legacy WCLab_ID +""" from __future__ import annotations @@ -34,6 +46,8 @@ class RadionuclidesTransferer(Transferer): """ Transfer for the legacy Radionuclides table. + + Uses Integer FK to ChemistrySampleInfo via chemistry_sample_info_id. """ source_table = "Radionuclides" @@ -41,21 +55,28 @@ class RadionuclidesTransferer(Transferer): def __init__(self, *args, batch_size: int = 1000, **kwargs): super().__init__(*args, **kwargs) self.batch_size = batch_size - self._sample_pt_ids: set[UUID] = set() - self._thing_id_by_sample_pt_id: dict[UUID, int] = {} + # Cache: legacy UUID -> (Integer id, thing_id) + self._sample_info_cache: dict[UUID, tuple[int, int]] = {} self._build_sample_info_cache() def _build_sample_info_cache(self) -> None: + """Build cache of nma_sample_pt_id -> (id, thing_id) for FK lookups.""" with session_ctx() as session: - sample_infos = session.query( - NMA_Chemistry_SampleInfo.sample_pt_id, NMA_Chemistry_SampleInfo.thing_id - ).all() - self._sample_pt_ids = {sample_pt_id for sample_pt_id, _ in sample_infos} - self._thing_id_by_sample_pt_id = { - sample_pt_id: thing_id for sample_pt_id, thing_id in sample_infos + sample_infos = ( + session.query( + NMA_Chemistry_SampleInfo.nma_sample_pt_id, + NMA_Chemistry_SampleInfo.id, + NMA_Chemistry_SampleInfo.thing_id, + ) + .filter(NMA_Chemistry_SampleInfo.nma_sample_pt_id.isnot(None)) + .all() + ) + self._sample_info_cache = { + nma_sample_pt_id: (csi_id, thing_id) + for nma_sample_pt_id, csi_id, thing_id in sample_infos } logger.info( - f"Built ChemistrySampleInfo cache with {len(self._sample_pt_ids)} entries" + f"Built ChemistrySampleInfo cache with {len(self._sample_info_cache)} entries" ) def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -64,7 +85,7 @@ def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: return input_df, cleaned_df def _filter_to_valid_sample_infos(self, df: pd.DataFrame) -> pd.DataFrame: - valid_sample_pt_ids = self._sample_pt_ids + valid_sample_pt_ids = set(self._sample_info_cache.keys()) mask = df["SamplePtID"].apply( lambda value: self._uuid_val(value) in valid_sample_pt_ids ) @@ -89,25 +110,31 @@ def _transfer_hook(self, session: Session) -> None: row_dict = self._row_dict(row) if row_dict is None: continue - if row_dict.get("GlobalID") is None: + if row_dict.get("nma_GlobalID") is None: skipped_global_id += 1 logger.warning( - "Skipping Radionuclides SamplePtID=%s - GlobalID missing or invalid", - row_dict.get("SamplePtID"), + "Skipping Radionuclides nma_SamplePtID=%s - nma_GlobalID missing or invalid", + row_dict.get("nma_SamplePtID"), ) continue if row_dict.get("thing_id") is None: skipped_thing_id += 1 logger.warning( - "Skipping Radionuclides SamplePtID=%s - Thing not found", - row_dict.get("SamplePtID"), + "Skipping Radionuclides nma_SamplePtID=%s - Thing not found", + row_dict.get("nma_SamplePtID"), + ) + continue + if row_dict.get("chemistry_sample_info_id") is None: + logger.warning( + "Skipping Radionuclides nma_SamplePtID=%s - chemistry_sample_info_id not found", + row_dict.get("nma_SamplePtID"), ) continue row_dicts.append(row_dict) if skipped_global_id > 0: logger.warning( - "Skipped %s Radionuclides records without valid GlobalID", + "Skipped %s Radionuclides records without valid nma_GlobalID", skipped_global_id, ) if skipped_thing_id > 0: @@ -116,7 +143,7 @@ def _transfer_hook(self, session: Session) -> None: skipped_thing_id, ) - rows = self._dedupe_rows(row_dicts, key="GlobalID") + rows = self._dedupe_rows(row_dicts, key="nma_GlobalID") insert_stmt = insert(NMA_Radionuclides) excluded = insert_stmt.excluded @@ -125,12 +152,14 @@ def _transfer_hook(self, session: Session) -> None: logger.info( f"Upserting batch {i}-{i+len(chunk)-1} ({len(chunk)} rows) into Radionuclides" ) + # Upsert on nma_GlobalID (legacy UUID PK, now UNIQUE) stmt = insert_stmt.values(chunk).on_conflict_do_update( - index_elements=["GlobalID"], + index_elements=["nma_GlobalID"], set_={ "thing_id": excluded.thing_id, - "SamplePtID": excluded.SamplePtID, - "SamplePointID": excluded.SamplePointID, + "chemistry_sample_info_id": excluded.chemistry_sample_info_id, + "nma_SamplePtID": excluded.nma_SamplePtID, + "nma_SamplePointID": excluded.nma_SamplePointID, "Analyte": excluded.Analyte, "Symbol": excluded.Symbol, "SampleValue": excluded.SampleValue, @@ -141,9 +170,9 @@ def _transfer_hook(self, session: Session) -> None: "Notes": excluded.Notes, "Volume": excluded.Volume, "VolumeUnit": excluded.VolumeUnit, - "OBJECTID": excluded.OBJECTID, + "nma_OBJECTID": excluded.nma_OBJECTID, "AnalysesAgency": excluded.AnalysesAgency, - "WCLab_ID": excluded.WCLab_ID, + "nma_WCLab_ID": excluded.nma_WCLab_ID, }, ) session.execute(stmt) @@ -181,8 +210,9 @@ def int_val(key: str) -> Optional[int]: if isinstance(analysis_date, datetime): analysis_date = analysis_date.replace(tzinfo=None) - sample_pt_id = self._uuid_val(val("SamplePtID")) - if sample_pt_id is None: + # Get legacy UUID FK + legacy_sample_pt_id = self._uuid_val(val("SamplePtID")) + if legacy_sample_pt_id is None: self._capture_error( val("SamplePtID"), f"Invalid SamplePtID: {val('SamplePtID')}", @@ -190,13 +220,25 @@ def int_val(key: str) -> Optional[int]: ) return None - global_id = self._uuid_val(val("GlobalID")) - thing_id = self._thing_id_by_sample_pt_id.get(sample_pt_id) + # Look up Integer FK and thing_id from cache + cache_entry = self._sample_info_cache.get(legacy_sample_pt_id) + chemistry_sample_info_id = cache_entry[0] if cache_entry else None + thing_id = cache_entry[1] if cache_entry else None + + nma_global_id = self._uuid_val(val("GlobalID")) return { + # Legacy UUID PK -> nma_global_id (unique audit column) + "nma_GlobalID": nma_global_id, + # FKs "thing_id": thing_id, - "SamplePtID": sample_pt_id, - "SamplePointID": val("SamplePointID"), + "chemistry_sample_info_id": chemistry_sample_info_id, + # Legacy ID columns (renamed with nma_ prefix) + "nma_SamplePtID": legacy_sample_pt_id, + "nma_SamplePointID": val("SamplePointID"), + "nma_OBJECTID": val("OBJECTID"), + "nma_WCLab_ID": val("WCLab_ID"), + # Data columns "Analyte": val("Analyte"), "Symbol": val("Symbol"), "SampleValue": float_val("SampleValue"), @@ -207,10 +249,7 @@ def int_val(key: str) -> Optional[int]: "Notes": val("Notes"), "Volume": int_val("Volume"), "VolumeUnit": val("VolumeUnit"), - "OBJECTID": val("OBJECTID"), - "GlobalID": global_id, "AnalysesAgency": val("AnalysesAgency"), - "WCLab_ID": val("WCLab_ID"), } def _uuid_val(self, value: Any) -> Optional[UUID]: @@ -229,26 +268,8 @@ def _dedupe_rows( self, rows: list[dict[str, Any]], key: str ) -> list[dict[str, Any]]: """ - Deduplicate rows within a batch by the given key to avoid ON CONFLICT loops - when inserting into the database. - - For any given ``key`` value, only a single row is kept in the returned list. - If multiple rows share the same ``key`` value, the *last* occurrence in - ``rows`` overwrites earlier ones (i.e. "later rows win"), because the - internal mapping is updated on each encounter of that key. - - This behavior is appropriate when: - * The input batch is ordered such that later rows represent the most - recent or authoritative data for a given key, and - * Only one row per key should be written in a single batch to prevent - repeated ON CONFLICT handling for the same key. - - Callers should be aware that this can silently drop earlier rows with the - same key. If preserving all conflicting rows or applying a custom conflict - resolution strategy is important, the caller should: - * Pre-process and consolidate rows before passing them to this method, or - * Implement a different deduplication/merge strategy tailored to their - needs. + Deduplicate rows within a batch by the given key to avoid ON CONFLICT loops. + Later rows win. """ deduped = {} for row in rows: diff --git a/transfers/soil_rock_results.py b/transfers/soil_rock_results.py index 35fa48663..1aae4e3ad 100644 --- a/transfers/soil_rock_results.py +++ b/transfers/soil_rock_results.py @@ -13,6 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== +""" +Transfer Soil_Rock_Results from NM_Aquifer to NMA_Soil_Rock_Results. + +Already has Integer PK. Updated for legacy column rename: +- point_id -> nma_point_id +""" from __future__ import annotations @@ -71,12 +77,15 @@ def _transfer_hook(self, session: Session) -> None: def _row_dict(self, row: dict[str, Any]) -> dict[str, Any]: point_id = row.get("Point_ID") return { - "point_id": point_id, + # Legacy ID column (use Python attribute name for bulk_insert_mappings) + "nma_point_id": point_id, + # Data columns (use Python attribute names, not database column names) "sample_type": row.get("Sample Type"), "date_sampled": row.get("Date Sampled"), "d13c": self._float_val(row.get("d13C")), "d18o": self._float_val(row.get("d18O")), "sampled_by": row.get("Sampled by"), + # FK to Thing "thing_id": self._thing_id_cache.get(point_id), } diff --git a/transfers/stratigraphy_legacy.py b/transfers/stratigraphy_legacy.py index 326f6434a..82bf8a3a5 100644 --- a/transfers/stratigraphy_legacy.py +++ b/transfers/stratigraphy_legacy.py @@ -1,4 +1,12 @@ -"""Transfer Stratigraphy.csv into the NMA_Stratigraphy legacy table.""" +"""Transfer Stratigraphy.csv into the NMA_Stratigraphy legacy table. + +Updated for Integer PK schema: +- id: Integer PK (autoincrement, generated by DB) +- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit +- nma_well_id: Legacy WellID UUID +- nma_point_id: Legacy PointID string +- nma_object_id: Legacy OBJECTID, UNIQUE +""" from __future__ import annotations @@ -63,11 +71,12 @@ def _transfer_hook(self, session: Session) -> None: # type: ignore[override] start + len(chunk) - 1, len(chunk), ) + # Upsert on nma_GlobalID (legacy UUID PK, now UNIQUE) stmt = insert_stmt.values(chunk).on_conflict_do_update( - index_elements=["GlobalID"], + index_elements=["nma_GlobalID"], set_={ - "WellID": excluded.WellID, - "PointID": excluded.PointID, + "nma_WellID": excluded.nma_WellID, + "nma_PointID": excluded.nma_PointID, "thing_id": excluded.thing_id, "StratTop": excluded.StratTop, "StratBottom": excluded.StratBottom, @@ -77,7 +86,7 @@ def _transfer_hook(self, session: Session) -> None: # type: ignore[override] "ContributingUnit": excluded.ContributingUnit, "StratSource": excluded.StratSource, "StratNotes": excluded.StratNotes, - "OBJECTID": excluded.OBJECTID, + "nma_OBJECTID": excluded.nma_OBJECTID, }, ) session.execute(stmt) @@ -104,16 +113,21 @@ def _row_dict(self, row: pd.Series) -> Dict[str, Any] | None: self._capture_error(point_id, "No Thing found for PointID", "thing_id") return None - global_id = self._uuid_value(getattr(row, "GlobalID", None)) - if global_id is None: + nma_global_id = self._uuid_value(getattr(row, "GlobalID", None)) + if nma_global_id is None: self._capture_error(point_id, "Invalid GlobalID", "GlobalID") return None return { - "GlobalID": global_id, - "WellID": self._uuid_value(getattr(row, "WellID", None)), - "PointID": point_id, + # Legacy UUID PK -> nma_global_id (unique audit column) + "nma_GlobalID": nma_global_id, + # Legacy ID columns (renamed with nma_ prefix) + "nma_WellID": self._uuid_value(getattr(row, "WellID", None)), + "nma_PointID": point_id, + "nma_OBJECTID": self._int_value(getattr(row, "OBJECTID", None)), + # FK to Thing "thing_id": thing_id, + # Data columns "StratTop": self._float_value(getattr(row, "StratTop", None)), "StratBottom": self._float_value(getattr(row, "StratBottom", None)), "UnitIdentifier": self._string_value(getattr(row, "UnitIdentifier", None)), @@ -126,7 +140,6 @@ def _row_dict(self, row: pd.Series) -> Dict[str, Any] | None: ), "StratSource": self._string_value(getattr(row, "StratSource", None)), "StratNotes": self._string_value(getattr(row, "StratNotes", None)), - "OBJECTID": self._int_value(getattr(row, "OBJECTID", None)), } def _uuid_value(self, value: Any) -> UUID | None: diff --git a/transfers/transfer.py b/transfers/transfer.py index 2d33176b2..caaa97945 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -62,6 +62,12 @@ WellScreenTransferer, cleanup_locations, ) +from transfers.thing_transfer import ( + transfer_springs, + transfer_perennial_stream, + transfer_ephemeral_stream, + transfer_met, +) from transfers.minor_trace_chemistry_transfer import MinorTraceChemistryTransferer from transfers.asset_transfer import AssetTransferer @@ -115,6 +121,11 @@ class TransferOptions: transfer_minor_trace_chemistry: bool transfer_nma_stratigraphy: bool transfer_associated_data: bool + # Non-well location types + transfer_springs: bool + transfer_perennial_streams: bool + transfer_ephemeral_streams: bool + transfer_met_stations: bool def load_transfer_options() -> TransferOptions: @@ -153,6 +164,11 @@ def load_transfer_options() -> TransferOptions: ), transfer_nma_stratigraphy=get_bool_env("TRANSFER_NMA_STRATIGRAPHY", True), transfer_associated_data=get_bool_env("TRANSFER_ASSOCIATED_DATA", True), + # Non-well location types + transfer_springs=get_bool_env("TRANSFER_SPRINGS", True), + transfer_perennial_streams=get_bool_env("TRANSFER_PERENNIAL_STREAMS", True), + transfer_ephemeral_streams=get_bool_env("TRANSFER_EPHEMERAL_STREAMS", True), + transfer_met_stations=get_bool_env("TRANSFER_MET_STATIONS", True), ) @@ -322,8 +338,40 @@ def transfer_all(metrics, limit=100, profile_waterlevels: bool = True): # Get transfer flags transfer_options = load_transfer_options() - transfer_options.transfer_pressure = False - transfer_options.transfer_acoustic = False + + # ========================================================================= + # PHASE 1.5: Non-well location types (parallel, after wells, before other transfers) + # These create Things and Locations that chemistry/other transfers depend on. + # ========================================================================= + non_well_tasks = [] + if transfer_options.transfer_springs: + non_well_tasks.append(("Springs", transfer_springs)) + if transfer_options.transfer_perennial_streams: + non_well_tasks.append(("PerennialStreams", transfer_perennial_stream)) + if transfer_options.transfer_ephemeral_streams: + non_well_tasks.append(("EphemeralStreams", transfer_ephemeral_stream)) + if transfer_options.transfer_met_stations: + non_well_tasks.append(("MetStations", transfer_met)) + + if non_well_tasks: + message("PHASE 1.5: NON-WELL LOCATION TYPES (PARALLEL)") + with ThreadPoolExecutor(max_workers=len(non_well_tasks)) as executor: + futures = { + executor.submit( + _execute_session_transfer_with_timing, name, func, limit + ): name + for name, func in non_well_tasks + } + + for future in as_completed(futures): + name = futures[future] + try: + result_name, result, elapsed = future.result() + logger.info( + f"Non-well transfer {result_name} completed in {elapsed:.2f}s" + ) + except Exception as e: + logger.critical(f"Non-well transfer {name} failed: {e}") use_parallel = get_bool_env("TRANSFER_PARALLEL", True) if use_parallel: