From 121dabca138be54e4f786d0da618222c1bef08a0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Sep 2025 12:06:10 -0600 Subject: [PATCH 1/5] WIP: adding nma pk wellscreens to well screens --- db/thing.py | 1 + transfers/well_transfer.py | 1 + 2 files changed, 2 insertions(+) diff --git a/db/thing.py b/db/thing.py index 0f31e0b45..b1071127b 100644 --- a/db/thing.py +++ b/db/thing.py @@ -112,6 +112,7 @@ class WellScreen(Base, AutoBaseMixin, ReleaseMixin): screen_description = Column( String(1000), nullable=True, info={"unit": "description of the screen"} ) + nma_pk_wellscreens = Column(String(100), nullable=True) # Define a relationship to well if needed thing = relationship("Thing") diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index a1ad901a4..b74eab159 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -144,6 +144,7 @@ def transfer_wellscreens(session, limit=None): } try: model = CreateWellScreen.model_validate(well_screen_data) + setattr(model, "nma_pk_wellscreens", row.GlobalID) model_adder(session, WellScreen, model) except ValidationError as e: print(f"Validation error for row {i} with PointID {row.PointID}: {e}") From 487234b3d660f397ad089a8f472ef07ea1876cee Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Sep 2025 14:39:32 -0600 Subject: [PATCH 2/5] refactor: use Pydantic to validate incoming data --- transfers/contact_transfer.py | 167 ++++++++++++++++++++++++---------- 1 file changed, 121 insertions(+), 46 deletions(-) diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index 5e6eff8e7..28c025b97 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -18,6 +18,8 @@ from transfers.util import read_csv, filter_to_valid_point_ids from db import Thing, Contact, ThingContactAssociation, Email, Phone, Address +from schemas.contact import CreateContact + def extract_owner_role(comment): # if comment is None: @@ -32,6 +34,14 @@ def extract_owner_role(comment): return "Owner" +""" +Developer's notes + +Use Pydantic to perform model validations since all restrictions will +be built into the models +""" + + def transfer_contacts(session): odf = read_csv("ownersdata.csv") @@ -41,96 +51,161 @@ def transfer_contacts(session): for i, row in odf.iterrows(): thing = session.query(Thing).where(Thing.name == row.PointID).first() if thing is None: - print(f"Thing with PointID {row.PointID} not foaund. Skipping owner.") + print(f"Thing with PointID {row.PointID} not found. Skipping owner.") continue # TODO: extract role from OwnerComment # role = extract_owner_role(row.OwnerComment) role = "Owner" + release_status = "private" # TODO: put in guards for null values - # name OR organization must be defined, otherwise skip - if not (row.FirstName or row.LastName) and not row.Company: - print( - f"Skipping first contact for PointID {row.PointID} due to missing name and organization." - ) - else: - print(f"Transferring first contact for PointID {row.PointID}") - contact1 = Contact( - name=f"{row.FirstName} {row.LastName}", - role=role, - contact_type="Primary", - organization=row.Company, # assumes organization applies to both contacts - nma_pk_owners=row.OwnerKey, - ) + try: + + if row.FirstName is None and row.LastName is None: + name = None + elif row.FirstName is not None and row.LastName is None: + name = row.FirstName + elif row.FirstName is None and row.LastName is not None: + name = row.LastName + else: + name = f"{row.FirstName} {row.LastName}" + + first_contact_data = { + "thing_id": thing.id, + "release_status": release_status, + "name": name, + "role": role, + "contact_type": "Primary", + "organization": row.Company, + "nma_pk_owners": row.OwnerKey, + } + + CreateContact.model_validate(first_contact_data) + + first_contact_data.pop("thing_id") + first_contact = Contact(**first_contact_data) + assoc = ThingContactAssociation() assoc.thing = thing - assoc.contact = contact1 - session.add(assoc) - session.add(contact1) + assoc.contact = first_contact if row.Email: - contact1.emails.append(Email(email=row.Email, email_type="Primary")) + first_contact.emails.append( + Email( + email=row.Email, + email_type="Primary", + release_status=release_status, + ) + ) if row.Phone: - contact1.phones.append( - Phone(phone_number=row.Phone, phone_type="Primary") + first_contact.phones.append( + Phone( + phone_number=row.Phone, + phone_type="Primary", + release_status=release_status, + ) ) if row.CellPhone: - contact1.phones.append( - Phone(phone_number=row.CellPhone, phone_type="Mobile") + first_contact.phones.append( + Phone( + phone_number=row.CellPhone, + phone_type="Mobile", + release_status=release_status, + ) ) if row.MailingAddress: - contact1.addresses.append( + first_contact.addresses.append( Address( address_line_1=row.MailingAddress, city=row.MailCity, state=row.MailState, postal_code=row.MailZipCode, address_type="Mailing", + release_status=release_status, ) ) - contact1.addresses.append( + first_contact.addresses.append( Address( address_line_1=row.PhysicalAddress, city=row.PhysicalCity, state=row.PhysicalState, postal_code=row.PhysicalZipCode, address_type="Physical", + release_status=release_status, ) ) - # TODO: put in guards for null values - if not (row.SecondFirstName or row.SecondLastName) and not row.Company: + session.add(assoc) + session.add(first_contact) + session.commit() + + except Exception as e: print( - f"Skipping second contact for PointID {row.PointID} due to missing name and organization." - ) - else: - print(f"Transferring second contact for PointID {row.PointID}") - contact2 = Contact( - name=f"{row.SecondFirstName} {row.SecondLastName}", - role="Owner", - contact_type="Secondary", - organization=row.Company, # Assumes organization applies to both contacts - nma_pk_owners=row.OwnerKey, + f"Skipping first contact for PointID {row.PointID} due to validation error: {e}" ) + from pprint import pprint + + pprint(e) + session.rollback() + + try: + if row.SecondFirstName is None and row.SecondLastName is None: + name = None + elif row.SecondFirstName is not None and row.SecondLastName is None: + name = row.SecondFirstName + elif row.SecondFirstName is None and row.SecondLastName is not None: + name = row.SecondLastName + else: + name = f"{row.SecondFirstName} {row.SecondLastName}" + + second_contact_data = { + "thing_id": thing.id, + "release_status": release_status, + "name": name, + "role": "Owner", + "contact_type": "Secondary", + "organization": row.Company, + "nma_pk_owners": row.OwnerKey, + } + + CreateContact.model_validate(second_contact_data) + + second_contact_data.pop("thing_id") + second_contact = Contact(**second_contact_data) + + assoc = ThingContactAssociation() + assoc.thing = thing + assoc.contact = second_contact + if row.SecondCtctEmail: - contact2.emails.append( - Email(email=row.SecondCtctEmail, email_type="Primary") + second_contact.emails.append( + Email( + email=row.SecondCtctEmail, + email_type="Primary", + release_status=release_status, + ) ) + if row.SecondCtctPhone: - contact2.phones.append( - Phone(phone_number=row.SecondCtctPhone, phone_type="Primary") + second_contact.phones.append( + Phone( + phone_number=row.SecondCtctPhone, + phone_type="Primary", + release_status=release_status, + ) ) - assoc = ThingContactAssociation() - assoc.thing = thing - assoc.contact = contact2 session.add(assoc) - session.add(contact2) + session.add(second_contact) - session.commit() + except Exception as e: + print( + f"Skipping second contact for PointID {row.PointID} due to validation error: {e}" + ) + session.rollback() # ============= EOF ============================================= From 6073590c96cc059b65466c1224ae3b4512638f06 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Sep 2025 15:30:08 -0600 Subject: [PATCH 3/5] style: use pydantic to validate well screens --- transfers/well_transfer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index b74eab159..13515ea42 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -21,7 +21,6 @@ from sqlalchemy import select from db import LocationThingAssociation, Thing, WellScreen, Location -from services.crud_helper import model_adder from schemas.thing import CreateWellScreen from services.lexicon_helper import add_lexicon_term from services.thing_helper import add_thing @@ -141,11 +140,14 @@ def transfer_wellscreens(session, limit=None): # "screen_type": row.ScreenType, "screen_description": row.ScreenDescription, "release_status": "draft", + "nma_pk_wellscreens": row.GlobalID, } try: - model = CreateWellScreen.model_validate(well_screen_data) - setattr(model, "nma_pk_wellscreens", row.GlobalID) - model_adder(session, WellScreen, model) + # TODO: add validation logic here to ensure no overlapping screens for the same well + CreateWellScreen.model_validate(well_screen_data) + well_screen = WellScreen(**well_screen_data) + session.add(well_screen) + session.commit() except ValidationError as e: print(f"Validation error for row {i} with PointID {row.PointID}: {e}") continue From a352adfbbc823a141a9c8ed4147779e8725d7554 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Sep 2025 16:16:23 -0600 Subject: [PATCH 4/5] refactor: infer nullability from Mapped annotation --- db/contact.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/db/contact.py b/db/contact.py index 9a1d06f2e..b60a3167e 100644 --- a/db/contact.py +++ b/db/contact.py @@ -24,10 +24,10 @@ class ThingContactAssociation(Base, AutoBaseMixin): thing_id: Mapped[int] = mapped_column( - Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + Integer, ForeignKey("thing.id", ondelete="CASCADE") ) contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id", ondelete="CASCADE"), nullable=False + Integer, ForeignKey("contact.id", ondelete="CASCADE") ) contact: Mapped[List["Contact"]] = relationship("Contact") @@ -74,10 +74,10 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): class Phone(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id", ondelete="CASCADE"), nullable=False + Integer, ForeignKey("contact.id", ondelete="CASCADE") ) - phone_number: Mapped[str] = mapped_column(String(20), nullable=False) - phone_type: Mapped[str] = lexicon_term(nullable=False) + phone_number: Mapped[str] = mapped_column(String(20)) + phone_type: Mapped[str] = lexicon_term() contact: Mapped["Contact"] = relationship( "Contact", back_populates="phones", passive_deletes=True @@ -87,10 +87,10 @@ class Phone(Base, AutoBaseMixin, ReleaseMixin): class Email(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id", ondelete="CASCADE"), nullable=False + Integer, ForeignKey("contact.id", ondelete="CASCADE") ) - email: Mapped[str] = mapped_column(String(100), nullable=False) - email_type: Mapped[str] = lexicon_term(nullable=False) + email: Mapped[str] = mapped_column(String(100)) + email_type: Mapped[str] = lexicon_term() contact: Mapped["Contact"] = relationship( "Contact", back_populates="emails", passive_deletes=True @@ -101,15 +101,15 @@ class Email(Base, AutoBaseMixin, ReleaseMixin): class Address(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id", ondelete="CASCADE"), nullable=False + Integer, ForeignKey("contact.id", ondelete="CASCADE") ) - address_line_1: Mapped[str] = mapped_column(String(255), nullable=False) - address_line_2: Mapped[str | None] = mapped_column(String(255), nullable=True) - city: Mapped[str] = mapped_column(String(100), nullable=False) - state: Mapped[str] = mapped_column(String(50), nullable=False) - postal_code: Mapped[str] = mapped_column(String(20), nullable=False) - country: Mapped[str] = lexicon_term(nullable=False, default="United States") - address_type: Mapped[str] = lexicon_term(nullable=False) + address_line_1: Mapped[str] = mapped_column(String(255)) + address_line_2: Mapped[str | None] = mapped_column(String(255)) + city: Mapped[str] = mapped_column(String(100)) + state: Mapped[str] = mapped_column(String(50)) + postal_code: Mapped[str] = mapped_column(String(20)) + country: Mapped[str] = lexicon_term(default="United States") + address_type: Mapped[str] = lexicon_term() contact: Mapped["Contact"] = relationship( "Contact", back_populates="addresses", passive_deletes=True From 5dcb5cb416cd1473b12c668c0958862c164a2e1b Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 4 Sep 2025 17:02:56 -0600 Subject: [PATCH 5/5] refactor: redo annotations for mapped columns --- db/contact.py | 42 +++++++++++++++++++++--------------------- db/thing.py | 25 +++++++++++++------------ 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/db/contact.py b/db/contact.py index b60a3167e..bbff76178 100644 --- a/db/contact.py +++ b/db/contact.py @@ -24,10 +24,10 @@ class ThingContactAssociation(Base, AutoBaseMixin): thing_id: Mapped[int] = mapped_column( - Integer, ForeignKey("thing.id", ondelete="CASCADE") + ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id", ondelete="CASCADE") + ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) contact: Mapped[List["Contact"]] = relationship("Contact") @@ -35,11 +35,11 @@ class ThingContactAssociation(Base, AutoBaseMixin): class Contact(Base, AutoBaseMixin, ReleaseMixin): - name: Mapped[str | None] = mapped_column(String(100)) - organization: Mapped[str | None] = mapped_column(String(100)) - role: Mapped[str] = lexicon_term() - contact_type: Mapped[str] = lexicon_term() - nma_pk_owners: Mapped[str | None] = mapped_column(String(100)) + name: Mapped[str] = mapped_column(String(100), nullable=True) + organization: Mapped[str] = mapped_column(String(100), nullable=True) + role: Mapped[str] = lexicon_term(nullable=False) + contact_type: Mapped[str] = lexicon_term(nullable=False) + nma_pk_owners: Mapped[str] = mapped_column(String(100), nullable=True) phones: Mapped[List["Phone"]] = relationship( "Phone", back_populates="contact", passive_deletes=True @@ -74,10 +74,10 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin): class Phone(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id", ondelete="CASCADE") + ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) - phone_number: Mapped[str] = mapped_column(String(20)) - phone_type: Mapped[str] = lexicon_term() + phone_number: Mapped[str] = mapped_column(String(20), nullable=False) + phone_type: Mapped[str] = lexicon_term(nullable=False) contact: Mapped["Contact"] = relationship( "Contact", back_populates="phones", passive_deletes=True @@ -87,10 +87,10 @@ class Phone(Base, AutoBaseMixin, ReleaseMixin): class Email(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id", ondelete="CASCADE") + Integer, ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) - email: Mapped[str] = mapped_column(String(100)) - email_type: Mapped[str] = lexicon_term() + email: Mapped[str] = mapped_column(String(100), nullable=False) + email_type: Mapped[str] = lexicon_term(nullable=False) contact: Mapped["Contact"] = relationship( "Contact", back_populates="emails", passive_deletes=True @@ -101,15 +101,15 @@ class Email(Base, AutoBaseMixin, ReleaseMixin): class Address(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id", ondelete="CASCADE") + ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) - address_line_1: Mapped[str] = mapped_column(String(255)) - address_line_2: Mapped[str | None] = mapped_column(String(255)) - city: Mapped[str] = mapped_column(String(100)) - state: Mapped[str] = mapped_column(String(50)) - postal_code: Mapped[str] = mapped_column(String(20)) - country: Mapped[str] = lexicon_term(default="United States") - address_type: Mapped[str] = lexicon_term() + address_line_1: Mapped[str] = mapped_column(String(255), nullable=False) + address_line_2: Mapped[str | None] = mapped_column(String(255), nullable=True) + city: Mapped[str] = mapped_column(String(100), nullable=False) + state: Mapped[str] = mapped_column(String(50), nullable=False) + postal_code: Mapped[str] = mapped_column(String(20), nullable=False) + country: Mapped[str] = lexicon_term(default="United States", nullable=False) + address_type: Mapped[str] = lexicon_term(nullable=False) contact: Mapped["Contact"] = relationship( "Contact", back_populates="addresses", passive_deletes=True diff --git a/db/thing.py b/db/thing.py index b1071127b..cbeed6c59 100644 --- a/db/thing.py +++ b/db/thing.py @@ -15,7 +15,7 @@ # =============================================================================== from sqlalchemy import Integer, ForeignKey, String, Column, Float from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.orm import relationship, mapped_column +from sqlalchemy.orm import relationship, mapped_column, Mapped from sqlalchemy_utils import TSVectorType from db import lexicon_term @@ -98,23 +98,24 @@ class ThingIdLink(Base, AutoBaseMixin, ReleaseMixin): class WellScreen(Base, AutoBaseMixin, ReleaseMixin): - thing_id = Column( - Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + thing_id: Mapped[int] = mapped_column( + ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) - screen_depth_top = Column( - Float, nullable=False, info={"unit": "feet below ground surface"} + screen_depth_top: Mapped[float] = mapped_column( + info={"unit": "feet below ground surface"}, nullable=True ) - screen_depth_bottom = Column( - Float, nullable=False, info={"unit": "feet below ground surface"} + screen_depth_bottom: Mapped[float] = mapped_column( + info={"unit": "feet below ground surface"}, nullable=True ) - screen_type = lexicon_term() # e.g., "PVC", "Steel", etc. + screen_type: Mapped[str] = lexicon_term(nullable=True) # e.g., "PVC", "Steel", etc. - screen_description = Column( - String(1000), nullable=True, info={"unit": "description of the screen"} + screen_description: Mapped[str] = mapped_column( + String(1000), info={"unit": "description of the screen"}, nullable=True ) - nma_pk_wellscreens = Column(String(100), nullable=True) + nma_pk_wellscreens: Mapped[str] = mapped_column(String(100), nullable=True) + # Define a relationship to well if needed - thing = relationship("Thing") + thing: Mapped["Thing"] = relationship("Thing") # TODO: this could be the model used to handle AMP monitoring