From dc18c2971d3e431ebd1da54b08bd4310c65fc70d Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Fri, 17 Jul 2026 12:45:44 -0700 Subject: [PATCH 1/2] refactor(system): inline the PathSegment type alias PathSegment named the StructSegment | ArraySegment subset, while FieldSegment names the full StructSegment | ArraySegment | MapSegment set. Side by side, PathSegment read as a misleading middle name for something that excludes MapSegment. It was used at two sites, so inline the union and drop the alias. Signed-off-by: Seth Fitzsimmons --- .../src/overture/schema/system/field_path.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/overture-schema-system/src/overture/schema/system/field_path.py b/packages/overture-schema-system/src/overture/schema/system/field_path.py index 7b60fea56..df06232fe 100644 --- a/packages/overture-schema-system/src/overture/schema/system/field_path.py +++ b/packages/overture-schema-system/src/overture/schema/system/field_path.py @@ -38,7 +38,6 @@ "MapPath", "MapProjection", "MapSegment", - "PathSegment", "ScalarPath", "StructSegment", "coerce", @@ -88,9 +87,6 @@ class MapSegment: projection: MapProjection -PathSegment: TypeAlias = StructSegment | ArraySegment - - @dataclass(frozen=True, slots=True) class ScalarPath: """Locate a non-iterated value in a row.""" @@ -116,7 +112,7 @@ class ArrayPath: Invariant: `segments` contains at least one `ArraySegment`. """ - segments: tuple[PathSegment, ...] + segments: tuple[StructSegment | ArraySegment, ...] def __post_init__(self) -> None: if not any(isinstance(s, ArraySegment) for s in self.segments): @@ -334,14 +330,14 @@ def __str__(self) -> str: # The element type of any `FieldPath.segments`, across all three variants. -# Broader than `PathSegment` (array/scalar paths only): a `MapPath` adds a -# trailing `MapSegment`. Consumers that walk an arbitrary `FieldPath`'s -# segments -- rather than a statically known `ArrayPath` -- annotate with -# this so a `MapSegment` is not a type error. +# Broader than an `ArrayPath`'s `StructSegment | ArraySegment`: a `MapPath` +# adds a trailing `MapSegment`. Consumers that walk an arbitrary +# `FieldPath`'s segments -- rather than a statically known `ArrayPath` -- +# annotate with this so a `MapSegment` is not a type error. FieldSegment: TypeAlias = StructSegment | ArraySegment | MapSegment -def _segment_str(seg: PathSegment) -> str: +def _segment_str(seg: StructSegment | ArraySegment) -> str: if isinstance(seg, ArraySegment): return seg.name + "[]" * seg.iter_count return seg.name From fb155b802b89e7ec7efa5c5cea7f0a0b41645d4c Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Fri, 17 Jul 2026 12:47:19 -0700 Subject: [PATCH 2/2] docs(pyspark): relate Check.expr and Check.read_columns expr and read_columns are two views of one computation: expr is the single composed Column, read_columns its read-set. The docstring now explains why they are carried separately, since the builder knows the columns as it composes expr and records them rather than recovering them from the finished Column. Signed-off-by: Seth Fitzsimmons --- .../src/overture/schema/pyspark/check.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py index 9500cf4e7..ac6d993b2 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py @@ -27,12 +27,21 @@ class Check: and report grouping), not how to access the data. The expression in `expr` already encodes the access pattern. - `read_columns` names every top-level schema column the expression - dereferences -- one for a plain field check, several for a model-level - check that spans columns, plus any discriminator a variant gate reads. - `validate_model` drops a check when any column it reads is skipped or - structurally absent, so an unresolvable `F.col()` never reaches Spark; - it also treats these as the columns a check can be suppressed by name. + `expr` and `read_columns` are two views of one computation, and each is + a "column" in a different sense. `read_columns` are real columns of the + underlying schema model -- the top-level columns the check must read to + evaluate. There is always at least one; a model-level constraint that + spans fields names several, plus any discriminator a variant gate reads. + `expr` is a *virtual column*: it is not a column of the schema model but + one synthesized by the generated validation machinery to hold the + composed expression the Spark engine evaluates. The two travel together + because the builder knows the read-set as it composes `expr`; recording + it is surer than recovering it from the finished `Column`. + + `validate_model` drops a check when any column in `read_columns` is + skipped or structurally absent, so an unresolvable `F.col()` never + reaches Spark; it also treats these as the columns a check can be + suppressed by name. """ field: str