diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py index 9500cf4e7..ac6d993b2 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py @@ -27,12 +27,21 @@ class Check: and report grouping), not how to access the data. The expression in `expr` already encodes the access pattern. - `read_columns` names every top-level schema column the expression - dereferences -- one for a plain field check, several for a model-level - check that spans columns, plus any discriminator a variant gate reads. - `validate_model` drops a check when any column it reads is skipped or - structurally absent, so an unresolvable `F.col()` never reaches Spark; - it also treats these as the columns a check can be suppressed by name. + `expr` and `read_columns` are two views of one computation, and each is + a "column" in a different sense. `read_columns` are real columns of the + underlying schema model -- the top-level columns the check must read to + evaluate. There is always at least one; a model-level constraint that + spans fields names several, plus any discriminator a variant gate reads. + `expr` is a *virtual column*: it is not a column of the schema model but + one synthesized by the generated validation machinery to hold the + composed expression the Spark engine evaluates. The two travel together + because the builder knows the read-set as it composes `expr`; recording + it is surer than recovering it from the finished `Column`. + + `validate_model` drops a check when any column in `read_columns` is + skipped or structurally absent, so an unresolvable `F.col()` never + reaches Spark; it also treats these as the columns a check can be + suppressed by name. """ field: str diff --git a/packages/overture-schema-system/src/overture/schema/system/field_path.py b/packages/overture-schema-system/src/overture/schema/system/field_path.py index 7b60fea56..df06232fe 100644 --- a/packages/overture-schema-system/src/overture/schema/system/field_path.py +++ b/packages/overture-schema-system/src/overture/schema/system/field_path.py @@ -38,7 +38,6 @@ "MapPath", "MapProjection", "MapSegment", - "PathSegment", "ScalarPath", "StructSegment", "coerce", @@ -88,9 +87,6 @@ class MapSegment: projection: MapProjection -PathSegment: TypeAlias = StructSegment | ArraySegment - - @dataclass(frozen=True, slots=True) class ScalarPath: """Locate a non-iterated value in a row.""" @@ -116,7 +112,7 @@ class ArrayPath: Invariant: `segments` contains at least one `ArraySegment`. """ - segments: tuple[PathSegment, ...] + segments: tuple[StructSegment | ArraySegment, ...] def __post_init__(self) -> None: if not any(isinstance(s, ArraySegment) for s in self.segments): @@ -334,14 +330,14 @@ def __str__(self) -> str: # The element type of any `FieldPath.segments`, across all three variants. -# Broader than `PathSegment` (array/scalar paths only): a `MapPath` adds a -# trailing `MapSegment`. Consumers that walk an arbitrary `FieldPath`'s -# segments -- rather than a statically known `ArrayPath` -- annotate with -# this so a `MapSegment` is not a type error. +# Broader than an `ArrayPath`'s `StructSegment | ArraySegment`: a `MapPath` +# adds a trailing `MapSegment`. Consumers that walk an arbitrary +# `FieldPath`'s segments -- rather than a statically known `ArrayPath` -- +# annotate with this so a `MapSegment` is not a type error. FieldSegment: TypeAlias = StructSegment | ArraySegment | MapSegment -def _segment_str(seg: PathSegment) -> str: +def _segment_str(seg: StructSegment | ArraySegment) -> str: if isinstance(seg, ArraySegment): return seg.name + "[]" * seg.iter_count return seg.name