Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
adopt review feedback
  • Loading branch information
sungwy committed Mar 20, 2025
commit adce6310d7ef8fb7b8e2148fc534174b6f048f83
17 changes: 5 additions & 12 deletions pyiceberg/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
from __future__ import annotations

import re
from enum import IntEnum
from functools import cached_property
from typing import (
Any,
Expand All @@ -54,7 +53,7 @@
from pydantic_core.core_schema import ValidatorFunctionWrapHandler

from pyiceberg.exceptions import ValidationError
from pyiceberg.typedef import IcebergBaseModel, IcebergRootModel, L
from pyiceberg.typedef import IcebergBaseModel, IcebergRootModel, L, TableVersion
from pyiceberg.utils.parsing import ParseNumberFromBrackets
from pyiceberg.utils.singleton import Singleton

Expand All @@ -63,12 +62,6 @@
FIXED_PARSER = ParseNumberFromBrackets(FIXED)


class TableVersion(IntEnum):
ONE = 1
TWO = 2
THREE = 3


def transform_dict_value_to_str(dict: Dict[str, Any]) -> Dict[str, str]:
"""Transform all values in the dictionary to string. Raise an error if any value is None."""
for key, value in dict.items():
Expand Down Expand Up @@ -190,7 +183,7 @@ def is_struct(self) -> bool:

def minimum_format_version(self) -> TableVersion:
"""Minimum Iceberg format version after which this type is supported."""
return TableVersion.ONE
return 1


class PrimitiveType(Singleton, IcebergRootModel[str], IcebergType):
Expand Down Expand Up @@ -734,7 +727,7 @@ class TimestampNanoType(PrimitiveType):
root: Literal["timestamp_ns"] = Field(default="timestamp_ns")

def minimum_format_version(self) -> TableVersion:
return TableVersion.THREE
return 3


class TimestamptzNanoType(PrimitiveType):
Expand All @@ -753,7 +746,7 @@ class TimestamptzNanoType(PrimitiveType):
root: Literal["timestamptz_ns"] = Field(default="timestamptz_ns")

def minimum_format_version(self) -> TableVersion:
return TableVersion.THREE
return 3


class StringType(PrimitiveType):
Expand Down Expand Up @@ -820,4 +813,4 @@ class UnknownType(PrimitiveType):
root: Literal["unknown"] = Field(default="unknown")

def minimum_format_version(self) -> TableVersion:
return TableVersion.THREE
return 3
1 change: 1 addition & 0 deletions tests/integration/test_reads.py
Original file line number Diff line number Diff line change
Expand Up @@ -1008,6 +1008,7 @@ def test_read_unknown_type(catalog: Catalog) -> None:
tbl = catalog.create_table(
identifier,
schema=arrow_table.schema,
properties={"format-version": "3"},
)

tbl.append(arrow_table)
Expand Down
16 changes: 13 additions & 3 deletions tests/table/test_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,12 @@
LongType,
MapType,
NestedField,
PrimitiveType,
StringType,
StructType,
TimestampNanoType,
TimestamptzNanoType,
UnknownType,
)


Expand Down Expand Up @@ -843,9 +845,17 @@ def test_new_table_metadata_with_v3_schema() -> None:
assert actual.sort_orders == [expected_sort_order]


def test_new_table_metadata_format_v2_with_v3_schema_fails() -> None:
@pytest.mark.parametrize(
"field_type",
[
TimestampNanoType(),
TimestamptzNanoType(),
UnknownType(),
],
)
def test_new_table_metadata_format_v2_with_v3_schema_fails(field_type: PrimitiveType) -> None:
schema = Schema(
NestedField(field_id=34, name="qux", field_type=TimestampNanoType(), required=False),
NestedField(field_id=34, name="qux", field_type=field_type, required=False),
schema_id=10,
)

Expand All @@ -858,7 +868,7 @@ def test_new_table_metadata_format_v2_with_v3_schema_fails() -> None:
order_id=34,
)

with pytest.raises(ValueError, match="timestamp_ns is only supported in 3 or higher. Current format version is: 2"):
with pytest.raises(ValueError, match=f"{field_type} is only supported in 3 or higher. Current format version is: 2"):
new_table_metadata(
schema=schema,
partition_spec=partition_spec,
Expand Down