From 379ee41121420734fc3f7fbf20d8e2efb2888923 Mon Sep 17 00:00:00 2001 From: Aaron Sequeira Date: Sun, 19 Jul 2026 14:46:11 +0300 Subject: [PATCH] GH-50515: [C++][Compute] Respect parent validity in nested struct cast When casting a struct with a nullable->non-nullable field change, CastStruct::Exec checks GetNullCount() on the child array without considering the parent struct's validity bitmap. This rejects casts where the child's physical nulls are fully masked by parent-level nulls. Intersect the parent and child validity bitmaps before deciding whether unmasked nulls exist, using BinaryBitBlockCounter for the common case and explicit fallbacks for absent bitmaps. --- .../compute/kernels/scalar_cast_nested.cc | 45 ++++++++- .../arrow/compute/kernels/scalar_cast_test.cc | 97 +++++++++++++++++++ python/pyarrow/tests/test_compute.py | 43 ++++++++ 3 files changed, 181 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_nested.cc b/cpp/src/arrow/compute/kernels/scalar_cast_nested.cc index 392fd9fbb705..47dd4578c2b1 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_nested.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_nested.cc @@ -29,6 +29,7 @@ #include "arrow/compute/kernels/common_internal.h" #include "arrow/compute/kernels/scalar_cast_internal.h" #include "arrow/util/bitmap_ops.h" +#include "arrow/util/bit_block_counter.h" #include "arrow/util/int_util.h" #include "arrow/util/logging_internal.h" @@ -384,12 +385,48 @@ struct CastStruct { const auto& in_field = in_type.field(in_field_index); const auto& in_values = (in_array.child_data[in_field_index].ToArrayData()->Slice( in_array.offset, in_array.length)); + if (in_field->nullable() && !out_field->nullable() && in_values->GetNullCount() > 0) { - return Status::Invalid( - "field '", in_field->name(), "' of type ", in_field->type()->ToString(), - " has nulls. Can't cast to non-nullable field '", out_field->name(), - "' of type ", out_field_type->ToString()); + bool has_nulls = false; + const uint8_t* parent_bitmap = in_array.buffers[0].data; + const uint8_t* child_bitmap = in_values->buffers.empty() ? nullptr : (in_values->buffers[0] ? in_values->buffers[0]->data() : nullptr); + + if (parent_bitmap == nullptr) { + // Parent has no nulls. Since child has nulls, they are unmasked. + has_nulls = true; + } else if (child_bitmap == nullptr) { + // Child has nulls but no bitmap (e.g. NullArray, RunEndEncoded, Union). + // We must semantically check if any valid parent element corresponds to a null child. + for (int64_t i = 0; i < in_array.length; ++i) { + if (arrow::bit_util::GetBit(parent_bitmap, in_array.offset + i) && + in_values->IsNull(i)) { + has_nulls = true; + break; + } + } + } else { + // Both parent and child have bitmaps. Check if parent is valid AND child is null. + arrow::internal::BinaryBitBlockCounter bit_counter( + parent_bitmap, in_array.offset, + child_bitmap, in_values->offset, in_array.length); + int64_t position = 0; + while (position < in_array.length) { + arrow::internal::BitBlockCount block = bit_counter.NextAndNotWord(); + if (block.popcount > 0) { + has_nulls = true; + break; + } + position += block.length; + } + } + + if (has_nulls) { + return Status::Invalid( + "field '", in_field->name(), "' of type ", in_field->type()->ToString(), + " has nulls. Can't cast to non-nullable field '", out_field->name(), + "' of type ", out_field_type->ToString()); + } } ARROW_ASSIGN_OR_RAISE(Datum cast_values, Cast(in_values, out_field_type, options, ctx->exec_context())); diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc index 51e6ca534cd5..56a5b655e594 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc @@ -4145,6 +4145,103 @@ TEST(Cast, StructToStructSubsetWithNulls) { CheckStructToStructSubsetWithNulls(NumericTypes()); } +TEST(Cast, StructNestedNullabilityAbsentParent) { + auto inner_type_dest = struct_({field("a", int32(), /*nullable=*/false)}); + auto outer_type_dest = struct_({field("inner", inner_type_dest)}); + auto inner_type_src = struct_({field("a", int32())}); + auto outer_type_src = struct_({field("inner", inner_type_src)}); + + auto src = ArrayFromJSON(outer_type_src, R"([ + {"inner": {"a": 1}}, + {"inner": {"a": null}} + ])"); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("has nulls. Can't cast to non-nullable field"), + Cast(src, CastOptions::Safe(outer_type_dest))); +} + +TEST(Cast, StructNestedNullabilityMasked) { + auto inner_type_dest = struct_({field("a", int32(), /*nullable=*/false)}); + auto outer_type_dest = struct_({field("inner", inner_type_dest)}); + auto inner_type_src = struct_({field("a", int32())}); + auto outer_type_src = struct_({field("inner", inner_type_src)}); + + auto src = ArrayFromJSON(outer_type_src, R"([ + {"inner": {"a": 1}}, + null + ])"); + auto expected = ArrayFromJSON(outer_type_dest, R"([ + {"inner": {"a": 1}}, + null + ])"); + CheckCast(src, expected); +} + +TEST(Cast, StructNestedNullabilitySliced) { + auto inner_type_dest = struct_({field("a", int32(), /*nullable=*/false)}); + auto outer_type_dest = struct_({field("inner", inner_type_dest)}); + auto inner_type_src = struct_({field("a", int32())}); + auto outer_type_src = struct_({field("inner", inner_type_src)}); + + auto src = ArrayFromJSON(outer_type_src, R"([ + {"inner": {"a": 1}}, + {"inner": {"a": 2}}, + {"inner": {"a": null}}, + null, + {"inner": {"a": 5}} + ])"); + auto expected = ArrayFromJSON(outer_type_dest, R"([ + {"inner": {"a": 1}}, + {"inner": {"a": 2}}, + {"inner": {"a": null}}, + null, + {"inner": {"a": 5}} + ])"); + + CheckCast(src->Slice(3, 2), expected->Slice(3, 2)); + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("has nulls. Can't cast to non-nullable field"), + Cast(src->Slice(2, 2), CastOptions::Safe(outer_type_dest))); +} + +TEST(Cast, StructNestedNullabilityAbsentChild) { + auto inner_type_dest = struct_({field("a", int32(), /*nullable=*/false)}); + auto outer_type_dest = struct_({field("inner", inner_type_dest)}); + auto inner_type_src = struct_({field("a", int32())}); + auto outer_type_src = struct_({field("inner", inner_type_src)}); + + auto src = ArrayFromJSON(outer_type_src, R"([ + {"inner": {"a": 1}}, + {"inner": {"a": 2}} + ])"); + auto expected = ArrayFromJSON(outer_type_dest, R"([ + {"inner": {"a": 1}}, + {"inner": {"a": 2}} + ])"); + CheckCast(src, expected); +} + +TEST(Cast, StructNestedNullabilityDeep) { + auto deep_inner_dest = struct_({field("a", int32(), /*nullable=*/false)}); + auto deep_mid_dest = struct_({field("mid", deep_inner_dest)}); + auto deep_outer_dest = struct_({field("outer", deep_mid_dest)}); + + auto deep_inner_src = struct_({field("a", int32())}); + auto deep_mid_src = struct_({field("mid", deep_inner_src)}); + auto deep_outer_src = struct_({field("outer", deep_mid_src)}); + + auto src = ArrayFromJSON(deep_outer_src, R"([ + {"outer": {"mid": {"a": 1}}}, + null + ])"); + auto expected = ArrayFromJSON(deep_outer_dest, R"([ + {"outer": {"mid": {"a": 1}}}, + null + ])"); + CheckCast(src, expected); +} + TEST(Cast, StructToSameSizedButDifferentNamedStruct) { std::vector src_field_names = {"a", "b"}; std::shared_ptr a, b; diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 1e08e73668e5..9ba41bdeba56 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -2395,6 +2395,49 @@ def check_cast_float_to_decimal(float_ty, float_val, decimal_ty, decimal_ctx, f"diff_digits = {diff_digits!r}") +def test_cast_struct_nested_nullability(): + # Arrow #50515: casting nested structs with non-nullable inner fields + # when the parent struct is nullable and marked as null + + # Target schema: outer struct is nullable, inner struct has non-nullable field + target_type = pa.struct([ + pa.field('inner', pa.struct([ + pa.field('a', pa.int32(), nullable=False) + ]), nullable=True) + ]) + + # Source array: + # [0] inner: {a: 1} + # [1] null + arr = pa.array([{'inner': {'a': 1}}, None]) + + # Casting should succeed, propagating the null appropriately + result = pc.cast(arr, target_type) + assert result.to_pylist() == [{'inner': {'a': 1}}, None] + + # True violation: outer struct is valid, but inner struct has null field 'a' + arr_fail = pa.array([{'inner': {'a': 1}}, {'inner': {'a': None}}]) + with pytest.raises(pa.ArrowInvalid, match="has nulls"): + pc.cast(arr_fail, target_type) + + # Slice test: a larger array sliced to only include masked nulls + arr_large = pa.array([ + {'inner': {'a': 1}}, + {'inner': {'a': None}}, + None, + {'inner': {'a': 5}} + ]) + + # Slice [2:4] includes `None, {'inner': {'a': 5}}` + # The true violation at index 1 is excluded. MUST succeed. + result_sliced = pc.cast(arr_large.slice(2, 2), target_type) + assert result_sliced.to_pylist() == [None, {'inner': {'a': 5}}] + + # Slice [1:3] includes `{'inner': {'a': None}}, None` + # The true violation at index 1 is included. MUST fail. + with pytest.raises(pa.ArrowInvalid, match="has nulls"): + pc.cast(arr_large.slice(1, 2), target_type) + # Cannot test float32 as case generators above assume float64 @pytest.mark.numpy @pytest.mark.parametrize('float_ty', [pa.float64()], ids=str)