From f8f6a90802ec6bb0ffbce9123d293b4eb76d17fd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 24 Jun 2026 00:38:11 +0000 Subject: [PATCH 1/3] Replace ToCanonical trait usages with execute This also removes ArrayBuilder::extend_from_array and moves that logic in to ArrayRef::append_to_builder Plumb ExecutionCtx through more methods instead of creating fresh one. Signed-off-by: Robert Kruszewski --- encodings/alp/src/alp/compute/cast.rs | 2 +- encodings/alp/src/alp/compute/filter.rs | 2 +- encodings/alp/src/alp/compute/mask.rs | 4 +- encodings/alp/src/alp/compute/take.rs | 2 +- encodings/alp/src/alp_rd/compute/cast.rs | 5 +- encodings/alp/src/alp_rd/compute/filter.rs | 2 + encodings/alp/src/alp_rd/compute/mask.rs | 2 + encodings/alp/src/alp_rd/compute/take.rs | 2 + encodings/bytebool/src/compute.rs | 16 +- encodings/datetime-parts/src/compute/cast.rs | 5 +- .../datetime-parts/src/compute/filter.rs | 4 +- encodings/datetime-parts/src/compute/take.rs | 5 +- .../src/decimal_byte_parts/compute/cast.rs | 5 +- .../src/decimal_byte_parts/compute/filter.rs | 12 +- encodings/experimental/onpair/src/array.rs | 14 +- .../src/bitpacking/array/bitpack_compress.rs | 2 +- .../fastlanes/src/bitpacking/compute/cast.rs | 2 +- .../src/bitpacking/compute/filter.rs | 6 +- .../fastlanes/src/bitpacking/compute/take.rs | 4 +- encodings/fastlanes/src/delta/compute/cast.rs | 5 +- encodings/fastlanes/src/for/compute/cast.rs | 2 +- encodings/fastlanes/src/for/compute/mod.rs | 24 ++- encodings/fastlanes/src/rle/compute/cast.rs | 2 +- encodings/fsst/src/array.rs | 14 +- encodings/fsst/src/canonical.rs | 2 +- encodings/fsst/src/compute/cast.rs | 2 +- encodings/fsst/src/compute/mod.rs | 2 +- encodings/pco/src/compute/cast.rs | 2 +- encodings/runend/src/arbitrary.rs | 33 ++- encodings/runend/src/compute/cast.rs | 2 +- encodings/runend/src/compute/take.rs | 4 +- encodings/sequence/src/compute/cast.rs | 2 +- encodings/sequence/src/compute/filter.rs | 7 +- encodings/sequence/src/compute/take.rs | 9 +- encodings/sparse/src/compute/cast.rs | 2 +- encodings/sparse/src/compute/filter.rs | 2 + encodings/sparse/src/compute/mod.rs | 2 + encodings/sparse/src/compute/take.rs | 4 +- encodings/sparse/src/lib.rs | 2 +- encodings/zigzag/src/compute/cast.rs | 2 +- encodings/zigzag/src/compute/mod.rs | 12 +- encodings/zstd/src/compute/cast.rs | 2 +- encodings/zstd/src/zstd_buffers.rs | 20 +- fuzz/src/array/mod.rs | 55 +++-- fuzz/src/compress.rs | 9 +- .../benches/dict_unreferenced_mask.rs | 19 +- vortex-array/benches/filter_bool.rs | 3 +- .../benches/listview_builder_extend.rs | 14 +- vortex-array/benches/take_patches.rs | 32 ++- vortex-array/benches/varbinview_compact.rs | 49 +++-- .../fns/uncompressed_size_in_bytes/mod.rs | 10 +- vortex-array/src/array/mod.rs | 2 +- vortex-array/src/array/vtable/mod.rs | 3 +- vortex-array/src/arrays/arbitrary.rs | 6 +- vortex-array/src/arrays/bool/compute/cast.rs | 2 +- .../src/arrays/bool/compute/fill_null.rs | 9 +- .../src/arrays/bool/compute/filter.rs | 5 +- vortex-array/src/arrays/bool/compute/mask.rs | 7 +- vortex-array/src/arrays/bool/compute/take.rs | 11 +- vortex-array/src/arrays/bool/vtable/mod.rs | 10 +- .../src/arrays/bool/vtable/operations.rs | 19 +- vortex-array/src/arrays/chunked/array.rs | 50 ++--- .../src/arrays/chunked/compute/cast.rs | 2 +- .../src/arrays/chunked/compute/filter.rs | 7 +- .../src/arrays/chunked/compute/mask.rs | 7 +- .../src/arrays/chunked/compute/take.rs | 21 +- .../src/arrays/chunked/compute/zip.rs | 7 +- .../src/arrays/chunked/paired_chunks.rs | 20 +- vortex-array/src/arrays/chunked/tests.rs | 49 ++--- .../src/arrays/chunked/vtable/canonical.rs | 2 +- vortex-array/src/arrays/chunked/vtable/mod.rs | 13 +- vortex-array/src/arrays/constant/arbitrary.rs | 6 +- .../src/arrays/constant/compute/cast.rs | 6 +- .../src/arrays/constant/compute/mod.rs | 32 ++- .../src/arrays/constant/compute/take.rs | 17 +- .../src/arrays/constant/vtable/mod.rs | 2 +- vortex-array/src/arrays/datetime/test.rs | 9 +- .../src/arrays/decimal/compute/cast.rs | 75 ++++--- .../src/arrays/decimal/compute/fill_null.rs | 18 +- .../src/arrays/decimal/compute/take.rs | 5 +- vortex-array/src/arrays/decimal/vtable/mod.rs | 13 ++ vortex-array/src/arrays/dict/arbitrary.rs | 1 - vortex-array/src/arrays/dict/array.rs | 41 ++-- vortex-array/src/arrays/dict/compute/cast.rs | 2 +- .../src/arrays/dict/compute/is_constant.rs | 2 +- .../src/arrays/dict/compute/min_max.rs | 2 +- vortex-array/src/arrays/dict/compute/mod.rs | 18 +- vortex-array/src/arrays/dict/compute/rules.rs | 5 +- .../src/arrays/extension/compute/cast.rs | 2 +- .../src/arrays/extension/compute/mod.rs | 17 +- .../src/arrays/extension/compute/rules.rs | 17 +- .../src/arrays/extension/vtable/mod.rs | 13 ++ .../src/arrays/filter/execute/bool.rs | 17 +- .../src/arrays/filter/execute/decimal.rs | 12 +- .../arrays/filter/execute/fixed_size_list.rs | 10 +- .../src/arrays/filter/execute/listview.rs | 10 +- vortex-array/src/arrays/filter/execute/mod.rs | 10 +- .../src/arrays/filter/execute/primitive.rs | 17 +- .../src/arrays/filter/execute/struct_.rs | 12 +- .../src/arrays/filter/execute/varbinview.rs | 41 ++-- vortex-array/src/arrays/filter/vtable.rs | 2 +- .../src/arrays/fixed_size_list/array.rs | 10 - .../arrays/fixed_size_list/tests/filter.rs | 2 +- .../arrays/fixed_size_list/tests/nested.rs | 57 ++--- .../src/arrays/fixed_size_list/tests/take.rs | 10 +- .../src/arrays/fixed_size_list/vtable/mod.rs | 13 ++ vortex-array/src/arrays/list/compute/cast.rs | 2 +- vortex-array/src/arrays/list/compute/mod.rs | 10 +- vortex-array/src/arrays/list/compute/take.rs | 24 ++- vortex-array/src/arrays/list/test_harness.rs | 9 +- vortex-array/src/arrays/list/vtable/mod.rs | 13 ++ vortex-array/src/arrays/listview/array.rs | 34 ++- vortex-array/src/arrays/listview/mod.rs | 3 +- vortex-array/src/arrays/listview/rebuild.rs | 6 +- .../src/arrays/listview/tests/basic.rs | 21 +- .../src/arrays/listview/tests/filter.rs | 2 +- .../src/arrays/listview/tests/operations.rs | 47 ++-- .../src/arrays/listview/tests/take.rs | 2 +- .../src/arrays/listview/vtable/mod.rs | 17 ++ .../src/arrays/masked/compute/filter.rs | 7 +- .../src/arrays/masked/compute/mask.rs | 7 +- .../src/arrays/masked/compute/take.rs | 7 +- vortex-array/src/arrays/masked/tests.rs | 10 +- vortex-array/src/arrays/null/compute/cast.rs | 5 +- vortex-array/src/arrays/null/compute/mod.rs | 48 +++-- vortex-array/src/arrays/null/compute/take.rs | 10 +- vortex-array/src/arrays/null/mod.rs | 15 ++ vortex-array/src/arrays/patched/vtable/mod.rs | 3 +- .../src/arrays/primitive/array/patch.rs | 5 +- .../src/arrays/primitive/array/top_value.rs | 12 +- .../src/arrays/primitive/compute/cast.rs | 54 ++--- .../src/arrays/primitive/compute/fill_null.rs | 21 +- .../src/arrays/primitive/compute/mask.rs | 7 +- .../src/arrays/primitive/compute/take/mod.rs | 18 +- vortex-array/src/arrays/primitive/tests.rs | 12 ++ .../src/arrays/primitive/vtable/mod.rs | 3 +- .../src/arrays/scalar_fn/vtable/validity.rs | 26 ++- .../src/arrays/struct_/compute/cast.rs | 2 +- .../src/arrays/struct_/compute/mod.rs | 8 + vortex-array/src/arrays/struct_/vtable/mod.rs | 13 ++ .../src/arrays/varbin/compute/cast.rs | 2 +- .../src/arrays/varbin/compute/compare.rs | 23 +- .../src/arrays/varbin/compute/filter.rs | 10 +- .../src/arrays/varbin/compute/mask.rs | 12 +- .../src/arrays/varbin/compute/take.rs | 5 +- .../src/arrays/varbin/vtable/canonical.rs | 21 +- vortex-array/src/arrays/varbinview/compact.rs | 72 ++++--- .../src/arrays/varbinview/compute/cast.rs | 2 +- .../src/arrays/varbinview/compute/mask.rs | 4 + .../src/arrays/varbinview/compute/mod.rs | 5 +- .../src/arrays/varbinview/compute/take.rs | 5 +- .../src/arrays/varbinview/compute/zip.rs | 6 +- .../src/arrays/varbinview/vtable/mod.rs | 10 +- vortex-array/src/arrow/executor/byte.rs | 33 +-- vortex-array/src/arrow/executor/byte_view.rs | 2 +- vortex-array/src/builders/bool.rs | 22 +- vortex-array/src/builders/decimal.rs | 56 +++-- vortex-array/src/builders/extension.rs | 23 +- vortex-array/src/builders/fixed_size_list.rs | 139 ++++++------ vortex-array/src/builders/list.rs | 203 ++++++++++-------- vortex-array/src/builders/listview.rs | 158 +++++++------- vortex-array/src/builders/mod.rs | 28 +-- vortex-array/src/builders/null.rs | 7 +- vortex-array/src/builders/primitive.rs | 14 +- vortex-array/src/builders/struct_.rs | 46 ++-- vortex-array/src/builders/tests.rs | 65 +++--- vortex-array/src/builders/varbinview.rs | 26 +-- vortex-array/src/canonical.rs | 2 +- vortex-array/src/compute/conformance/cast.rs | 169 ++++++++------- .../src/compute/conformance/filter.rs | 51 +++-- vortex-array/src/compute/conformance/mask.rs | 83 ++++--- vortex-array/src/compute/conformance/take.rs | 112 +++++----- vortex-array/src/display/mod.rs | 6 +- vortex-array/src/executor.rs | 6 +- vortex-array/src/expr/exprs.rs | 48 +++-- vortex-array/src/patches.rs | 37 ++-- .../src/scalar_fn/fns/binary/boolean.rs | 11 +- .../src/scalar_fn/fns/list_contains/mod.rs | 32 +-- vortex-array/src/scalar_fn/fns/merge.rs | 41 ++-- vortex-array/src/scalar_fn/fns/not/mod.rs | 13 +- vortex-array/src/scalar_fn/fns/pack.rs | 50 +++-- vortex-array/src/scalar_fn/fns/select.rs | 22 +- .../src/scalar_fn/fns/variant_get/mod.rs | 2 +- vortex-btrblocks/src/schemes/binary/zstd.rs | 5 +- vortex-btrblocks/src/schemes/string/zstd.rs | 5 +- vortex-cxx/src/read.rs | 3 +- vortex-file/src/tests.rs | 6 +- vortex-layout/src/layouts/file_stats.rs | 21 +- vortex-layout/src/layouts/zoned/builder.rs | 9 +- vortex-layout/src/layouts/zoned/writer.rs | 5 +- vortex-python/src/iter/mod.rs | 3 +- vortex-web/crate/src/wasm.rs | 2 +- 192 files changed, 1976 insertions(+), 1475 deletions(-) diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index d77a7257544..324500889ee 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -154,7 +154,7 @@ mod tests { let array_primitive = array.execute::(&mut ctx)?; let alp = alp_encode(array_primitive.as_view(), None, &mut ctx).vortex_expect("cannot fail"); - test_cast_conformance(&alp.into_array()); + test_cast_conformance(&alp.into_array(), &mut ctx); Ok(()) } diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index f38a87f19b0..8c7212cdeab 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -65,6 +65,6 @@ mod test { let mut ctx = array_session().create_execution_ctx(); let array_primitive = array.execute::(&mut ctx).unwrap(); let alp = alp_encode(array_primitive.as_view(), None, &mut ctx).unwrap(); - test_filter_conformance(&alp.into_array()); + test_filter_conformance(&alp.into_array(), &mut ctx); } } diff --git a/encodings/alp/src/alp/compute/mask.rs b/encodings/alp/src/alp/compute/mask.rs index b117ac216d0..796b0fa0813 100644 --- a/encodings/alp/src/alp/compute/mask.rs +++ b/encodings/alp/src/alp/compute/mask.rs @@ -81,7 +81,7 @@ mod test { let mut ctx = array_session().create_execution_ctx(); let array_primitive = array.execute::(&mut ctx).unwrap(); let alp = alp_encode(array_primitive.as_view(), None, &mut ctx).unwrap(); - test_mask_conformance(&alp.into_array()); + test_mask_conformance(&alp.into_array(), &mut ctx); } #[test] @@ -95,7 +95,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let alp = alp_encode(array.as_view(), None, &mut ctx).unwrap(); assert!(alp.patches().is_some(), "expected patches"); - test_mask_conformance(&alp.into_array()); + test_mask_conformance(&alp.into_array(), &mut ctx); } #[test] diff --git a/encodings/alp/src/alp/compute/take.rs b/encodings/alp/src/alp/compute/take.rs index 9b193a3cfbe..d7673455de4 100644 --- a/encodings/alp/src/alp/compute/take.rs +++ b/encodings/alp/src/alp/compute/take.rs @@ -59,6 +59,6 @@ mod test { let mut ctx = array_session().create_execution_ctx(); let array_primitive = array.execute::(&mut ctx).unwrap(); let alp = alp_encode(array_primitive.as_view(), None, &mut ctx).unwrap(); - test_take_conformance(&alp.into_array()); + test_take_conformance(&alp.into_array(), &mut ctx); } } diff --git a/encodings/alp/src/alp_rd/compute/cast.rs b/encodings/alp/src/alp_rd/compute/cast.rs index 717b97b760e..8a651708d3f 100644 --- a/encodings/alp/src/alp_rd/compute/cast.rs +++ b/encodings/alp/src/alp_rd/compute/cast.rs @@ -149,6 +149,9 @@ mod tests { encoder.encode(arr.as_view(), &mut array_session().create_execution_ctx()) })] fn test_cast_alprd_conformance(#[case] alprd: crate::alp_rd::ALPRDArray) { - test_cast_conformance(&alprd.into_array()); + test_cast_conformance( + &alprd.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/encodings/alp/src/alp_rd/compute/filter.rs b/encodings/alp/src/alp_rd/compute/filter.rs index 38d3fa96156..5d98b4561ce 100644 --- a/encodings/alp/src/alp_rd/compute/filter.rs +++ b/encodings/alp/src/alp_rd/compute/filter.rs @@ -95,6 +95,7 @@ mod test { &mut ctx, ) .into_array(), + &mut ctx, ); } @@ -111,6 +112,7 @@ mod test { &mut ctx, ) .into_array(), + &mut ctx, ); } } diff --git a/encodings/alp/src/alp_rd/compute/mask.rs b/encodings/alp/src/alp_rd/compute/mask.rs index cac98683adb..335f18a33cb 100644 --- a/encodings/alp/src/alp_rd/compute/mask.rs +++ b/encodings/alp/src/alp_rd/compute/mask.rs @@ -63,6 +63,7 @@ mod tests { &mut ctx, ) .into_array(), + &mut ctx, ); } @@ -79,6 +80,7 @@ mod tests { &mut ctx, ) .into_array(), + &mut ctx, ); } } diff --git a/encodings/alp/src/alp_rd/compute/take.rs b/encodings/alp/src/alp_rd/compute/take.rs index dee2454ea05..6db6eae2978 100644 --- a/encodings/alp/src/alp_rd/compute/take.rs +++ b/encodings/alp/src/alp_rd/compute/take.rs @@ -149,6 +149,7 @@ mod test { &mut ctx, ) .into_array(), + &mut ctx, ); } @@ -165,6 +166,7 @@ mod test { &mut ctx, ) .into_array(), + &mut ctx, ); } } diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index 112abc921a4..d68154b6f2b 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -297,17 +297,25 @@ mod tests { #[test] fn test_mask_byte_bool() { - test_mask_conformance(&bb(vec![true, false, true, true, false]).into_array()); + test_mask_conformance( + &bb(vec![true, false, true, true, false]).into_array(), + &mut SESSION.create_execution_ctx(), + ); test_mask_conformance( &bb_opt(vec![Some(true), Some(true), None, Some(false), None]).into_array(), + &mut SESSION.create_execution_ctx(), ); } #[test] fn test_filter_byte_bool() { - test_filter_conformance(&bb(vec![true, false, true, true, false]).into_array()); + test_filter_conformance( + &bb(vec![true, false, true, true, false]).into_array(), + &mut SESSION.create_execution_ctx(), + ); test_filter_conformance( &bb_opt(vec![Some(true), Some(true), None, Some(false), None]).into_array(), + &mut SESSION.create_execution_ctx(), ); } @@ -317,7 +325,7 @@ mod tests { #[case(bb(vec![true, false]))] #[case(bb(vec![true]))] fn test_take_byte_bool_conformance(#[case] array: ByteBoolArray) { - test_take_conformance(&array.into_array()); + test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -338,7 +346,7 @@ mod tests { #[case(bb(vec![true]))] #[case(bb_opt(vec![Some(true), None]))] fn test_cast_bytebool_conformance(#[case] array: ByteBoolArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[rstest] diff --git a/encodings/datetime-parts/src/compute/cast.rs b/encodings/datetime-parts/src/compute/cast.rs index 90aba343ae3..fbfb6f87e39 100644 --- a/encodings/datetime-parts/src/compute/cast.rs +++ b/encodings/datetime-parts/src/compute/cast.rs @@ -140,6 +140,9 @@ mod tests { ), &mut array_session().create_execution_ctx()).unwrap())] fn test_cast_datetime_parts_conformance(#[case] array: DateTimePartsArray) { use vortex_array::compute::conformance::cast::test_cast_conformance; - test_cast_conformance(&array.into_array()); + test_cast_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/encodings/datetime-parts/src/compute/filter.rs b/encodings/datetime-parts/src/compute/filter.rs index 696660510d5..6578467eddd 100644 --- a/encodings/datetime-parts/src/compute/filter.rs +++ b/encodings/datetime-parts/src/compute/filter.rs @@ -54,7 +54,7 @@ mod test { TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); let array = DateTimeParts::try_from_temporal(temporal, &mut ctx).unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut ctx); // Test with nullable values let timestamps = PrimitiveArray::from_option_iter([ @@ -70,6 +70,6 @@ mod test { TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); let array = DateTimeParts::try_from_temporal(temporal, &mut ctx).unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut ctx); } } diff --git a/encodings/datetime-parts/src/compute/take.rs b/encodings/datetime-parts/src/compute/take.rs index 9f5a50d3a9b..2b49cfaa15d 100644 --- a/encodings/datetime-parts/src/compute/take.rs +++ b/encodings/datetime-parts/src/compute/take.rs @@ -136,6 +136,9 @@ mod tests { Some("UTC".into()) ), &mut array_session().create_execution_ctx()).unwrap())] fn test_take_datetime_parts_conformance(#[case] array: DateTimePartsArray) { - test_take_conformance(&array.into_array()); + test_take_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs index 376ad12f564..303e9d3247b 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs @@ -118,6 +118,9 @@ mod tests { DecimalDType::new(10, 2), ).unwrap())] fn test_cast_decimal_byte_parts_conformance(#[case] array: DecimalBytePartsArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs index c82171b0e13..a1b50ad788a 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs @@ -27,6 +27,8 @@ impl FilterReduce for DecimalByteParts { #[cfg(test)] mod test { use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::filter::test_filter_conformance; use vortex_array::dtype::DecimalDType; @@ -41,7 +43,10 @@ mod test { let decimal_dtype = DecimalDType::new(8, 2); let array = DecimalByteParts::try_new(msp, decimal_dtype).unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); // Test with nullable values let msp = PrimitiveArray::from_option_iter([Some(10i64), None, Some(30), Some(40), None]) @@ -49,6 +54,9 @@ mod test { let decimal_dtype = DecimalDType::new(18, 4); let array = DecimalByteParts::try_new(msp, decimal_dtype).unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/encodings/experimental/onpair/src/array.rs b/encodings/experimental/onpair/src/array.rs index f4c8c1aed85..7fe1cfbce5e 100644 --- a/encodings/experimental/onpair/src/array.rs +++ b/encodings/experimental/onpair/src/array.rs @@ -475,14 +475,12 @@ impl VTable for OnPair { ctx: &mut ExecutionCtx, ) -> VortexResult<()> { let Some(builder) = builder.as_any_mut().downcast_mut::() else { - builder.extend_from_array( - &array - .array() - .clone() - .execute::(ctx)? - .into_array(), - ); - return Ok(()); + return array + .array() + .clone() + .execute::(ctx)? + .into_array() + .append_to_builder(builder, ctx); }; let next_buffer_index = builder.completed_block_count() + u32::from(builder.in_progress()); diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index f7a3485c113..a393db6ecc8 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -521,7 +521,7 @@ mod test { let mut primitive_builder = PrimitiveBuilder::::with_capacity(chunked.dtype().nullability(), 10 * 100); - primitive_builder.extend_from_array(&chunked); + chunked.append_to_builder(&mut primitive_builder, &mut ctx)?; let ca_into = primitive_builder.finish(); assert_arrays_eq!(into_ca, ca_into, &mut ctx); diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index 3b917aa36af..f7cfc56acf0 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -266,6 +266,6 @@ mod tests { #[case(bp(&buffer![0u32, 1000, 2000, 3000, 4000].into_array(), 12))] #[case(bp(&PrimitiveArray::from_option_iter([Some(1u32), None, Some(7), Some(15), None]).into_array(), 4))] fn test_cast_bitpacked_conformance(#[case] array: BitPackedArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index 21184d785a5..1530c33874d 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -273,17 +273,17 @@ mod test { // Test with u8 values let unpacked = buffer![1u8, 2, 3, 4, 5].into_array(); let bitpacked = BitPackedData::encode(&unpacked, 3, &mut ctx).unwrap(); - test_filter_conformance(&bitpacked.into_array()); + test_filter_conformance(&bitpacked.into_array(), &mut ctx); // Test with u32 values let unpacked = buffer![100u32, 200, 300, 400, 500].into_array(); let bitpacked = BitPackedData::encode(&unpacked, 9, &mut ctx).unwrap(); - test_filter_conformance(&bitpacked.into_array()); + test_filter_conformance(&bitpacked.into_array(), &mut ctx); // Test with nullable values let unpacked = PrimitiveArray::from_option_iter([Some(1u16), None, Some(3), Some(4), None]); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 3, &mut ctx).unwrap(); - test_filter_conformance(&bitpacked.into_array()); + test_filter_conformance(&bitpacked.into_array(), &mut ctx); } /// Regression test for signed integers with patches. diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 7ffe051ca40..8731df0ffb3 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -170,6 +170,7 @@ mod test { use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::compute::conformance::take::test_take_conformance; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; @@ -328,7 +329,6 @@ mod test { #[case(bp(buffer![42u32].into_array(), 6))] #[case(bp(PrimitiveArray::from_iter((0..1024).map(|i| i as u32)).into_array(), 8))] fn test_take_bitpacked_conformance(#[case] bitpacked: BitPackedArray) { - use vortex_array::compute::conformance::take::test_take_conformance; - test_take_conformance(&bitpacked.into_array()); + test_take_conformance(&bitpacked.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index 2ee10e7619a..c6de602bb0b 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -244,6 +244,9 @@ mod tests { let delta_array = Delta::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) .unwrap(); - test_cast_conformance(&delta_array.into_array()); + test_cast_conformance( + &delta_array.into_array(), + &mut SESSION.create_execution_ctx(), + ); } } diff --git a/encodings/fastlanes/src/for/compute/cast.rs b/encodings/fastlanes/src/for/compute/cast.rs index b865af43d12..e56188d9e00 100644 --- a/encodings/fastlanes/src/for/compute/cast.rs +++ b/encodings/fastlanes/src/for/compute/cast.rs @@ -118,6 +118,6 @@ mod tests { Scalar::from(-100i32) ))] fn test_cast_for_conformance(#[case] array: FoRArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/encodings/fastlanes/src/for/compute/mod.rs b/encodings/fastlanes/src/for/compute/mod.rs index 84e8812b598..168fcbc4192 100644 --- a/encodings/fastlanes/src/for/compute/mod.rs +++ b/encodings/fastlanes/src/for/compute/mod.rs @@ -49,8 +49,11 @@ mod test { use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::filter::test_filter_conformance; + use vortex_array::compute::conformance::take::test_take_conformance; use vortex_array::scalar::Scalar; use vortex_buffer::buffer; use vortex_error::VortexExpect; @@ -68,18 +71,27 @@ mod test { buffer![100i32, 101, 102, 103, 104].into_array(), Scalar::from(100i32), ); - test_filter_conformance(&for_array.into_array()); + test_filter_conformance( + &for_array.into_array(), + &mut array_session().create_execution_ctx(), + ); let for_array = fa( buffer![1000u64, 1001, 1002, 1003, 1004].into_array(), Scalar::from(1000u64), ); - test_filter_conformance(&for_array.into_array()); + test_filter_conformance( + &for_array.into_array(), + &mut array_session().create_execution_ctx(), + ); let values = PrimitiveArray::from_option_iter([Some(50i16), None, Some(52), Some(53), None]); let for_array = fa(values.into_array(), Scalar::from(50i16)); - test_filter_conformance(&for_array.into_array()); + test_filter_conformance( + &for_array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[rstest] @@ -92,8 +104,10 @@ mod test { #[case(fa(buffer![-100i32, -99, -98, -97, -96].into_array(), Scalar::from(-100i32)))] #[case(fa(buffer![42i64].into_array(), Scalar::from(40i64)))] fn test_take_for_conformance(#[case] for_array: FoRArray) { - use vortex_array::compute::conformance::take::test_take_conformance; - test_take_conformance(&for_array.into_array()); + test_take_conformance( + &for_array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index 92947d927bc..38e7927f8e6 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -162,6 +162,6 @@ mod tests { fn test_cast_rle_conformance(#[case] primitive: PrimitiveArray) { let mut ctx = SESSION.create_execution_ctx(); let rle_array = rle(&primitive, &mut ctx); - test_cast_conformance(&rle_array.into_array()); + test_cast_conformance(&rle_array.into_array(), &mut ctx); } } diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 532fea8bae5..9438b772469 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -285,14 +285,12 @@ impl VTable for FSST { ctx: &mut ExecutionCtx, ) -> VortexResult<()> { let Some(builder) = builder.as_any_mut().downcast_mut::() else { - builder.extend_from_array( - &array - .array() - .clone() - .execute::(ctx)? - .into_array(), - ); - return Ok(()); + return array + .array() + .clone() + .execute::(ctx)? + .into_array() + .append_to_builder(builder, ctx); }; // Decompress the whole block of data into a new buffer, and create some views diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index 43712aad7dd..5c58dbbd501 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -180,7 +180,7 @@ mod tests { .append_to_builder(&mut builder, &mut ctx)?; { - let arr = builder.finish_into_canonical().into_varbinview(); + let arr = builder.finish_into_canonical(&mut ctx).into_varbinview(); let mask = arr.validity()?.execute_mask(arr.len(), &mut ctx)?; let res1 = (0..arr.len()) .map(|i| mask.value(i).then(|| arr.bytes_at(i).to_vec())) diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs index bdca68e841b..a1469681590 100644 --- a/encodings/fsst/src/compute/cast.rs +++ b/encodings/fsst/src/compute/cast.rs @@ -147,7 +147,7 @@ mod tests { let array = array.into_array(); let compressor = fsst_train_compressor(&array, &mut ctx)?; let fsst = fsst_compress(&array, &compressor, &mut ctx)?; - test_cast_conformance(&fsst.into_array()); + test_cast_conformance(&fsst.into_array(), &mut ctx); Ok(()) } } diff --git a/encodings/fsst/src/compute/mod.rs b/encodings/fsst/src/compute/mod.rs index 0065e0f499a..d447b4b4958 100644 --- a/encodings/fsst/src/compute/mod.rs +++ b/encodings/fsst/src/compute/mod.rs @@ -117,7 +117,7 @@ mod tests { let varbin = varbin.into_array(); let compressor = fsst_train_compressor(&varbin, &mut ctx)?; let array = fsst_compress(&varbin, &compressor, &mut ctx)?; - test_take_conformance(&array.into_array()); + test_take_conformance(&array.into_array(), &mut ctx); Ok(()) } diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index 5f562fafca6..c59e205e2e1 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -188,6 +188,6 @@ mod tests { fn test_cast_pco_conformance(#[case] values: PrimitiveArray) { let mut ctx = SESSION.create_execution_ctx(); let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); - test_cast_conformance(&pco.into_array()); + test_cast_conformance(&pco.into_array(), &mut ctx); } } diff --git a/encodings/runend/src/arbitrary.rs b/encodings/runend/src/arbitrary.rs index baa15e0fd13..c9e5d692ba7 100644 --- a/encodings/runend/src/arbitrary.rs +++ b/encodings/runend/src/arbitrary.rs @@ -4,9 +4,8 @@ use arbitrary::Arbitrary; use arbitrary::Result; use arbitrary::Unstructured; +use vortex_array::ArrayRef; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::arbitrary::ArbitraryArray; use vortex_array::arrays::arbitrary::ArbitraryArrayConfig; @@ -31,7 +30,7 @@ impl<'a> Arbitrary<'a> for ArbitraryRunEndArray { let ptype: PType = u.arbitrary()?; let nullability: Nullability = u.arbitrary()?; let dtype = DType::Primitive(ptype, nullability); - Self::with_dtype(u, &dtype, None) + Self::with_dtype(u, &dtype) } } @@ -39,15 +38,16 @@ impl ArbitraryRunEndArray { /// Generate an arbitrary RunEndArray with the given dtype for values. /// /// The dtype must be a primitive or boolean type. - pub fn with_dtype(u: &mut Unstructured, dtype: &DType, len: Option) -> Result { + pub fn with_dtype(u: &mut Unstructured, dtype: &DType) -> Result { // Number of runs (values/ends pairs) let num_runs = u.int_in_range(0..=20)?; - // TODO(ctx): trait fixes - Arbitrary::arbitrary has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); if num_runs == 0 { // Empty RunEndArray - let ends = PrimitiveArray::from_iter(Vec::::new()).into_array(); + let ends = unsafe { + PrimitiveArray::new_unchecked(Buffer::::empty(), Validity::NonNullable) + .into_array() + }; let values = ArbitraryArray::arbitrary_with_config( u, &ArbitraryArrayConfig { @@ -56,8 +56,7 @@ impl ArbitraryRunEndArray { }, )? .0; - let runend_array = RunEnd::try_new(ends, values, &mut ctx) - .vortex_expect("Empty RunEndArray creation should succeed"); + let runend_array = unsafe { RunEnd::new_unchecked(ends, values, 0, 0) }; return Ok(ArbitraryRunEndArray(runend_array)); } @@ -71,12 +70,12 @@ impl ArbitraryRunEndArray { )? .0; + let len = u.int_in_range(0..=2048)?; // Generate strictly increasing ends // Each end must be > previous end, and first end must be >= 1 let ends = random_strictly_sorted_ends(u, num_runs, len)?; - let runend_array = RunEnd::try_new(ends, values, &mut ctx) - .vortex_expect("RunEndArray creation should succeed in arbitrary impl"); + let runend_array = unsafe { RunEnd::new_unchecked(ends, values, 0, len) }; Ok(ArbitraryRunEndArray(runend_array)) } @@ -89,8 +88,8 @@ impl ArbitraryRunEndArray { fn random_strictly_sorted_ends( u: &mut Unstructured, num_runs: usize, - target_len: Option, -) -> Result { + target_len: usize, +) -> Result { // Choose a random unsigned PType for ends let ends_ptype = *u.choose(&[PType::U8, PType::U16, PType::U32, PType::U64])?; @@ -101,17 +100,17 @@ fn random_strictly_sorted_ends( for i in 0..num_runs { // Each run must have at least length 1, so increment by at least 1 - let increment = match (i == num_runs - 1, target_len) { - (true, Some(target)) => { + let increment = match i == num_runs - 1 { + true => { // Last element should reach target_len - let target = target as u64; + let target = target_len as u64; if target > current { target - current } else { 1 } } - _ => { + false => { // Random increment between 1 and 10 u.int_in_range(1..=10)? } diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index fe740739ca9..4722f9e5549 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -184,6 +184,6 @@ mod tests { fn test_cast_runend_conformance(#[case] build: RunEndBuilder) { let mut ctx = SESSION.create_execution_ctx(); let array = build(&mut ctx); - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut ctx); } } diff --git a/encodings/runend/src/compute/take.rs b/encodings/runend/src/compute/take.rs index c61854635e3..4ec601e71a6 100644 --- a/encodings/runend/src/compute/take.rs +++ b/encodings/runend/src/compute/take.rs @@ -751,7 +751,7 @@ mod tests { .unwrap() })] fn test_take_runend_conformance(#[case] array: RunEndArray) { - test_take_conformance(&array.into_array()); + test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[rstest] @@ -765,6 +765,6 @@ mod tests { array.slice(2..8).unwrap() })] fn test_take_sliced_runend_conformance(#[case] sliced: ArrayRef) { - test_take_conformance(&sliced); + test_take_conformance(&sliced, &mut SESSION.create_execution_ctx()); } } diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs index e0a63d0c922..fb18cf11181 100644 --- a/encodings/sequence/src/compute/cast.rs +++ b/encodings/sequence/src/compute/cast.rs @@ -213,6 +213,6 @@ mod tests { 5, ).unwrap())] fn test_cast_sequence_conformance(#[case] sequence: SequenceArray) { - test_cast_conformance(&sequence.into_array()); + test_cast_conformance(&sequence.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/encodings/sequence/src/compute/filter.rs b/encodings/sequence/src/compute/filter.rs index 3ef24188fbf..aae06610d82 100644 --- a/encodings/sequence/src/compute/filter.rs +++ b/encodings/sequence/src/compute/filter.rs @@ -48,6 +48,8 @@ fn filter_impl(mul: T, base: T, mask: &Mask, validity: Validity) mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; use vortex_array::compute::conformance::filter::LARGE_SIZE; use vortex_array::compute::conformance::filter::MEDIUM_SIZE; use vortex_array::compute::conformance::filter::test_filter_conformance; @@ -70,6 +72,9 @@ mod tests { #[case(Sequence::try_new_typed(0u32, 5, Nullability::NonNullable, MEDIUM_SIZE).unwrap())] #[case(Sequence::try_new_typed(0u64, 1, Nullability::NonNullable, LARGE_SIZE).unwrap())] fn test_filter_sequence_conformance(#[case] array: SequenceArray) { - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/encodings/sequence/src/compute/take.rs b/encodings/sequence/src/compute/take.rs index 4b056d0ae7f..c59842224e2 100644 --- a/encodings/sequence/src/compute/take.rs +++ b/encodings/sequence/src/compute/take.rs @@ -107,6 +107,7 @@ mod test { use vortex_array::VortexSessionExecute; use vortex_array::array_session; use vortex_array::arrays::PrimitiveArray; + use vortex_array::compute::conformance::take::test_take_conformance; use vortex_array::dtype::Nullability; use crate::Sequence; @@ -161,9 +162,11 @@ mod test { Nullability::Nullable, 1000 ).unwrap())] - fn test_take_conformance(#[case] sequence: SequenceArray) { - use vortex_array::compute::conformance::take::test_take_conformance; - test_take_conformance(&sequence.into_array()); + fn sequence_take_conformance(#[case] sequence: SequenceArray) { + test_take_conformance( + &sequence.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs index 39c21805426..ab2b45eebdc 100644 --- a/encodings/sparse/src/compute/cast.rs +++ b/encodings/sparse/src/compute/cast.rs @@ -131,7 +131,7 @@ mod tests { Scalar::from(0u8) ).unwrap())] fn test_cast_sparse_conformance(#[case] array: SparseArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] diff --git a/encodings/sparse/src/compute/filter.rs b/encodings/sparse/src/compute/filter.rs index 91e67297fd2..c1818262b9b 100644 --- a/encodings/sparse/src/compute/filter.rs +++ b/encodings/sparse/src/compute/filter.rs @@ -147,6 +147,7 @@ mod tests { ) .unwrap() .into_array(), + &mut SESSION.create_execution_ctx(), ); let ten_fill_value = Scalar::from(10i32); @@ -159,6 +160,7 @@ mod tests { ) .unwrap() .into_array(), + &mut SESSION.create_execution_ctx(), ) } } diff --git a/encodings/sparse/src/compute/mod.rs b/encodings/sparse/src/compute/mod.rs index bdae87e560d..81dfa89080a 100644 --- a/encodings/sparse/src/compute/mod.rs +++ b/encodings/sparse/src/compute/mod.rs @@ -138,6 +138,7 @@ mod tests { ) .unwrap() .into_array(), + &mut SESSION.create_execution_ctx(), ); let ten_fill_value = Scalar::from(10i32); @@ -150,6 +151,7 @@ mod tests { ) .unwrap() .into_array(), + &mut SESSION.create_execution_ctx(), ) } diff --git a/encodings/sparse/src/compute/take.rs b/encodings/sparse/src/compute/take.rs index e260eb26e9e..3babc19e57b 100644 --- a/encodings/sparse/src/compute/take.rs +++ b/encodings/sparse/src/compute/take.rs @@ -64,6 +64,7 @@ mod test { use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::compute::conformance::take::test_take_conformance; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; @@ -227,7 +228,6 @@ mod test { Scalar::from(-1i32), ).unwrap())] fn test_take_sparse_conformance(#[case] sparse: SparseArray) { - use vortex_array::compute::conformance::take::test_take_conformance; - test_take_conformance(&sparse.into_array()); + test_take_conformance(&sparse.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 8ae38b56052..a37aab0c774 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -588,7 +588,7 @@ impl SparseData { // TODO(robert): Support other dtypes, only thing missing is getting most common value out of the array let primitive = array.clone().execute::(ctx)?; let (top_pvalue, _) = primitive - .top_value()? + .top_value(ctx)? .vortex_expect("Non empty or all null array"); Scalar::primitive_value(top_pvalue, top_pvalue.ptype(), array.dtype().nullability()) diff --git a/encodings/zigzag/src/compute/cast.rs b/encodings/zigzag/src/compute/cast.rs index c3fae84e346..94123e4498d 100644 --- a/encodings/zigzag/src/compute/cast.rs +++ b/encodings/zigzag/src/compute/cast.rs @@ -143,6 +143,6 @@ mod tests { #[case(zigzag_encode(PrimitiveArray::from_option_iter([Some(-5i16), None, Some(0), Some(5), None]).as_view()).unwrap())] #[case(zigzag_encode(PrimitiveArray::from_iter([i32::MIN, -1, 0, 1, i32::MAX]).as_view()).unwrap())] fn test_cast_zigzag_conformance(#[case] array: ZigZagArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 1da341989c5..e1c2f27f5d7 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -147,7 +147,7 @@ mod tests { let zigzag = zigzag_encode( PrimitiveArray::new(buffer![-189i32, -160, 1, 42, -73], Validity::AllValid).as_view(), )?; - test_filter_conformance(&zigzag.into_array()); + test_filter_conformance(&zigzag.into_array(), &mut SESSION.create_execution_ctx()); // Test with i64 values let zigzag = zigzag_encode( @@ -157,13 +157,13 @@ mod tests { ) .as_view(), )?; - test_filter_conformance(&zigzag.into_array()); + test_filter_conformance(&zigzag.into_array(), &mut SESSION.create_execution_ctx()); // Test with nullable values let array = PrimitiveArray::from_option_iter([Some(-10i16), None, Some(20), Some(-30), None]); let zigzag = zigzag_encode(array.as_view())?; - test_filter_conformance(&zigzag.into_array()); + test_filter_conformance(&zigzag.into_array(), &mut SESSION.create_execution_ctx()); Ok(()) } @@ -176,13 +176,13 @@ mod tests { PrimitiveArray::new(buffer![-100i32, 200, -300, 400, -500], Validity::AllValid) .as_view(), )?; - test_mask_conformance(&zigzag.into_array()); + test_mask_conformance(&zigzag.into_array(), &mut SESSION.create_execution_ctx()); // Test with i8 values let zigzag = zigzag_encode( PrimitiveArray::new(buffer![-127i8, 0, 127, -1, 1], Validity::AllValid).as_view(), )?; - test_mask_conformance(&zigzag.into_array()); + test_mask_conformance(&zigzag.into_array(), &mut SESSION.create_execution_ctx()); Ok(()) } @@ -198,7 +198,7 @@ mod tests { let mut ctx = SESSION.create_execution_ctx(); let array_primitive = array.execute::(&mut ctx)?; let zigzag = zigzag_encode(array_primitive.as_view())?; - test_take_conformance(&zigzag.into_array()); + test_take_conformance(&zigzag.into_array(), &mut ctx); Ok(()) } diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs index c5f15eba284..dba59198c88 100644 --- a/encodings/zstd/src/compute/cast.rs +++ b/encodings/zstd/src/compute/cast.rs @@ -200,6 +200,6 @@ mod tests { fn test_cast_zstd_conformance(#[case] values: PrimitiveArray) { let zstd = Zstd::from_primitive(&values, 0, 0, &mut SESSION.create_execution_ctx()).unwrap(); - test_cast_conformance(&zstd.into_array()); + test_cast_conformance(&zstd.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index f6cb9af586c..2edd82e4a84 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -20,11 +20,13 @@ use vortex_array::ArrayView; use vortex_array::EqMode; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; +use vortex_array::LEGACY_SESSION; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; use vortex_array::session::ArraySessionExt; +use vortex_array::validity::Validity; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; @@ -498,26 +500,20 @@ impl OperationsVTable for ZstdBuffers { // TODO(os): maybe we should not support scalar_at, it is really slow, and adding a cache // layer here is weird. Valid use of zstd buffers array would be by executing it first into // canonical - let inner_array = ZstdBuffers::decompress_and_build_inner( - &array.into_owned(), - &vortex_array::LEGACY_SESSION, - )?; + let inner_array = + ZstdBuffers::decompress_and_build_inner(&array.into_owned(), ctx.session())?; inner_array.execute_scalar(index, ctx) } } impl ValidityVTable for ZstdBuffers { - fn validity( - array: ArrayView<'_, ZstdBuffers>, - ) -> VortexResult { + fn validity(array: ArrayView<'_, ZstdBuffers>) -> VortexResult { if !array.dtype().is_nullable() { - return Ok(vortex_array::validity::Validity::NonNullable); + return Ok(Validity::NonNullable); } - let inner_array = ZstdBuffers::decompress_and_build_inner( - &array.into_owned(), - &vortex_array::LEGACY_SESSION, - )?; + let inner_array = + ZstdBuffers::decompress_and_build_inner(&array.into_owned(), &LEGACY_SESSION)?; inner_array.validity() } } diff --git a/fuzz/src/array/mod.rs b/fuzz/src/array/mod.rs index 7c1bbaad091..b6402cb1e1d 100644 --- a/fuzz/src/array/mod.rs +++ b/fuzz/src/array/mod.rs @@ -41,6 +41,7 @@ use strum::IntoEnumIterator; use tracing::debug; use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::NumericalAggregateOpts; @@ -553,25 +554,32 @@ fn random_action_from_list( /// Compress an array using the given strategy. #[cfg(feature = "zstd")] -pub fn compress_array(array: &ArrayRef, strategy: CompressorStrategy) -> ArrayRef { - let mut ctx = SESSION.create_execution_ctx(); +pub fn compress_array( + array: &ArrayRef, + strategy: CompressorStrategy, + ctx: &mut ExecutionCtx, +) -> ArrayRef { match strategy { CompressorStrategy::Default => BtrBlocksCompressor::default() - .compress(array, &mut ctx) + .compress(array, ctx) .vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test"), CompressorStrategy::Compact => BtrBlocksCompressorBuilder::default() .with_compact() .build() - .compress(array, &mut ctx) + .compress(array, ctx) .vortex_expect("Compact compress should succeed in fuzz test"), } } /// Compress an array using the given strategy (only Default). #[cfg(not(feature = "zstd"))] -pub fn compress_array(array: &ArrayRef, _strategy: CompressorStrategy) -> ArrayRef { +pub fn compress_array( + array: &ArrayRef, + _strategy: CompressorStrategy, + ctx: &mut ExecutionCtx, +) -> ArrayRef { BtrBlocksCompressor::default() - .compress(array, &mut SESSION.create_execution_ctx()) + .compress(array, ctx) .vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test") } @@ -602,14 +610,14 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { .clone() .execute::(&mut ctx) .vortex_expect("execute canonical should succeed in fuzz test"); - current_array = compress_array(&canonical.into_array(), strategy); - assert_array_eq(&expected.array(), ¤t_array, i)?; + current_array = compress_array(&canonical.into_array(), strategy, &mut ctx); + assert_array_eq(&expected.array(), ¤t_array, i, &mut ctx)?; } Action::Slice(range) => { current_array = current_array .slice(range) .vortex_expect("slice operation should succeed in fuzz test"); - assert_array_eq(&expected.array(), ¤t_array, i)?; + assert_array_eq(&expected.array(), ¤t_array, i, &mut ctx)?; } Action::Take(indices) => { if indices.is_empty() { @@ -618,14 +626,14 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { current_array = current_array .take(indices) .vortex_expect("take operation should succeed in fuzz test"); - assert_array_eq(&expected.array(), ¤t_array, i)?; + assert_array_eq(&expected.array(), ¤t_array, i, &mut ctx)?; } Action::SearchSorted(s, side) => { let mut sorted = sort_canonical_array(¤t_array, &mut ctx) .vortex_expect("sort_canonical_array should succeed in fuzz test"); if !current_array.is_canonical() { - sorted = compress_array(&sorted, CompressorStrategy::Default); + sorted = compress_array(&sorted, CompressorStrategy::Default, &mut ctx); } assert_search_sorted(sorted, s, side, expected.search(), i)?; } @@ -633,7 +641,7 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { current_array = current_array .filter(mask_val) .vortex_expect("filter operation should succeed in fuzz test"); - assert_array_eq(&expected.array(), ¤t_array, i)?; + assert_array_eq(&expected.array(), ¤t_array, i, &mut ctx)?; } Action::Compare(v, op) => { let compare_result = current_array @@ -642,7 +650,7 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { Operator::from(op), ) .vortex_expect("compare operation should succeed in fuzz test"); - if let Err(e) = assert_array_eq(&expected.array(), &compare_result, i) { + if let Err(e) = assert_array_eq(&expected.array(), &compare_result, i, &mut ctx) { vortex_panic!( "Failed to compare {}with {op} {v}\nError: {e}", current_array.display_tree() @@ -654,7 +662,7 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { let cast_result = current_array .cast(to.clone()) .vortex_expect("cast operation should succeed in fuzz test"); - if let Err(e) = assert_array_eq(&expected.array(), &cast_result, i) { + if let Err(e) = assert_array_eq(&expected.array(), &cast_result, i, &mut ctx) { vortex_panic!( "Failed to cast {} to dtype {to}\nError: {e}", current_array.display_tree() @@ -677,13 +685,13 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { current_array = current_array .fill_null(fill_value.clone()) .vortex_expect("fill_null operation should succeed in fuzz test"); - assert_array_eq(&expected.array(), ¤t_array, i)?; + assert_array_eq(&expected.array(), ¤t_array, i, &mut ctx)?; } Action::Mask(mask_val) => { current_array = current_array .mask(mask_val.into_array()) .vortex_expect("mask operation should succeed in fuzz test"); - assert_array_eq(&expected.array(), ¤t_array, i)?; + assert_array_eq(&expected.array(), ¤t_array, i, &mut ctx)?; } Action::ScalarAt(indices) => { let expected_scalars = expected.scalar_vec(); @@ -730,7 +738,12 @@ fn assert_search_sorted( /// Uses `all_non_distinct` for an efficient buffer-level comparison on the happy path. /// Falls back to element-wise scalar comparison only on mismatch to produce a detailed error. #[expect(clippy::result_large_err)] -pub fn assert_array_eq(lhs: &ArrayRef, rhs: &ArrayRef, step: usize) -> VortexFuzzResult<()> { +pub fn assert_array_eq( + lhs: &ArrayRef, + rhs: &ArrayRef, + step: usize, + ctx: &mut ExecutionCtx, +) -> VortexFuzzResult<()> { if lhs.dtype() != rhs.dtype() { return Err(VortexFuzzError::DTypeMismatch( lhs.clone(), @@ -751,10 +764,8 @@ pub fn assert_array_eq(lhs: &ArrayRef, rhs: &ArrayRef, step: usize) -> VortexFuz )); } - let mut ctx = SESSION.create_execution_ctx(); - // Fast path: buffer-level comparison. - let identical = all_non_distinct(lhs, rhs, &mut ctx) + let identical = all_non_distinct(lhs, rhs, ctx) .map_err(|e| VortexFuzzError::VortexError(e, Backtrace::capture()))?; if identical { return Ok(()); @@ -762,8 +773,8 @@ pub fn assert_array_eq(lhs: &ArrayRef, rhs: &ArrayRef, step: usize) -> VortexFuz // Slow path: find the first differing element for a detailed error message. for idx in 0..lhs.len() { - let l = lhs.execute_scalar(idx, &mut ctx).vortex_expect("scalar_at"); - let r = rhs.execute_scalar(idx, &mut ctx).vortex_expect("scalar_at"); + let l = lhs.execute_scalar(idx, ctx).vortex_expect("scalar_at"); + let r = rhs.execute_scalar(idx, ctx).vortex_expect("scalar_at"); if l != r { return Err(VortexFuzzError::ArrayNotEqual( diff --git a/fuzz/src/compress.rs b/fuzz/src/compress.rs index 1284a7be45e..9953de79b0a 100644 --- a/fuzz/src/compress.rs +++ b/fuzz/src/compress.rs @@ -6,6 +6,8 @@ //! This module generates arbitrary instances of compressed encodings (DictArray, etc.), //! then verifies that `to_canonical()` works and produces correct `len` and `dtype`. +use std::backtrace::Backtrace; + use arbitrary::Arbitrary; use arbitrary::Unstructured; use vortex_array::ArrayRef; @@ -17,6 +19,8 @@ use vortex_array::arrays::dict::ArbitraryDictArray; use vortex_runend::ArbitraryRunEndArray; use crate::SESSION; +use crate::error::VortexFuzzError; +use crate::error::VortexFuzzResult; /// Which compressed encoding to generate. #[derive(Debug, Clone, Copy)] @@ -64,10 +68,7 @@ impl<'a> Arbitrary<'a> for FuzzCompressRoundtrip { /// - `Ok(false)` - reject from corpus /// - `Err(_)` - a bug was found #[expect(clippy::result_large_err)] -pub fn run_compress_roundtrip(fuzz: FuzzCompressRoundtrip) -> crate::error::VortexFuzzResult { - use crate::error::Backtrace; - use crate::error::VortexFuzzError; - +pub fn run_compress_roundtrip(fuzz: FuzzCompressRoundtrip) -> VortexFuzzResult { let FuzzCompressRoundtrip { array } = fuzz; // Store original properties diff --git a/vortex-array/benches/dict_unreferenced_mask.rs b/vortex-array/benches/dict_unreferenced_mask.rs index 1ec143682af..ec8653d6688 100644 --- a/vortex-array/benches/dict_unreferenced_mask.rs +++ b/vortex-array/benches/dict_unreferenced_mask.rs @@ -3,14 +3,21 @@ #![expect(clippy::unwrap_used)] +use std::sync::LazyLock; + use divan::Bencher; use rand::RngExt; use rand::SeedableRng; use rand::rngs::StdRng; use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::array_session; use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArrayExt; +use vortex_session::VortexSession; + +static SESSION: LazyLock = LazyLock::new(array_session); fn main() { divan::main(); @@ -41,8 +48,8 @@ fn bench_many_codes_few_values(bencher: Bencher, num_values: i32) { let array = DictArray::try_new(codes, values).unwrap(); bencher - .with_inputs(|| &array) - .bench_refs(|array| array.compute_referenced_values_mask(false).unwrap()); + .with_inputs(|| (&array, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| array.compute_referenced_values_mask(false, ctx).unwrap()); } /// Benchmark with many nulls in the codes array. @@ -73,8 +80,8 @@ fn bench_many_nulls(bencher: Bencher, fraction_valid: f64) { let array = DictArray::try_new(codes, values).unwrap(); bencher - .with_inputs(|| &array) - .bench_refs(|array| array.compute_referenced_values_mask(false).unwrap()); + .with_inputs(|| (&array, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| array.compute_referenced_values_mask(false, ctx).unwrap()); } /// Benchmark with sparse code coverage (many unreferenced values). @@ -107,6 +114,6 @@ fn bench_sparse_coverage(bencher: Bencher, fraction_coverage: f64) { let array = DictArray::try_new(codes, values).unwrap(); bencher - .with_inputs(|| &array) - .bench_refs(|array| array.compute_referenced_values_mask(false).unwrap()); + .with_inputs(|| (&array, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| array.compute_referenced_values_mask(false, ctx).unwrap()); } diff --git a/vortex-array/benches/filter_bool.rs b/vortex-array/benches/filter_bool.rs index cd31805bb77..69f7f1d8ee6 100644 --- a/vortex-array/benches/filter_bool.rs +++ b/vortex-array/benches/filter_bool.rs @@ -18,6 +18,7 @@ use rand::prelude::*; use rand_distr::Zipf; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; +use vortex_array::array_session; use vortex_array::arrays::BoolArray; use vortex_buffer::BitBuffer; use vortex_mask::Mask; @@ -27,7 +28,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); +static SESSION: LazyLock = LazyLock::new(array_session); const SIZES: &[usize] = &[1_000, 10_000, 100_000, 250_000]; const DENSITY_SWEEP_SIZE: usize = 100_000; diff --git a/vortex-array/benches/listview_builder_extend.rs b/vortex-array/benches/listview_builder_extend.rs index 75487564ea8..99f35e9fe82 100644 --- a/vortex-array/benches/listview_builder_extend.rs +++ b/vortex-array/benches/listview_builder_extend.rs @@ -3,14 +3,16 @@ #![expect(clippy::cast_possible_truncation)] #![expect(clippy::cast_possible_wrap)] +#![expect(clippy::unwrap_used)] use std::sync::Arc; use divan::Bencher; use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::array_session; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::builders::ArrayBuilder; use vortex_array::builders::ListViewBuilder; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability::NonNullable; @@ -63,13 +65,14 @@ fn extend_from_array_zctl(bencher: Bencher, (num_lists, list_size): (usize, usiz let source = source.into_array(); bencher.with_inputs(|| &source).bench_refs(|source| { - let mut builder = ListViewBuilder::::with_capacity( + let mut ctx = array_session().create_execution_ctx(); + let mut builder = ListViewBuilder::::with_capacity( Arc::new(DType::Primitive(I32, NonNullable)), NonNullable, num_lists * list_size, num_lists, ); - builder.extend_from_array(source); + source.append_to_builder(&mut builder, &mut ctx).unwrap(); divan::black_box(builder.finish_into_listview()) }); } @@ -85,13 +88,14 @@ fn extend_from_array_non_zctl_overlapping( let source = source.into_array(); bencher.with_inputs(|| &source).bench_refs(|source| { - let mut builder = ListViewBuilder::::with_capacity( + let mut ctx = array_session().create_execution_ctx(); + let mut builder = ListViewBuilder::::with_capacity( Arc::new(DType::Primitive(I32, NonNullable)), Nullable, num_lists * list_size, num_lists, ); - builder.extend_from_array(source); + source.append_to_builder(&mut builder, &mut ctx).unwrap(); divan::black_box(builder.finish_into_listview()) }); } diff --git a/vortex-array/benches/take_patches.rs b/vortex-array/benches/take_patches.rs index 5dc0c04c325..a52c6136910 100644 --- a/vortex-array/benches/take_patches.rs +++ b/vortex-array/benches/take_patches.rs @@ -12,11 +12,11 @@ use rand::SeedableRng; use rand::rngs::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical as _; use vortex_array::VortexSessionExecute; +use vortex_array::arrays::PrimitiveArray; use vortex_array::patches::Patches; use vortex_buffer::Buffer; +use vortex_error::VortexExpect; use vortex_session::VortexSession; fn main() { @@ -54,8 +54,10 @@ fn take_search(bencher: Bencher, (patches_sparsity, index_multiple): (f64, f64)) bencher .with_inputs(|| (&patches, &indices, SESSION.create_execution_ctx())) .bench_refs(|(patches, indices, ctx)| { - #[expect(deprecated)] - let prim = indices.to_primitive(); + let prim = indices + .clone() + .execute::(ctx) + .vortex_expect("operation should succeed in benchmark"); patches.take_search(prim, false, ctx) }); } @@ -73,8 +75,10 @@ fn take_search_chunked(bencher: Bencher, (patches_sparsity, index_multiple): (f6 bencher .with_inputs(|| (&patches, &indices, SESSION.create_execution_ctx())) .bench_refs(|(patches, indices, ctx)| { - #[expect(deprecated)] - let prim = indices.to_primitive(); + let prim = indices + .clone() + .execute::(ctx) + .vortex_expect("operation should succeed in benchmark"); patches.take_search(prim, false, ctx) }); } @@ -92,8 +96,10 @@ fn take_map(bencher: Bencher, (patches_sparsity, index_multiple): (f64, f64)) { bencher .with_inputs(|| (&patches, &indices, SESSION.create_execution_ctx())) .bench_refs(|(patches, indices, ctx)| { - #[expect(deprecated)] - let prim = indices.to_primitive(); + let prim = indices + .clone() + .execute::(ctx) + .vortex_expect("operation should succeed in benchmark"); patches.take_map(prim, false, ctx) }); } @@ -105,15 +111,7 @@ fn fixture(len: usize, sparsity: f64, rng: &mut StdRng) -> Patches { .collect::>(); let sparse_len = indices.len(); let values = Buffer::from_iter((0..sparse_len).map(|x| x as u64)).into_array(); - Patches::new( - len, - 0, - indices.into_array(), - values, - // TODO(0ax1): handle chunk offsets - None, - ) - .unwrap() + Patches::new(len, 0, indices.into_array(), values, None).unwrap() } fn fixture_with_chunk_offsets(len: usize, sparsity: f64, rng: &mut StdRng) -> Patches { diff --git a/vortex-array/benches/varbinview_compact.rs b/vortex-array/benches/varbinview_compact.rs index 99243ce3606..d68420a264f 100644 --- a/vortex-array/benches/varbinview_compact.rs +++ b/vortex-array/benches/varbinview_compact.rs @@ -1,20 +1,23 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::sync::LazyLock; + use divan::Bencher; use rand::RngExt; use rand::SeedableRng; use rand::rngs::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical as _; +use vortex_array::VortexSessionExecute; +use vortex_array::array_session; use vortex_array::arrays::VarBinViewArray; use vortex_array::builders::VarBinViewBuilder; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_buffer::Buffer; use vortex_error::VortexExpect; +use vortex_session::VortexSession; fn main() { divan::main(); @@ -28,6 +31,8 @@ const ARGS: &[(usize, usize)] = &[ (1 << 14, 90), ]; +static SESSION: LazyLock = LazyLock::new(array_session); + #[divan::bench(args = ARGS)] fn compact(bencher: Bencher, args: (usize, usize)) { compact_impl(bencher, args); @@ -46,31 +51,39 @@ fn compact_impl(bencher: Bencher, (output_size, utilization_pct): (usize, usize) .into_array() .take(indices) .vortex_expect("operation should succeed in benchmark"); - #[expect(deprecated)] - let array = taken.to_varbinview(); - - bencher.with_inputs(|| &array).bench_refs(|array| { - array - .compact_buffers() - .vortex_expect("operation should succeed in benchmark") - }) + let mut ctx = SESSION.create_execution_ctx(); + let array = taken + .execute::(&mut ctx) + .vortex_expect("operation should succeed in benchmark"); + + bencher + .with_inputs(|| (&array, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| { + array + .compact_buffers(ctx) + .vortex_expect("operation should succeed in benchmark") + }) } fn compact_sliced_impl(bencher: Bencher, (output_size, utilization_pct): (usize, usize)) { + let mut ctx = SESSION.create_execution_ctx(); let base_size = (output_size * 100) / utilization_pct; let base_array = build_varbinview_fixture(base_size); let sliced = base_array .into_array() .slice(0..output_size) .vortex_expect("slice should succeed"); - #[expect(deprecated)] - let array = sliced.to_varbinview(); - - bencher.with_inputs(|| &array).bench_refs(|array| { - array - .compact_buffers() - .vortex_expect("operation should succeed in benchmark") - }) + let array = sliced + .execute::(&mut ctx) + .vortex_expect("operation should succeed in benchmark"); + + bencher + .with_inputs(|| (&array, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| { + array + .compact_buffers(ctx) + .vortex_expect("operation should succeed in benchmark") + }) } /// Creates a base VarBinViewArray with mix of inlined and outlined strings. diff --git a/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs b/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs index 0b07c7cf98b..067f9a010a1 100644 --- a/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs @@ -311,6 +311,7 @@ pub(crate) fn packed_bit_buffer_size_in_bytes(len: usize) -> VortexResult { #[cfg(test)] mod tests { use vortex_buffer::buffer; + use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; @@ -353,9 +354,12 @@ mod tests { fn materialized_uncompressed_size_in_bytes(array: &ArrayRef) -> u64 { let mut builder = builder_with_capacity(array.dtype(), array.len()); - unsafe { - builder.extend_from_array_unchecked(array); - } + array + .append_to_builder( + builder.as_mut(), + &mut array_session().create_execution_ctx(), + ) + .vortex_expect("appended"); builder.finish().nbytes() } diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 2c76945c726..a0943864738 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -155,7 +155,7 @@ pub(crate) trait DynArrayData: 'static + private::Sealed + Send + Sync + Debug { /// Execute the array by taking a single encoding-specific execution step. /// /// This is the checked entry point. If the encoding reports - /// [`ExecutionStep::Done`](crate::ExecutionStep::Done), implementations must validate that the + /// [`ExecutionStep::Done`](ExecutionStep::Done), implementations must validate that the /// returned array preserves this array's logical `len` and `dtype`, and must transfer this /// array's statistics to the returned array. fn execute(&self, this: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult; diff --git a/vortex-array/src/array/vtable/mod.rs b/vortex-array/src/array/vtable/mod.rs index ffa8e9dd854..2613eec2c61 100644 --- a/vortex-array/src/array/vtable/mod.rs +++ b/vortex-array/src/array/vtable/mod.rs @@ -179,8 +179,7 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { .clone() .execute::(ctx)? .into_array(); - builder.extend_from_array(&canonical); - Ok(()) + canonical.append_to_builder(builder, ctx) } /// Returns the name of the slot at the given index. diff --git a/vortex-array/src/arrays/arbitrary.rs b/vortex-array/src/arrays/arbitrary.rs index 035c23fd4db..5d5d9f60bf9 100644 --- a/vortex-array/src/arrays/arbitrary.rs +++ b/vortex-array/src/arrays/arbitrary.rs @@ -15,11 +15,10 @@ use vortex_error::VortexExpect; use crate::ArrayRef; use crate::IntoArray; -#[expect(deprecated)] -use crate::ToCanonical as _; use crate::arrays::BoolArray; use crate::arrays::ChunkedArray; use crate::arrays::NullArray; +use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::arrays::VarBinArray; @@ -130,9 +129,8 @@ fn random_array_chunk( PType::I32 => random_primitive::(u, *n, chunk_len), PType::I64 => random_primitive::(u, *n, chunk_len), PType::F16 => { - #[expect(deprecated)] let prim = random_primitive::(u, *n, chunk_len)? - .to_primitive() + .as_::() .reinterpret_cast(PType::F16) .into_array(); Ok(prim) diff --git a/vortex-array/src/arrays/bool/compute/cast.rs b/vortex-array/src/arrays/bool/compute/cast.rs index fe9332346ca..088a0707ccf 100644 --- a/vortex-array/src/arrays/bool/compute/cast.rs +++ b/vortex-array/src/arrays/bool/compute/cast.rs @@ -100,6 +100,6 @@ mod tests { #[case(BoolArray::from_iter(vec![true]))] #[case(BoolArray::from_iter(vec![false, false]))] fn test_cast_bool_conformance(#[case] array: BoolArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/vortex-array/src/arrays/bool/compute/fill_null.rs b/vortex-array/src/arrays/bool/compute/fill_null.rs index 59b9ca0b717..5fa18299106 100644 --- a/vortex-array/src/arrays/bool/compute/fill_null.rs +++ b/vortex-array/src/arrays/bool/compute/fill_null.rs @@ -48,11 +48,11 @@ mod tests { use vortex_buffer::bitbuffer; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -62,16 +62,17 @@ mod tests { #[case(true, bitbuffer![true, true, false, true])] #[case(false, bitbuffer![true, false, false, false])] fn bool_fill_null(#[case] fill_value: bool, #[case] expected: BitBuffer) { + let mut ctx = array_session().create_execution_ctx(); let bool_array = BoolArray::new( BitBuffer::from_iter([true, true, false, false]), Validity::from_iter([true, false, true, false]), ); - #[expect(deprecated)] let non_null_array = bool_array .into_array() .fill_null(Scalar::from(fill_value)) .unwrap() - .to_bool(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(non_null_array.to_bit_buffer(), expected); assert_eq!( non_null_array.dtype(), diff --git a/vortex-array/src/arrays/bool/compute/filter.rs b/vortex-array/src/arrays/bool/compute/filter.rs index f463f27f437..2759c6e3077 100644 --- a/vortex-array/src/arrays/bool/compute/filter.rs +++ b/vortex-array/src/arrays/bool/compute/filter.rs @@ -366,7 +366,10 @@ mod tests { #[case(BoolArray::from_iter((0..100).map(|i| i % 2 == 0)))] #[case(BoolArray::from_iter((0..1024).map(|i| i % 3 != 0)))] fn test_filter_bool_conformance(#[case] array: BoolArray) { - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[cfg(target_arch = "x86_64")] diff --git a/vortex-array/src/arrays/bool/compute/mask.rs b/vortex-array/src/arrays/bool/compute/mask.rs index 6fd278f2d3d..783c933fe5a 100644 --- a/vortex-array/src/arrays/bool/compute/mask.rs +++ b/vortex-array/src/arrays/bool/compute/mask.rs @@ -29,6 +29,8 @@ mod test { use rstest::rstest; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::BoolArray; use crate::compute::conformance::mask::test_mask_conformance; @@ -39,6 +41,9 @@ mod test { #[case(BoolArray::from_iter([false, false]))] #[case(BoolArray::from_iter((0..100).map(|i| i % 2 == 0)))] fn test_mask_bool_conformance(#[case] array: BoolArray) { - test_mask_conformance(&array.into_array()); + test_mask_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/bool/compute/take.rs b/vortex-array/src/arrays/bool/compute/take.rs index abff3527bfe..8469c7e25c2 100644 --- a/vortex-array/src/arrays/bool/compute/take.rs +++ b/vortex-array/src/arrays/bool/compute/take.rs @@ -88,8 +88,6 @@ mod test { use vortex_buffer::buffer; use crate::IntoArray as _; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::BoolArray; @@ -110,11 +108,11 @@ mod test { Some(false), ]); - #[expect(deprecated)] let b = reference .take(buffer![0, 3, 4].into_array()) .unwrap() - .to_bool(); + .execute::(&mut ctx) + .unwrap(); assert_eq!( b.to_bit_buffer(), BoolArray::from_iter([Some(false), None, Some(false)]).to_bit_buffer() @@ -190,6 +188,9 @@ mod test { #[case(BoolArray::from_iter([true, false]))] #[case(BoolArray::from_iter([true]))] fn test_take_bool_conformance(#[case] array: BoolArray) { - test_take_conformance(&array.into_array()); + test_take_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index 348a1f5f69a..e27f5c7464d 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -180,12 +180,10 @@ impl VTable for Bool { builder: &mut dyn ArrayBuilder, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - if let Some(builder) = builder.as_any_mut().downcast_mut::() { - return builder.append_bool_array(&array.into_owned(), ctx); - } - - builder.extend_from_array(array.as_ref()); - Ok(()) + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + vortex_bail!("append_to_builder for Bool requires a BoolBuilder"); + }; + builder.append_bool_array(&array.into_owned(), ctx) } fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { diff --git a/vortex-array/src/arrays/bool/vtable/operations.rs b/vortex-array/src/arrays/bool/vtable/operations.rs index 9bae2b9e19c..c29ab20331b 100644 --- a/vortex-array/src/arrays/bool/vtable/operations.rs +++ b/vortex-array/src/arrays/bool/vtable/operations.rs @@ -28,8 +28,6 @@ mod tests { use std::iter; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::BoolArray; @@ -38,9 +36,14 @@ mod tests { #[test] fn test_slice_hundred_elements() { + let mut ctx = array_session().create_execution_ctx(); let arr = BoolArray::from_iter(iter::repeat_n(Some(true), 100)); - #[expect(deprecated)] - let sliced_arr = arr.into_array().slice(8..16).unwrap().to_bool(); + let sliced_arr = arr + .into_array() + .slice(8..16) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!(sliced_arr.len(), 8); assert_eq!(sliced_arr.to_bit_buffer().len(), 8); assert_eq!(sliced_arr.to_bit_buffer().offset(), 0); @@ -50,8 +53,12 @@ mod tests { fn test_slice() { let mut ctx = array_session().create_execution_ctx(); let arr = BoolArray::from_iter([Some(true), Some(true), None, Some(false), None]); - #[expect(deprecated)] - let sliced_arr = arr.into_array().slice(1..4).unwrap().to_bool(); + let sliced_arr = arr + .into_array() + .slice(1..4) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( sliced_arr, diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 54544ba1b4d..3963d6ba950 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -18,6 +18,8 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::ArraySlots; +use crate::Canonical; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayParts; @@ -194,7 +196,12 @@ impl Array { Ok(unsafe { Self::new_unchecked(chunks, dtype) }) } - pub fn rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> VortexResult { + pub fn rechunk( + &self, + target_bytesize: u64, + target_rowsize: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { let mut new_chunks = Vec::new(); let mut chunks_to_combine = Vec::new(); let mut new_chunk_n_bytes = 0; @@ -207,12 +214,11 @@ impl Array { || new_chunk_n_elements + n_elements > target_rowsize) && !chunks_to_combine.is_empty() { - #[expect(deprecated)] let canonical = unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } .into_array() - .to_canonical()? + .execute::(ctx)? .into_array(); new_chunks.push(canonical); @@ -231,11 +237,10 @@ impl Array { } if !chunks_to_combine.is_empty() { - #[expect(deprecated)] let canonical = unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } .into_array() - .to_canonical()? + .execute::(ctx)? .into_array(); new_chunks.push(canonical); } @@ -298,7 +303,7 @@ mod test { ) .unwrap(); - let rechunked = chunked.rechunk(1 << 16, 1 << 16).unwrap(); + let rechunked = chunked.rechunk(1 << 16, 1 << 16, &mut ctx).unwrap(); assert_arrays_eq!(chunked, rechunked, &mut ctx); } @@ -312,7 +317,7 @@ mod test { ) .unwrap(); - let rechunked = chunked.rechunk(1 << 16, 1 << 16).unwrap(); + let rechunked = chunked.rechunk(1 << 16, 1 << 16, &mut ctx).unwrap(); assert_eq!(rechunked.nchunks(), 1); assert_arrays_eq!(chunked, rechunked, &mut ctx); @@ -330,7 +335,7 @@ mod test { ) .unwrap(); - let rechunked = chunked.rechunk(1 << 16, 5).unwrap(); + let rechunked = chunked.rechunk(1 << 16, 5, &mut ctx).unwrap(); assert_eq!(rechunked.nchunks(), 2); assert!(rechunked.iter_chunks().all(|c| c.len() < 5)); @@ -352,7 +357,7 @@ mod test { ) .unwrap(); - let rechunked = chunked.rechunk(1 << 16, 5).unwrap(); + let rechunked = chunked.rechunk(1 << 16, 5, &mut ctx).unwrap(); // greedy so should be: [0, 1, 2] [42, 42, 42, 42, 42, 42] [4, 5, 6, 7] [8, 9] assert_eq!(rechunked.nchunks(), 4); @@ -361,6 +366,7 @@ mod test { #[test] fn test_empty_chunks_all_valid() -> VortexResult<()> { + let mut ctx = array_session().create_execution_ctx(); // Create chunks where some are empty but all non-empty chunks have all valid values let chunks = vec![ PrimitiveArray::new(buffer![1u64, 2, 3], Validity::AllValid).into_array(), @@ -373,18 +379,15 @@ mod test { ChunkedArray::try_new(chunks, DType::Primitive(PType::U64, Nullability::Nullable))?; // Should be all_valid since all non-empty chunks are all_valid - assert!(chunked.all_valid(&mut array_session().create_execution_ctx())?); - assert!( - !chunked - .into_array() - .all_invalid(&mut array_session().create_execution_ctx())? - ); + assert!(chunked.all_valid(&mut ctx)?); + assert!(!chunked.into_array().all_invalid(&mut ctx)?); Ok(()) } #[test] fn test_empty_chunks_all_invalid() -> VortexResult<()> { + let mut ctx = array_session().create_execution_ctx(); // Create chunks where some are empty but all non-empty chunks have all invalid values let chunks = vec![ PrimitiveArray::new(buffer![1u64, 2], Validity::AllInvalid).into_array(), @@ -397,18 +400,15 @@ mod test { ChunkedArray::try_new(chunks, DType::Primitive(PType::U64, Nullability::Nullable))?; // Should be all_invalid since all non-empty chunks are all_invalid - assert!(!chunked.all_valid(&mut array_session().create_execution_ctx())?); - assert!( - chunked - .into_array() - .all_invalid(&mut array_session().create_execution_ctx())? - ); + assert!(!chunked.all_valid(&mut ctx)?); + assert!(chunked.into_array().all_invalid(&mut ctx)?); Ok(()) } #[test] fn test_empty_chunks_mixed_validity() -> VortexResult<()> { + let mut ctx = array_session().create_execution_ctx(); // Create chunks with mixed validity including empty chunks let chunks = vec![ PrimitiveArray::new(buffer![1u64, 2], Validity::AllValid).into_array(), @@ -421,12 +421,8 @@ mod test { ChunkedArray::try_new(chunks, DType::Primitive(PType::U64, Nullability::Nullable))?; // Should be neither all_valid nor all_invalid - assert!(!chunked.all_valid(&mut array_session().create_execution_ctx())?); - assert!( - !chunked - .into_array() - .all_invalid(&mut array_session().create_execution_ctx())? - ); + assert!(!chunked.all_valid(&mut ctx)?); + assert!(!chunked.into_array().all_invalid(&mut ctx)?); Ok(()) } diff --git a/vortex-array/src/arrays/chunked/compute/cast.rs b/vortex-array/src/arrays/chunked/compute/cast.rs index 80c6848b973..25e312de748 100644 --- a/vortex-array/src/arrays/chunked/compute/cast.rs +++ b/vortex-array/src/arrays/chunked/compute/cast.rs @@ -97,6 +97,6 @@ mod test { DType::Primitive(PType::U8, Nullability::NonNullable) ).unwrap().into_array())] fn test_cast_chunked_conformance(#[case] array: crate::ArrayRef) { - test_cast_conformance(&array); + test_cast_conformance(&array, &mut array_session().create_execution_ctx()); } } diff --git a/vortex-array/src/arrays/chunked/compute/filter.rs b/vortex-array/src/arrays/chunked/compute/filter.rs index 0e32543761d..5f50ce18387 100644 --- a/vortex-array/src/arrays/chunked/compute/filter.rs +++ b/vortex-array/src/arrays/chunked/compute/filter.rs @@ -203,6 +203,8 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; use crate::compute::conformance::filter::test_filter_conformance; @@ -275,6 +277,9 @@ mod test { DType::Primitive(PType::I64, Nullability::NonNullable), ).unwrap())] fn test_filter_chunked_conformance(#[case] chunked: ChunkedArray) { - test_filter_conformance(&chunked.into_array()); + test_filter_conformance( + &chunked.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/chunked/compute/mask.rs b/vortex-array/src/arrays/chunked/compute/mask.rs index bec7b5e83d0..01170a232bb 100644 --- a/vortex-array/src/arrays/chunked/compute/mask.rs +++ b/vortex-array/src/arrays/chunked/compute/mask.rs @@ -45,6 +45,8 @@ mod test { use vortex_buffer::buffer; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; use crate::compute::conformance::mask::test_mask_conformance; @@ -80,6 +82,9 @@ mod test { DType::Primitive(PType::F32, Nullability::NonNullable), ).unwrap())] fn test_mask_chunked_conformance(#[case] chunked: ChunkedArray) { - test_mask_conformance(&chunked.into_array()); + test_mask_conformance( + &chunked.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/chunked/compute/take.rs b/vortex-array/src/arrays/chunked/compute/take.rs index 34f443f1ce1..1b867981236 100644 --- a/vortex-array/src/arrays/chunked/compute/take.rs +++ b/vortex-array/src/arrays/chunked/compute/take.rs @@ -126,8 +126,6 @@ mod test { use vortex_error::VortexResult; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::BoolArray; @@ -268,6 +266,7 @@ mod test { #[test] fn test_take_shuffled_large() -> VortexResult<()> { + let mut ctx = array_session().create_execution_ctx(); let nchunks: i32 = 100; let chunk_len: i32 = 1_000; let total = nchunks * chunk_len; @@ -297,8 +296,7 @@ mod test { let result = arr.take(indices_arr.into_array())?; // Verify every element. - #[expect(deprecated)] - let result = result.to_primitive(); + let result = result.execute::(&mut ctx)?; let result_vals = result.as_slice::(); for (pos, &idx) in indices.iter().enumerate() { assert_eq!( @@ -353,14 +351,20 @@ mod test { .clone(), ) .unwrap(); - test_take_conformance(&arr.into_array()); + test_take_conformance( + &arr.into_array(), + &mut array_session().create_execution_ctx(), + ); // Test with nullable chunked array let a = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]); let b = PrimitiveArray::from_option_iter([Some(4i32), Some(5)]); let dtype = a.dtype().clone(); let arr = ChunkedArray::try_new(vec![a.into_array(), b.into_array()], dtype).unwrap(); - test_take_conformance(&arr.into_array()); + test_take_conformance( + &arr.into_array(), + &mut array_session().create_execution_ctx(), + ); // Test with multiple identical chunks let chunk = buffer![10i32, 20, 30, 40, 50].into_array(); @@ -369,6 +373,9 @@ mod test { chunk.dtype().clone(), ) .unwrap(); - test_take_conformance(&arr.into_array()); + test_take_conformance( + &arr.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/chunked/compute/zip.rs b/vortex-array/src/arrays/chunked/compute/zip.rs index 4037fd1dc49..dc95c194173 100644 --- a/vortex-array/src/arrays/chunked/compute/zip.rs +++ b/vortex-array/src/arrays/chunked/compute/zip.rs @@ -51,12 +51,11 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; + use crate::arrays::PrimitiveArray; use crate::arrays::chunked::ChunkedArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; @@ -65,6 +64,7 @@ mod tests { #[test] fn test_chunked_zip_aligns_across_boundaries() { + let mut ctx = array_session().create_execution_ctx(); let if_true = ChunkedArray::try_new( vec![ buffer![1i32, 2].into_array(), @@ -103,8 +103,7 @@ mod tests { assert_eq!(zipped.nchunks(), 4); let mut values: Vec = Vec::new(); for chunk in zipped.chunks() { - #[expect(deprecated)] - let primitive = chunk.to_primitive(); + let primitive = chunk.execute::(&mut ctx).unwrap(); values.extend_from_slice(primitive.as_slice::()); } assert_eq!(values, vec![1, 11, 3, 13, 5]); diff --git a/vortex-array/src/arrays/chunked/paired_chunks.rs b/vortex-array/src/arrays/chunked/paired_chunks.rs index 2145c88dbbd..c4a73a09518 100644 --- a/vortex-array/src/arrays/chunked/paired_chunks.rs +++ b/vortex-array/src/arrays/chunked/paired_chunks.rs @@ -121,9 +121,10 @@ mod tests { use vortex_error::VortexResult; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::ChunkedArray; + use crate::arrays::PrimitiveArray; use crate::arrays::chunked::paired_chunks::PairedChunksExt; use crate::dtype::DType; use crate::dtype::Nullability; @@ -138,13 +139,20 @@ mod tests { left: &ChunkedArray, right: &ChunkedArray, ) -> VortexResult, Vec, std::ops::Range)>> { + let mut ctx = array_session().create_execution_ctx(); let mut result = Vec::new(); for pair in left.paired_chunks(right) { let pair = pair?; - #[expect(deprecated)] - let l: Vec = pair.left.to_primitive().as_slice::().to_vec(); - #[expect(deprecated)] - let r: Vec = pair.right.to_primitive().as_slice::().to_vec(); + let l: Vec = pair + .left + .execute::(&mut ctx)? + .as_slice::() + .to_vec(); + let r: Vec = pair + .right + .execute::(&mut ctx)? + .as_slice::() + .to_vec(); result.push((l, r, pair.pos)); } Ok(result) diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index 24942300d52..8f8c289bfca 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -12,9 +12,11 @@ use vortex_session::VortexSession; use crate::Canonical; use crate::IntoArray; use crate::VortexSessionExecute; +use crate::array_session; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; use crate::arrays::ListArray; +use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::arrays::VarBinViewArray; @@ -23,8 +25,6 @@ use crate::arrays::dict_test::gen_dict_primitive_chunks; use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; use crate::builders::builder_with_capacity; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; @@ -32,7 +32,7 @@ use crate::dtype::PType::I32; use crate::executor::execute_into_builder; use crate::validity::Validity; -static SESSION: LazyLock = LazyLock::new(crate::array_session); +static SESSION: LazyLock = LazyLock::new(array_session); fn chunked_array() -> ChunkedArray { ChunkedArray::try_new( @@ -369,33 +369,14 @@ pub fn pack_nested_structs() -> VortexResult<()> { dtype, )? .into_array(); - #[expect(deprecated)] - let canonical_struct = chunked.to_struct(); - #[expect(deprecated)] - let canonical_varbin = canonical_struct.unmasked_fields()[0].to_varbinview(); - #[expect(deprecated)] - let original_varbin = struct_array.unmasked_fields()[0].to_varbinview(); - let orig_mask = original_varbin - .validity()? - .execute_mask(original_varbin.len(), &mut ctx)?; - let orig_values = (0..original_varbin.len()) - .map(|i| { - orig_mask - .value(i) - .then(|| original_varbin.bytes_at(i).to_vec()) - }) - .collect::>(); - let canon_mask = canonical_varbin - .validity()? - .execute_mask(canonical_varbin.len(), &mut ctx)?; - let canon_values = (0..canonical_varbin.len()) - .map(|i| { - canon_mask - .value(i) - .then(|| canonical_varbin.bytes_at(i).to_vec()) - }) - .collect::>(); - assert_eq!(orig_values, canon_values); + let canonical_struct = chunked.execute::(&mut ctx)?; + let canonical_varbin = canonical_struct.unmasked_fields()[0] + .clone() + .execute::(&mut ctx)?; + let original_varbin = struct_array.unmasked_fields()[0] + .clone() + .execute::(&mut ctx)?; + assert_arrays_eq!(original_varbin, canonical_varbin, &mut ctx); Ok(()) } @@ -424,8 +405,12 @@ pub fn pack_nested_lists() { ), ); - #[expect(deprecated)] - let canon_values = chunked_list.unwrap().as_array().to_listview(); + let canon_values = chunked_list + .unwrap() + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!( l1.execute_scalar(0, &mut ctx).unwrap(), diff --git a/vortex-array/src/arrays/chunked/vtable/canonical.rs b/vortex-array/src/arrays/chunked/vtable/canonical.rs index 995d90bc0aa..12c3bbcd9e4 100644 --- a/vortex-array/src/arrays/chunked/vtable/canonical.rs +++ b/vortex-array/src/arrays/chunked/vtable/canonical.rs @@ -73,7 +73,7 @@ pub(super) fn _canonicalize( _ => { let mut builder = builder_with_capacity_in(ctx.allocator(), array.dtype(), array.len()); array.array().append_to_builder(builder.as_mut(), ctx)?; - builder.finish_into_canonical() + builder.finish_into_canonical(ctx) } }) } diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 1054dd21d2d..f7d1668c85a 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -22,13 +22,13 @@ use crate::EqMode; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; -#[expect(deprecated)] -use crate::ToCanonical as _; +use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; +use crate::arrays::PrimitiveArray; use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::chunked::ChunkedData; use crate::arrays::chunked::array::CHUNK_OFFSETS_SLOT; @@ -170,7 +170,7 @@ impl VTable for Chunked { metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, + session: &VortexSession, ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( @@ -188,8 +188,11 @@ impl VTable for Chunked { &DType::Primitive(PType::U64, Nullability::NonNullable), nchunks + 1, )?; - #[expect(deprecated)] - let chunk_offsets_buf = chunk_offsets.to_primitive().to_buffer::(); + let mut ctx = session.create_execution_ctx(); + let chunk_offsets_buf = chunk_offsets + .clone() + .execute::(&mut ctx)? + .to_buffer::(); let chunk_offsets_usize = chunk_offsets_buf .iter() .copied() diff --git a/vortex-array/src/arrays/constant/arbitrary.rs b/vortex-array/src/arrays/constant/arbitrary.rs index c40bb5e82b7..37bde6414dd 100644 --- a/vortex-array/src/arrays/constant/arbitrary.rs +++ b/vortex-array/src/arrays/constant/arbitrary.rs @@ -16,15 +16,15 @@ pub struct ArbitraryConstantArray(pub ConstantArray); impl<'a> Arbitrary<'a> for ArbitraryConstantArray { fn arbitrary(u: &mut Unstructured<'a>) -> Result { let dtype: DType = u.arbitrary()?; - Self::with_dtype(u, &dtype, None) + Self::with_dtype(u, &dtype) } } impl ArbitraryConstantArray { /// Generate an arbitrary ConstantArray with the given dtype. - pub fn with_dtype(u: &mut Unstructured, dtype: &DType, len: Option) -> Result { + pub fn with_dtype(u: &mut Unstructured, dtype: &DType) -> Result { let scalar = random_scalar(u, dtype)?; - let len = len.unwrap_or(u.int_in_range(0..=100)?); + let len = u.int_in_range(0..=2048)?; Ok(ArbitraryConstantArray(ConstantArray::new(scalar, len))) } } diff --git a/vortex-array/src/arrays/constant/compute/cast.rs b/vortex-array/src/arrays/constant/compute/cast.rs index db3a41829f0..3efc95c0a0c 100644 --- a/vortex-array/src/arrays/constant/compute/cast.rs +++ b/vortex-array/src/arrays/constant/compute/cast.rs @@ -25,8 +25,8 @@ mod tests { use rstest::rstest; use crate::IntoArray; - use crate::LEGACY_SESSION; use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::ConstantArray; use crate::builtins::ArrayBuiltins; use crate::compute::conformance::cast::test_cast_conformance; @@ -44,7 +44,7 @@ mod tests { #[case(ConstantArray::new(Scalar::null_native::(), 4).into_array())] #[case(ConstantArray::new(Scalar::from(255u8), 1).into_array())] fn test_cast_constant_conformance(#[case] array: crate::ArrayRef) { - test_cast_conformance(&array); + test_cast_conformance(&array, &mut array_session().create_execution_ctx()); } #[test] @@ -57,7 +57,7 @@ mod tests { assert_eq!(casted.dtype(), &target_dtype); let scalar = casted - .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .execute_scalar(0, &mut array_session().create_execution_ctx()) .unwrap(); assert_eq!( scalar.as_decimal().decimal_value(), diff --git a/vortex-array/src/arrays/constant/compute/mod.rs b/vortex-array/src/arrays/constant/compute/mod.rs index 80fab02dc88..a996bf79a0f 100644 --- a/vortex-array/src/arrays/constant/compute/mod.rs +++ b/vortex-array/src/arrays/constant/compute/mod.rs @@ -29,21 +29,41 @@ mod test { #[test] fn test_mask_constant() { - test_mask_conformance(&ConstantArray::new(Scalar::null_native::(), 5).into_array()); - test_mask_conformance(&ConstantArray::new(Scalar::from(3u16), 5).into_array()); - test_mask_conformance(&ConstantArray::new(Scalar::from(1.0f32 / 0.0f32), 5).into_array()); + test_mask_conformance( + &ConstantArray::new(Scalar::null_native::(), 5).into_array(), + &mut array_session().create_execution_ctx(), + ); + test_mask_conformance( + &ConstantArray::new(Scalar::from(3u16), 5).into_array(), + &mut array_session().create_execution_ctx(), + ); + test_mask_conformance( + &ConstantArray::new(Scalar::from(1.0f32 / 0.0f32), 5).into_array(), + &mut array_session().create_execution_ctx(), + ); test_mask_conformance( &ConstantArray::new(Scalar::from(f16::from_f32(3.0f32)), 5).into_array(), + &mut array_session().create_execution_ctx(), ); } #[test] fn test_filter_constant() { - test_filter_conformance(&ConstantArray::new(Scalar::null_native::(), 5).into_array()); - test_filter_conformance(&ConstantArray::new(Scalar::from(3u16), 5).into_array()); - test_filter_conformance(&ConstantArray::new(Scalar::from(1.0f32 / 0.0f32), 5).into_array()); + test_filter_conformance( + &ConstantArray::new(Scalar::null_native::(), 5).into_array(), + &mut array_session().create_execution_ctx(), + ); + test_filter_conformance( + &ConstantArray::new(Scalar::from(3u16), 5).into_array(), + &mut array_session().create_execution_ctx(), + ); + test_filter_conformance( + &ConstantArray::new(Scalar::from(1.0f32 / 0.0f32), 5).into_array(), + &mut array_session().create_execution_ctx(), + ); test_filter_conformance( &ConstantArray::new(Scalar::from(f16::from_f32(3.0f32)), 5).into_array(), + &mut array_session().create_execution_ctx(), ); } diff --git a/vortex-array/src/arrays/constant/compute/take.rs b/vortex-array/src/arrays/constant/compute/take.rs index 9fb1373a5cf..2c9d2e792b4 100644 --- a/vortex-array/src/arrays/constant/compute/take.rs +++ b/vortex-array/src/arrays/constant/compute/take.rs @@ -73,8 +73,6 @@ mod tests { use vortex_mask::AllOr; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::ConstantArray; @@ -104,8 +102,7 @@ mod tests { taken.dtype() ); assert_arrays_eq!( - #[expect(deprecated)] - taken.to_primitive(), + taken.clone().execute::(&mut ctx).unwrap(), PrimitiveArray::new( buffer![42i32, 42, 42], Validity::from_iter([false, true, false]) @@ -116,7 +113,7 @@ mod tests { taken .validity() .unwrap() - .execute_mask(taken.len(), &mut array_session().create_execution_ctx()) + .execute_mask(taken.len(), &mut ctx) .unwrap() .indices(), AllOr::Some(valid_indices) @@ -135,8 +132,7 @@ mod tests { taken.dtype() ); assert_arrays_eq!( - #[expect(deprecated)] - taken.to_primitive(), + taken.clone().execute::(&mut ctx).unwrap(), PrimitiveArray::new(buffer![42i32, 42, 42], Validity::AllValid), &mut ctx ); @@ -144,7 +140,7 @@ mod tests { taken .validity() .unwrap() - .execute_mask(taken.len(), &mut array_session().create_execution_ctx()) + .execute_mask(taken.len(), &mut ctx) .unwrap() .indices(), AllOr::All @@ -158,6 +154,9 @@ mod tests { #[case(ConstantArray::new(Scalar::null_native::(), 5))] #[case(ConstantArray::new(true, 1))] fn test_take_constant_conformance(#[case] array: ConstantArray) { - test_take_conformance(&array.into_array()); + test_take_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index b2996e4985c..682035f2d78 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -236,7 +236,7 @@ impl VTable for Constant { .clone() .execute::(ctx)? .into_array(); - builder.extend_from_array(&canonical); + canonical.append_to_builder(builder, ctx)?; } } diff --git a/vortex-array/src/arrays/datetime/test.rs b/vortex-array/src/arrays/datetime/test.rs index 1ae114874d9..60081650f9f 100644 --- a/vortex-array/src/arrays/datetime/test.rs +++ b/vortex-array/src/arrays/datetime/test.rs @@ -8,8 +8,6 @@ use vortex_error::VortexResult; use crate::EqMode; use crate::IntoArray; -#[expect(deprecated)] -use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::PrimitiveArray; @@ -202,8 +200,11 @@ fn test_validity_preservation(#[case] validity: Validity) { let temporal_array = TemporalData::new_timestamp(milliseconds, TimeUnit::Milliseconds, Some("UTC".into())); - #[expect(deprecated)] - let prim = temporal_array.temporal_values().to_primitive(); + let prim = temporal_array + .temporal_values() + .clone() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); assert!( prim.validity() .vortex_expect("temporal validity should be derivable") diff --git a/vortex-array/src/arrays/decimal/compute/cast.rs b/vortex-array/src/arrays/decimal/compute/cast.rs index 80fafde5059..45d332e7f3d 100644 --- a/vortex-array/src/arrays/decimal/compute/cast.rs +++ b/vortex-array/src/arrays/decimal/compute/cast.rs @@ -386,13 +386,12 @@ mod tests { use vortex_buffer::buffer; use super::upcast_decimal_values; + use crate::Canonical; use crate::IntoArray; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::DecimalArray; use crate::builtins::ArrayBuiltins; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::DecimalDType; @@ -402,6 +401,7 @@ mod tests { #[test] fn cast_decimal_to_nullable() { + let mut ctx = array_session().create_execution_ctx(); let decimal_dtype = DecimalDType::new(10, 2); let array = DecimalArray::new( buffer![100i32, 200, 300], @@ -411,12 +411,12 @@ mod tests { // Cast to nullable let nullable_dtype = DType::Decimal(decimal_dtype, Nullability::Nullable); - #[expect(deprecated)] let casted = array .into_array() .cast(nullable_dtype.clone()) .unwrap() - .to_decimal(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(casted.dtype(), &nullable_dtype); assert!(matches!(casted.validity(), Ok(Validity::AllValid))); @@ -425,6 +425,7 @@ mod tests { #[test] fn cast_nullable_to_non_nullable() { + let mut ctx = array_session().create_execution_ctx(); let decimal_dtype = DecimalDType::new(10, 2); // Create nullable array with no nulls @@ -432,12 +433,12 @@ mod tests { // Cast to non-nullable let non_nullable_dtype = DType::Decimal(decimal_dtype, Nullability::NonNullable); - #[expect(deprecated)] let casted = array .into_array() .cast(non_nullable_dtype.clone()) .unwrap() - .to_decimal(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(casted.dtype(), &non_nullable_dtype); assert!(matches!(casted.validity(), Ok(Validity::NonNullable))); @@ -446,6 +447,7 @@ mod tests { #[test] #[should_panic(expected = "Cannot cast array with invalid values to non-nullable type")] fn cast_nullable_with_nulls_to_non_nullable_fails() { + let mut ctx = array_session().create_execution_ctx(); let decimal_dtype = DecimalDType::new(10, 2); // Create nullable array with nulls @@ -453,16 +455,16 @@ mod tests { // Attempt to cast to non-nullable should fail let non_nullable_dtype = DType::Decimal(decimal_dtype, Nullability::NonNullable); - #[expect(deprecated)] - let result = array + array .into_array() .cast(non_nullable_dtype) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); - result.unwrap(); + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())) + .unwrap(); } #[test] fn cast_different_scale_rescales() { + let mut ctx = array_session().create_execution_ctx(); let array = DecimalArray::new( buffer![100i32], DecimalDType::new(10, 2), @@ -471,12 +473,12 @@ mod tests { // Cast 1.00 to scale 3, where it is stored as 1000. let different_dtype = DType::Decimal(DecimalDType::new(15, 3), Nullability::NonNullable); - #[expect(deprecated)] let casted = array .into_array() .cast(different_dtype) .unwrap() - .to_decimal(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(casted.precision(), 15); assert_eq!(casted.scale(), 3); @@ -486,6 +488,7 @@ mod tests { #[test] fn cast_downcast_precision_succeeds_when_values_fit() { + let mut ctx = array_session().create_execution_ctx(); let array = DecimalArray::new( buffer![100i64], DecimalDType::new(18, 2), @@ -494,8 +497,12 @@ mod tests { // Downcasting precision is allowed when every value fits. let smaller_dtype = DType::Decimal(DecimalDType::new(10, 2), Nullability::NonNullable); - #[expect(deprecated)] - let casted = array.into_array().cast(smaller_dtype).unwrap().to_decimal(); + let casted = array + .into_array() + .cast(smaller_dtype) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!(casted.precision(), 10); assert_eq!(casted.scale(), 2); @@ -504,6 +511,7 @@ mod tests { #[test] fn cast_downcast_precision_checks_values() { + let mut ctx = array_session().create_execution_ctx(); let array = DecimalArray::new( buffer![1000i64], DecimalDType::new(18, 0), @@ -511,11 +519,10 @@ mod tests { ); let smaller_dtype = DType::Decimal(DecimalDType::new(3, 0), Nullability::NonNullable); - #[expect(deprecated)] let result = array .into_array() .cast(smaller_dtype) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); assert!(result.is_err()); assert!( @@ -528,6 +535,7 @@ mod tests { #[test] fn cast_lower_scale_requires_exact_rescale() { + let mut ctx = array_session().create_execution_ctx(); let array = DecimalArray::new( buffer![123456i64], DecimalDType::new(10, 4), @@ -535,11 +543,10 @@ mod tests { ); let lower_scale_dtype = DType::Decimal(DecimalDType::new(10, 2), Nullability::NonNullable); - #[expect(deprecated)] let result = array .into_array() .cast(lower_scale_dtype) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); assert!(result.is_err()); assert!( @@ -552,6 +559,7 @@ mod tests { #[test] fn cast_lower_scale_ignores_null_lane_failures() { + let mut ctx = array_session().create_execution_ctx(); let array = DecimalArray::new( buffer![100i64, 123456], DecimalDType::new(10, 4), @@ -559,12 +567,12 @@ mod tests { ); let lower_scale_dtype = DType::Decimal(DecimalDType::new(3, 2), Nullability::Nullable); - #[expect(deprecated)] let casted = array .into_array() .cast(lower_scale_dtype) .unwrap() - .to_decimal(); + .execute::(&mut ctx) + .unwrap(); let mask = casted .as_ref() @@ -582,6 +590,7 @@ mod tests { #[test] fn cast_upcast_precision_succeeds() { + let mut ctx = array_session().create_execution_ctx(); let array = DecimalArray::new( buffer![100i32, 200, 300], DecimalDType::new(10, 2), @@ -590,8 +599,12 @@ mod tests { // Cast to higher precision with same scale - should succeed let wider_dtype = DType::Decimal(DecimalDType::new(38, 2), Nullability::NonNullable); - #[expect(deprecated)] - let casted = array.into_array().cast(wider_dtype).unwrap().to_decimal(); + let casted = array + .into_array() + .cast(wider_dtype) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!(casted.precision(), 38); assert_eq!(casted.scale(), 2); @@ -602,6 +615,7 @@ mod tests { #[test] fn cast_widening_same_physical_type_is_zero_copy() { + let mut ctx = array_session().create_execution_ctx(); // Decimal(10,2) and Decimal(18,2) are both physically i64 with the same scale, so widening // the precision must reuse the values buffer rather than allocate and re-scan it. let array = DecimalArray::new( @@ -612,8 +626,12 @@ mod tests { let src_ptr = array.buffer::().as_ptr(); let wider_dtype = DType::Decimal(DecimalDType::new(18, 2), Nullability::NonNullable); - #[expect(deprecated)] - let casted = array.into_array().cast(wider_dtype).unwrap().to_decimal(); + let casted = array + .into_array() + .cast(wider_dtype) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!(casted.precision(), 18); assert_eq!(casted.scale(), 2); @@ -629,6 +647,7 @@ mod tests { #[test] fn cast_to_non_decimal_returns_err() { + let mut ctx = array_session().create_execution_ctx(); let array = DecimalArray::new( buffer![100i32], DecimalDType::new(10, 2), @@ -636,11 +655,10 @@ mod tests { ); // Try to cast to non-decimal type - should fail since no kernel can handle it - #[expect(deprecated)] let result = array .into_array() .cast(DType::Utf8(Nullability::NonNullable)) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); assert!(result.is_err()); assert!( @@ -657,7 +675,10 @@ mod tests { #[case(DecimalArray::from_option_iter([Some(100i32), None, Some(300)], DecimalDType::new(10, 2)))] #[case(DecimalArray::new(buffer![42i32], DecimalDType::new(5, 1), Validity::NonNullable))] fn test_cast_decimal_conformance(#[case] array: DecimalArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] diff --git a/vortex-array/src/arrays/decimal/compute/fill_null.rs b/vortex-array/src/arrays/decimal/compute/fill_null.rs index 1c2904b7b6b..c4de15de5cc 100644 --- a/vortex-array/src/arrays/decimal/compute/fill_null.rs +++ b/vortex-array/src/arrays/decimal/compute/fill_null.rs @@ -94,8 +94,6 @@ mod tests { use crate::arrays::DecimalArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::dtype::DecimalDType; use crate::dtype::Nullability; use crate::scalar::DecimalValue; @@ -110,7 +108,6 @@ mod tests { [None, Some(800i128), None, Some(1000i128), None], decimal_dtype, ); - #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::decimal( @@ -119,7 +116,8 @@ mod tests { Nullability::NonNullable, )) .unwrap() - .to_decimal(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, DecimalArray::from_iter([4200, 800, 4200, 1000, 4200], decimal_dtype), @@ -152,7 +150,6 @@ mod tests { decimal_dtype, ); - #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::decimal( @@ -161,7 +158,8 @@ mod tests { Nullability::NonNullable, )) .unwrap() - .to_decimal(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, DecimalArray::from_iter([25500, 25500, 25500, 25500, 25500], decimal_dtype), @@ -176,7 +174,6 @@ mod tests { let decimal_dtype = DecimalDType::new(3, 0); let arr = DecimalArray::from_option_iter([None, Some(10i8), None], decimal_dtype); // i8 max is 127, so 200 doesn't fit — the array should be widened to i16. - #[expect(deprecated)] let result = arr .into_array() .fill_null(Scalar::decimal( @@ -185,7 +182,8 @@ mod tests { Nullability::NonNullable, )) .unwrap() - .to_decimal(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( result, DecimalArray::from_iter([200i16, 10, 200], decimal_dtype), @@ -203,7 +201,6 @@ mod tests { decimal_dtype, Validity::NonNullable, ); - #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::decimal( @@ -212,7 +209,8 @@ mod tests { Nullability::NonNullable, )) .unwrap() - .to_decimal(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, DecimalArray::from_iter([800i128, 1000, 1200, 1400, 1600], decimal_dtype), diff --git a/vortex-array/src/arrays/decimal/compute/take.rs b/vortex-array/src/arrays/decimal/compute/take.rs index 762ac3d7365..ce786f8e4ab 100644 --- a/vortex-array/src/arrays/decimal/compute/take.rs +++ b/vortex-array/src/arrays/decimal/compute/take.rs @@ -131,6 +131,9 @@ mod tests { ) })] fn test_take_decimal_conformance(#[case] array: DecimalArray) { - test_take_conformance(&array.into_array()); + test_take_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index a52f2882238..32b3ea98510 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -19,6 +19,8 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::decimal::DecimalData; use crate::buffer::BufferHandle; +use crate::builders::ArrayBuilder; +use crate::builders::DecimalBuilder; use crate::dtype::DType; use crate::dtype::DecimalType; use crate::dtype::NativeDecimalType; @@ -188,6 +190,17 @@ impl VTable for Decimal { Ok(ExecutionResult::done(array)) } + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + vortex_bail!("append_to_builder for Decimal requires a DecimalBuilder"); + }; + builder.append_decimal_array(&array.into_owned(), ctx) + } + fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, diff --git a/vortex-array/src/arrays/dict/arbitrary.rs b/vortex-array/src/arrays/dict/arbitrary.rs index 4eb938181b9..2a70dd2586c 100644 --- a/vortex-array/src/arrays/dict/arbitrary.rs +++ b/vortex-array/src/arrays/dict/arbitrary.rs @@ -37,7 +37,6 @@ impl ArbitraryDictArray { pub fn with_dtype(u: &mut Unstructured, dtype: &DType, len: Option) -> Result { // Generate the number of unique values (dictionary size) let values_len = u.int_in_range(1..=20)?; - // Generate values array with the given dtype let values = ArbitraryArray::arbitrary_with_config( u, &ArbitraryArrayConfig { diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index f73f4965439..cefc5a381fd 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -15,15 +15,13 @@ use vortex_mask::AllOr; use crate::ArrayRef; use crate::ArraySlots; -use crate::LEGACY_SESSION; -#[expect(deprecated)] -use crate::ToCanonical as _; -use crate::VortexSessionExecute; +use crate::ExecutionCtx; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; use crate::array_slots; use crate::arrays::Dict; +use crate::arrays::PrimitiveArray; use crate::dtype::DType; use crate::dtype::PType; use crate::match_each_integer_ptype; @@ -129,13 +127,13 @@ pub trait DictArrayExt: TypedArrayRef + DictArraySlotsExt { self.all_values_referenced } - fn validate_all_values_referenced(&self) -> VortexResult<()> { + fn validate_all_values_referenced(&self, ctx: &mut ExecutionCtx) -> VortexResult<()> { if self.has_all_values_referenced() { if !self.codes().is_host() { return Ok(()); } - let referenced_mask = self.compute_referenced_values_mask(true)?; + let referenced_mask = self.compute_referenced_values_mask(true, ctx)?; let all_referenced = referenced_mask.true_count() == referenced_mask.len(); vortex_ensure!(all_referenced, "value in dict not referenced"); @@ -144,13 +142,14 @@ pub trait DictArrayExt: TypedArrayRef + DictArraySlotsExt { Ok(()) } - fn compute_referenced_values_mask(&self, referenced: bool) -> VortexResult { + fn compute_referenced_values_mask( + &self, + referenced: bool, + ctx: &mut ExecutionCtx, + ) -> VortexResult { let codes = self.codes(); - let codes_validity = codes - .validity()? - .execute_mask(codes.len(), &mut LEGACY_SESSION.create_execution_ctx())?; - #[expect(deprecated)] - let codes_primitive = self.codes().to_primitive(); + let codes_validity = codes.validity()?.execute_mask(codes.len(), ctx)?; + let codes_primitive = codes.clone().execute::(ctx)?; let values_len = self.values().len(); let init_value = !referenced; @@ -268,18 +267,9 @@ impl Array { self.into_data() .set_all_values_referenced(all_values_referenced) }; - let array = unsafe { + unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data).with_slots(slots)) - }; - - #[cfg(debug_assertions)] - if all_values_referenced { - array - .validate_all_values_referenced() - .vortex_expect("validation should succeed when all values are referenced"); } - - array } } @@ -299,8 +289,6 @@ mod test { use crate::ArrayRef; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::ChunkedArray; @@ -468,9 +456,8 @@ mod test { &mut array_session().create_execution_ctx(), )?; - #[expect(deprecated)] - let into_prim = array.to_primitive(); - let prim_into = builder.finish_into_canonical().into_primitive(); + let into_prim = array.execute::(&mut ctx)?; + let prim_into = builder.finish_into_canonical(&mut ctx).into_primitive(); assert_arrays_eq!(into_prim, prim_into, &mut ctx); Ok(()) diff --git a/vortex-array/src/arrays/dict/compute/cast.rs b/vortex-array/src/arrays/dict/compute/cast.rs index 56a433627a2..2597436f98f 100644 --- a/vortex-array/src/arrays/dict/compute/cast.rs +++ b/vortex-array/src/arrays/dict/compute/cast.rs @@ -184,7 +184,7 @@ mod tests { #[case(dict_encode(&PrimitiveArray::from_option_iter([Some(1i32), None, Some(2), Some(1), None]).into_array(), &mut SESSION.create_execution_ctx()).unwrap().into_array())] #[case(dict_encode(&buffer![1.5f32, 2.5, 1.5, 3.5].into_array(), &mut SESSION.create_execution_ctx()).unwrap().into_array())] fn test_cast_dict_conformance(#[case] array: ArrayRef) { - test_cast_conformance(&array); + test_cast_conformance(&array, &mut SESSION.create_execution_ctx()); } #[test] diff --git a/vortex-array/src/arrays/dict/compute/is_constant.rs b/vortex-array/src/arrays/dict/compute/is_constant.rs index 82518963106..85438683442 100644 --- a/vortex-array/src/arrays/dict/compute/is_constant.rs +++ b/vortex-array/src/arrays/dict/compute/is_constant.rs @@ -47,7 +47,7 @@ impl DynAggregateKernel for DictIsConstantKernel { let result = if dict.has_all_values_referenced() { is_constant(dict.values(), ctx)? } else { - let referenced_mask = dict.compute_referenced_values_mask(true)?; + let referenced_mask = dict.compute_referenced_values_mask(true, ctx)?; let mask = Mask::from(referenced_mask); let filtered_values = dict.values().filter(mask)?; is_constant(&filtered_values, ctx)? diff --git a/vortex-array/src/arrays/dict/compute/min_max.rs b/vortex-array/src/arrays/dict/compute/min_max.rs index ee38c0392d4..0122761c7bf 100644 --- a/vortex-array/src/arrays/dict/compute/min_max.rs +++ b/vortex-array/src/arrays/dict/compute/min_max.rs @@ -45,7 +45,7 @@ impl DynAggregateKernel for DictMinMaxKernel { min_max(dict.values(), ctx, *options)? } else { // Filter to only referenced values, then compute min/max. - let referenced_mask = dict.compute_referenced_values_mask(true)?; + let referenced_mask = dict.compute_referenced_values_mask(true, ctx)?; let mask = Mask::from(referenced_mask); let filtered_values = dict.values().filter(mask)?; min_max(&filtered_values, ctx, *options)? diff --git a/vortex-array/src/arrays/dict/compute/mod.rs b/vortex-array/src/arrays/dict/compute/mod.rs index e2b0d07b97d..5c013b2d321 100644 --- a/vortex-array/src/arrays/dict/compute/mod.rs +++ b/vortex-array/src/arrays/dict/compute/mod.rs @@ -215,7 +215,7 @@ mod test { &mut SESSION.create_execution_ctx(), ) .unwrap(); - test_mask_conformance(&array.into_array()); + test_mask_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); let array = dict_encode( &PrimitiveArray::from_option_iter([Some(2), None, Some(2), Some(0), Some(10)]) @@ -223,7 +223,7 @@ mod test { &mut SESSION.create_execution_ctx(), ) .unwrap(); - test_mask_conformance(&array.into_array()); + test_mask_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); let array = dict_encode( &VarBinArray::from_iter( @@ -240,7 +240,7 @@ mod test { &mut SESSION.create_execution_ctx(), ) .unwrap(); - test_mask_conformance(&array.into_array()); + test_mask_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -250,7 +250,7 @@ mod test { &mut SESSION.create_execution_ctx(), ) .unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); let array = dict_encode( &PrimitiveArray::from_option_iter([Some(2), None, Some(2), Some(0), Some(10)]) @@ -258,7 +258,7 @@ mod test { &mut SESSION.create_execution_ctx(), ) .unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); let array = dict_encode( &VarBinArray::from_iter( @@ -275,7 +275,7 @@ mod test { &mut SESSION.create_execution_ctx(), ) .unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -302,7 +302,7 @@ mod test { &mut SESSION.create_execution_ctx(), ) .unwrap(); - test_take_conformance(&array.into_array()); + test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); let array = dict_encode( &PrimitiveArray::from_option_iter([Some(2), None, Some(2), Some(0), Some(10)]) @@ -310,7 +310,7 @@ mod test { &mut SESSION.create_execution_ctx(), ) .unwrap(); - test_take_conformance(&array.into_array()); + test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); let array = dict_encode( &VarBinArray::from_iter( @@ -327,7 +327,7 @@ mod test { &mut SESSION.create_execution_ctx(), ) .unwrap(); - test_take_conformance(&array.into_array()); + test_take_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/vortex-array/src/arrays/dict/compute/rules.rs b/vortex-array/src/arrays/dict/compute/rules.rs index e4d102ee861..5dc5cebf3f1 100644 --- a/vortex-array/src/arrays/dict/compute/rules.rs +++ b/vortex-array/src/arrays/dict/compute/rules.rs @@ -267,6 +267,7 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; + use crate::array_session; use crate::arrays::BoolArray; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; @@ -298,7 +299,7 @@ mod tests { assert!(ArrayRef::ptr_eq(dict.values(), &values)); assert_eq!(codes.nchunks(), 2); - let mut ctx = crate::LEGACY_SESSION.create_execution_ctx(); + let mut ctx = array_session().create_execution_ctx(); assert_arrays_eq!( optimized, PrimitiveArray::from_iter([10u32, 20, 30, 10, 20]), @@ -321,7 +322,7 @@ mod tests { let optimized = array.optimize()?; assert!(optimized.is::()); - let mut ctx = crate::LEGACY_SESSION.create_execution_ctx(); + let mut ctx = array_session().create_execution_ctx(); assert_arrays_eq!( optimized, PrimitiveArray::from_iter([10u32, 20, 30, 10, 20]), diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index dfce610f43a..e080eb7f04c 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -146,7 +146,7 @@ mod tests { #[case(create_timestamp_array(TimeUnit::Nanoseconds, false))] #[case(create_timestamp_array(TimeUnit::Seconds, true))] fn test_cast_extension_conformance(#[case] array: ExtensionArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } fn create_timestamp_array(time_unit: TimeUnit, nullable: bool) -> ExtensionArray { diff --git a/vortex-array/src/arrays/extension/compute/mod.rs b/vortex-array/src/arrays/extension/compute/mod.rs index 8309ae43be2..256f31ac49b 100644 --- a/vortex-array/src/arrays/extension/compute/mod.rs +++ b/vortex-array/src/arrays/extension/compute/mod.rs @@ -15,6 +15,8 @@ mod test { use vortex_buffer::buffer; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::ExtensionArray; use crate::arrays::PrimitiveArray; use crate::compute::conformance::filter::test_filter_conformance; @@ -31,14 +33,20 @@ mod test { // Create storage array let storage = buffer![1i32, 2, 3, 4, 5].into_array(); let array = ExtensionArray::new(ext_dtype.clone(), storage); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); // Test with nullable extension type let ext_dtype_nullable = ext_dtype.with_nullability(Nullability::Nullable); let storage = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), Some(4), None]) .into_array(); let array = ExtensionArray::new(ext_dtype_nullable, storage); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[rstest] @@ -77,7 +85,10 @@ mod test { ExtensionArray::new(ext_dtype_large, storage) })] fn test_take_extension_array_conformance(#[case] array: ExtensionArray) { - test_take_conformance(&array.into_array()); + test_take_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index 611be1510a4..8af0bbf3a63 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -87,8 +87,7 @@ mod tests { use crate::EmptyMetadata; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; + use crate::VortexSessionExecute; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Extension; @@ -178,8 +177,11 @@ mod tests { assert_eq!(ext_result.ext_dtype(), &ext_dtype); // Check the storage values - #[expect(deprecated)] - let storage_prim = ext_result.storage_array().to_primitive(); + let storage_prim = ext_result + .storage_array() + .clone() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); let storage_result: &[i64] = &storage_prim.to_buffer::(); assert_eq!(storage_result, &[1, 3, 5]); } @@ -206,8 +208,11 @@ mod tests { assert_eq!(ext_result.len(), 3); // Check values: should be [Some(1), None, None] - #[expect(deprecated)] - let canonical = ext_result.storage_array().to_primitive(); + let canonical = ext_result + .storage_array() + .clone() + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); assert_eq!(canonical.len(), 3); } diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index a1fab1cd47a..a1429e2180b 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -26,6 +26,8 @@ use crate::arrays::extension::array::STORAGE_SLOT; use crate::arrays::extension::compute::rules::PARENT_RULES; use crate::arrays::extension::compute::rules::RULES; use crate::buffer::BufferHandle; +use crate::builders::ArrayBuilder; +use crate::builders::ExtensionBuilder; use crate::dtype::DType; use crate::serde::ArrayChildren; @@ -179,6 +181,17 @@ impl VTable for Extension { Ok(ExecutionResult::done(array)) } + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + vortex_bail!("append_to_builder for Extension requires an ExtensionBuilder"); + }; + builder.append_extension_array(&array.into_owned(), ctx) + } + fn reduce(array: ArrayView<'_, Self>) -> VortexResult> { RULES.evaluate(array) } diff --git a/vortex-array/src/arrays/filter/execute/bool.rs b/vortex-array/src/arrays/filter/execute/bool.rs index a22e55366fc..08364e3cac5 100644 --- a/vortex-array/src/arrays/filter/execute/bool.rs +++ b/vortex-array/src/arrays/filter/execute/bool.rs @@ -30,18 +30,22 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::filter::execute::bool::BoolArray; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::compute::conformance::filter::test_filter_conformance; #[test] fn filter_bool_test() { + let mut ctx = array_session().create_execution_ctx(); let arr = BoolArray::from_iter([true, true, false]); let mask = Mask::from_iter([true, false, true]); - #[expect(deprecated)] - let filtered = arr.filter(mask).unwrap().to_bool(); + let filtered = arr + .filter(mask) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!(2, filtered.len()); assert_eq!( @@ -58,6 +62,9 @@ mod test { #[case(BoolArray::from_iter((0..100).map(|i| i % 2 == 0)))] #[case(BoolArray::from_iter((0..1024).map(|i| i % 3 != 0)))] fn test_filter_bool_conformance(#[case] array: BoolArray) { - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/filter/execute/decimal.rs b/vortex-array/src/arrays/filter/execute/decimal.rs index 869e1fabe0c..5ffb4a963b2 100644 --- a/vortex-array/src/arrays/filter/execute/decimal.rs +++ b/vortex-array/src/arrays/filter/execute/decimal.rs @@ -34,6 +34,8 @@ pub fn filter_decimal(array: &DecimalArray, mask: &Arc) -> DecimalAr #[cfg(test)] mod test { use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::filter::execute::decimal::DecimalArray; use crate::compute::conformance::filter::test_filter_conformance; use crate::dtype::DecimalDType; @@ -49,7 +51,10 @@ mod test { Some(99999), ]; let array = DecimalArray::from_option_iter(values, decimal_dtype); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] @@ -57,6 +62,9 @@ mod test { let decimal_dtype = DecimalDType::new(38, 4); let values = vec![Some(12345i128), None, Some(-12345), Some(0), None]; let array = DecimalArray::from_option_iter(values, decimal_dtype); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/filter/execute/fixed_size_list.rs b/vortex-array/src/arrays/filter/execute/fixed_size_list.rs index 392a6373f76..6facbe3e38d 100644 --- a/vortex-array/src/arrays/filter/execute/fixed_size_list.rs +++ b/vortex-array/src/arrays/filter/execute/fixed_size_list.rs @@ -137,7 +137,10 @@ mod test { fn test_filter_fixed_size_list_conformance() { let elements = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5, 6, 7, 8, 9]); let array = FixedSizeListArray::new(elements.into_array(), 3, Validity::NonNullable, 3); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] @@ -146,7 +149,10 @@ mod test { PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), Some(4), Some(5), None]); let validity = Validity::from_iter([true, false, true]); let array = FixedSizeListArray::new(elements.into_array(), 2, validity, 3); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] diff --git a/vortex-array/src/arrays/filter/execute/listview.rs b/vortex-array/src/arrays/filter/execute/listview.rs index 0507452c89d..6d1521541bc 100644 --- a/vortex-array/src/arrays/filter/execute/listview.rs +++ b/vortex-array/src/arrays/filter/execute/listview.rs @@ -85,7 +85,7 @@ mod test { let sizes = buffer![2u32, 2, 2].into_array(); let array = ListViewArray::new(elements.into_array(), offsets, sizes, Validity::NonNullable); - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -96,7 +96,7 @@ mod test { let sizes = buffer![2u32, 2, 2].into_array(); let validity = Validity::from_iter([true, false, true]); let array = ListViewArray::new(elements.into_array(), offsets, sizes, validity); - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -109,7 +109,7 @@ mod test { ListViewArray::new_unchecked(elements, offsets, sizes, Validity::NonNullable) .with_zero_copy_to_list(true) }; - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -120,7 +120,7 @@ mod test { let offsets = buffer![5u32, 2, 8, 0, 1].into_array(); let sizes = buffer![3u32, 2, 2, 2, 4].into_array(); let array = ListViewArray::new(elements, offsets, sizes, Validity::NonNullable); - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -130,7 +130,7 @@ mod test { let offsets = buffer![0u32, 100, 200, 300, 400, 500, 600, 700, 800, 900].into_array(); let sizes = buffer![50u32, 50, 50, 50, 50, 50, 50, 50, 50, 50].into_array(); let array = ListViewArray::new(elements, offsets, sizes, Validity::NonNullable); - test_filter_conformance(&array.into_array()); + test_filter_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] diff --git a/vortex-array/src/arrays/filter/execute/mod.rs b/vortex-array/src/arrays/filter/execute/mod.rs index 52b0373ac49..f35f628c332 100644 --- a/vortex-array/src/arrays/filter/execute/mod.rs +++ b/vortex-array/src/arrays/filter/execute/mod.rs @@ -83,13 +83,19 @@ pub(super) fn execute_filter_fast_paths( } /// Filter a canonical array by a mask, returning a new canonical array. -pub(super) fn execute_filter(canonical: Canonical, mask: &Arc) -> Canonical { +pub(super) fn execute_filter( + canonical: Canonical, + mask: &Arc, + ctx: &mut ExecutionCtx, +) -> Canonical { match canonical { Canonical::Null(_) => Canonical::Null(NullArray::new(mask.true_count())), Canonical::Bool(a) => Canonical::Bool(bool::filter_bool(&a, mask)), Canonical::Primitive(a) => Canonical::Primitive(primitive::filter_primitive(&a, mask)), Canonical::Decimal(a) => Canonical::Decimal(decimal::filter_decimal(&a, mask)), - Canonical::VarBinView(a) => Canonical::VarBinView(varbinview::filter_varbinview(&a, mask)), + Canonical::VarBinView(a) => { + Canonical::VarBinView(varbinview::filter_varbinview(&a, mask, ctx)) + } Canonical::List(a) => Canonical::List(listview::filter_listview(&a, mask)), Canonical::FixedSizeList(a) => { Canonical::FixedSizeList(fixed_size_list::filter_fixed_size_list(&a, mask)) diff --git a/vortex-array/src/arrays/filter/execute/primitive.rs b/vortex-array/src/arrays/filter/execute/primitive.rs index 276e08b5526..b7abd8efa4b 100644 --- a/vortex-array/src/arrays/filter/execute/primitive.rs +++ b/vortex-array/src/arrays/filter/execute/primitive.rs @@ -59,20 +59,24 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::PrimitiveArray; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::compute::conformance::filter::LARGE_SIZE; use crate::compute::conformance::filter::MEDIUM_SIZE; use crate::compute::conformance::filter::test_filter_conformance; #[test] fn filter_run_variant_mixed_test() { + let mut ctx = array_session().create_execution_ctx(); let mask = [true, true, false, true, true, true, false, true]; let arr = PrimitiveArray::from_iter([1u32, 24, 54, 2, 3, 2, 3, 2]); - #[expect(deprecated)] - let filtered = arr.filter(Mask::from_iter(mask)).unwrap().to_primitive(); + let filtered = arr + .filter(Mask::from_iter(mask)) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!( filtered.len(), mask.iter().filter(|x| **x).collect_vec().len() @@ -104,6 +108,9 @@ mod test { #[case(PrimitiveArray::from_iter([0.1f32, 0.2, 0.3, 0.4, 0.5]))] #[case(PrimitiveArray::from_option_iter([Some(1.1f64), None, Some(2.2), Some(3.3), None]))] fn test_filter_primitive_conformance(#[case] array: PrimitiveArray) { - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/filter/execute/struct_.rs b/vortex-array/src/arrays/filter/execute/struct_.rs index 8980c422ef3..e4a8157d2e2 100644 --- a/vortex-array/src/arrays/filter/execute/struct_.rs +++ b/vortex-array/src/arrays/filter/execute/struct_.rs @@ -70,7 +70,10 @@ mod test { ]; let array = StructArray::try_new(["a", "b"].into(), fields, 5, Validity::NonNullable).unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] @@ -83,7 +86,10 @@ mod test { ]; let array = StructArray::try_new(["a", "b"].into(), fields, 5, Validity::NonNullable).unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] @@ -150,6 +156,7 @@ mod test { &StructArray::try_new(FieldNames::empty(), vec![], 5, Validity::NonNullable) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } @@ -184,6 +191,7 @@ mod test { ) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } } diff --git a/vortex-array/src/arrays/filter/execute/varbinview.rs b/vortex-array/src/arrays/filter/execute/varbinview.rs index ded9c804cfd..1f9df8f99cd 100644 --- a/vortex-array/src/arrays/filter/execute/varbinview.rs +++ b/vortex-array/src/arrays/filter/execute/varbinview.rs @@ -9,33 +9,44 @@ use vortex_mask::Mask; use vortex_mask::MaskValues; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::arrow::ArrowSessionExt; use crate::arrow::FromArrowArray; -pub fn filter_varbinview(array: &VarBinViewArray, mask: &Arc) -> VarBinViewArray { +pub fn filter_varbinview( + array: &VarBinViewArray, + mask: &Arc, + ctx: &mut ExecutionCtx, +) -> VarBinViewArray { // Delegate to the Arrow implementation of filter over `VarBinView`. - arrow_filter_fn(&array.clone().into_array(), &Mask::Values(Arc::clone(mask))) - .vortex_expect("VarBinViewArray is Arrow-compatible and supports arrow_filter_fn") - .as_::() - .into_owned() + arrow_filter_fn( + &array.clone().into_array(), + &Mask::Values(Arc::clone(mask)), + ctx, + ) + .vortex_expect("VarBinViewArray is Arrow-compatible and supports arrow_filter_fn") + .as_::() + .into_owned() } -fn arrow_filter_fn(array: &ArrayRef, mask: &Mask) -> vortex_error::VortexResult { +fn arrow_filter_fn( + array: &ArrayRef, + mask: &Mask, + ctx: &mut ExecutionCtx, +) -> vortex_error::VortexResult { let values = match &mask { Mask::Values(values) => values, Mask::AllTrue(_) | Mask::AllFalse(_) => unreachable!("check in filter invoke"), }; - let array_ref = LEGACY_SESSION.arrow().execute_arrow( - array.clone(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let array_ref = ctx + .session() + .arrow() + .clone() + .execute_arrow(array.clone(), None, ctx)?; let mask_array = BooleanArray::new(values.bit_buffer().clone().into(), None); let filtered = arrow_select::filter::filter(array_ref.as_ref(), &mask_array)?; @@ -45,6 +56,8 @@ fn arrow_filter_fn(array: &ArrayRef, mask: &Mask) -> vortex_error::VortexResult< #[cfg(test)] mod test { use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::VarBinViewArray; use crate::compute::conformance::filter::test_filter_conformance; @@ -52,6 +65,7 @@ mod test { fn test_filter_varbinview_conformance() { test_filter_conformance( &VarBinViewArray::from_iter_str(["one", "two", "three", "four", "five"]).into_array(), + &mut array_session().create_execution_ctx(), ); test_filter_conformance( @@ -63,6 +77,7 @@ mod test { Some("five"), ]) .into_array(), + &mut array_session().create_execution_ctx(), ); } } diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index af11393c5db..074c588ffda 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -158,7 +158,7 @@ impl VTable for Filter { // TODO(joe): fix the ownership of AnyCanonical let child = Canonical::from(array.child().as_::()); Ok(ExecutionResult::done( - execute_filter(child, &mask_values).into_array(), + execute_filter(child, &mask_values, ctx).into_array(), )) } diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index f24001c4e78..b031b239909 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -12,8 +12,6 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::ArraySlots; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; @@ -238,14 +236,6 @@ pub trait FixedSizeListArrayExt: TypedArrayRef { index, self.as_ref().len(), ); - #[expect(clippy::debug_assert_with_mut_call)] - { - debug_assert!( - self.fixed_size_list_validity() - .execute_is_valid(index, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap_or(false) - ); - } let start = self.list_size() as usize * index; let end = self.list_size() as usize * (index + 1); diff --git a/vortex-array/src/arrays/fixed_size_list/tests/filter.rs b/vortex-array/src/arrays/fixed_size_list/tests/filter.rs index 6f609120901..69677400692 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/filter.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/filter.rs @@ -182,7 +182,7 @@ fn test_filter_nested_fixed_size_lists() { #[case(create_fsl_single_element())] #[case(create_fsl_empty())] fn test_filter_fsl_conformance(#[case] array: ArrayRef) { - test_filter_conformance(&array); + test_filter_conformance(&array, &mut array_session().create_execution_ctx()); } // Helper functions for creating test arrays. diff --git a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs index 44420d22cb3..40fd2bc066a 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs @@ -6,8 +6,6 @@ use std::sync::Arc; use vortex_buffer::buffer; use crate::IntoArray; -#[expect(deprecated)] -use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::FixedSizeListArray; @@ -85,9 +83,10 @@ fn test_fsl_of_fsl_basic() { let first_outer_list = outer_fsl.fixed_size_list_elements_at(0).unwrap(); // Check first inner list [1,2]. - #[expect(deprecated)] let inner_list_0 = first_outer_list - .to_fixed_size_list() + .clone() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(0) .unwrap(); assert_eq!( @@ -104,9 +103,10 @@ fn test_fsl_of_fsl_basic() { ); // Check second inner list [3,4]. - #[expect(deprecated)] let inner_list_1 = first_outer_list - .to_fixed_size_list() + .clone() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(1) .unwrap(); assert_eq!( @@ -123,9 +123,9 @@ fn test_fsl_of_fsl_basic() { ); // Check third inner list [5,6]. - #[expect(deprecated)] let inner_list_2 = first_outer_list - .to_fixed_size_list() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(2) .unwrap(); assert_eq!( @@ -145,9 +145,10 @@ fn test_fsl_of_fsl_basic() { let second_outer_list = outer_fsl.fixed_size_list_elements_at(1).unwrap(); // Check first inner list [7,8]. - #[expect(deprecated)] let inner_list_0 = second_outer_list - .to_fixed_size_list() + .clone() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(0) .unwrap(); assert_eq!( @@ -164,9 +165,10 @@ fn test_fsl_of_fsl_basic() { ); // Check second inner list [9,10]. - #[expect(deprecated)] let inner_list_1 = second_outer_list - .to_fixed_size_list() + .clone() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(1) .unwrap(); assert_eq!( @@ -183,9 +185,9 @@ fn test_fsl_of_fsl_basic() { ); // Check third inner list [11,12]. - #[expect(deprecated)] let inner_list_2 = second_outer_list - .to_fixed_size_list() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(2) .unwrap(); assert_eq!( @@ -307,21 +309,23 @@ fn test_deeply_nested_fsl() { // Check the actual deeply nested values. // Structure: [[[1,2],[3,4]],[[5,6],[7,8]]]. let top_level = level3.fixed_size_list_elements_at(0).unwrap(); - #[expect(deprecated)] let level2_0 = top_level - .to_fixed_size_list() + .clone() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(0) .unwrap(); - #[expect(deprecated)] let level2_1 = top_level - .to_fixed_size_list() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(1) .unwrap(); // First level-2 list: [[1,2],[3,4]]. - #[expect(deprecated)] let level1_0_0 = level2_0 - .to_fixed_size_list() + .clone() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(0) .unwrap(); assert_eq!( @@ -337,9 +341,9 @@ fn test_deeply_nested_fsl() { 2i32.into() ); - #[expect(deprecated)] let level1_0_1 = level2_0 - .to_fixed_size_list() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(1) .unwrap(); assert_eq!( @@ -356,9 +360,10 @@ fn test_deeply_nested_fsl() { ); // Second level-2 list: [[5,6],[7,8]]. - #[expect(deprecated)] let level1_1_0 = level2_1 - .to_fixed_size_list() + .clone() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(0) .unwrap(); assert_eq!( @@ -374,9 +379,9 @@ fn test_deeply_nested_fsl() { 6i32.into() ); - #[expect(deprecated)] let level1_1_1 = level2_1 - .to_fixed_size_list() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap() .fixed_size_list_elements_at(1) .unwrap(); assert_eq!( diff --git a/vortex-array/src/arrays/fixed_size_list/tests/take.rs b/vortex-array/src/arrays/fixed_size_list/tests/take.rs index e75139e5fc8..af8f94f2d5f 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/take.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/take.rs @@ -33,7 +33,10 @@ use crate::validity::Validity; #[case::single_element(create_single_element_fsl())] #[case::empty(create_empty_fsl())] fn test_take_fsl_conformance(#[case] fsl: FixedSizeListArray) { - test_take_conformance(&fsl.into_array()); + test_take_conformance( + &fsl.into_array(), + &mut array_session().create_execution_ctx(), + ); } // FSL-specific edge case tests that aren't covered by conformance. @@ -88,7 +91,10 @@ fn test_take_degenerate_lists( let elements = PrimitiveArray::empty::(Nullability::NonNullable); let fsl = FixedSizeListArray::new(elements.into_array(), 0, validity, 5); - test_take_conformance(&fsl.clone().into_array()); + test_take_conformance( + &fsl.clone().into_array(), + &mut array_session().create_execution_ctx(), + ); // Also test the specific behavior. let indices_array = PrimitiveArray::from_option_iter(indices); diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index bfc172a5049..5c7fde36682 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -29,6 +29,8 @@ use crate::arrays::fixed_size_list::array::NUM_SLOTS; use crate::arrays::fixed_size_list::array::SLOT_NAMES; use crate::arrays::fixed_size_list::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; +use crate::builders::ArrayBuilder; +use crate::builders::FixedSizeListBuilder; use crate::dtype::DType; use crate::serde::ArrayChildren; use crate::validity::Validity; @@ -191,4 +193,15 @@ impl VTable for FixedSizeList { fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } + + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + vortex_bail!("append_to_builder for FixedSizeList requires a FixedSizeListBuilder"); + }; + builder.append_fixed_size_list_array(&array.into_owned(), ctx) + } } diff --git a/vortex-array/src/arrays/list/compute/cast.rs b/vortex-array/src/arrays/list/compute/cast.rs index 084d9613e75..9fd27f731f1 100644 --- a/vortex-array/src/arrays/list/compute/cast.rs +++ b/vortex-array/src/arrays/list/compute/cast.rs @@ -181,7 +181,7 @@ mod tests { #[case(create_nested_list())] #[case(create_empty_lists())] fn test_cast_list_conformance(#[case] array: ListArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } fn create_simple_list() -> ListArray { diff --git a/vortex-array/src/arrays/list/compute/mod.rs b/vortex-array/src/arrays/list/compute/mod.rs index d8ddbd37583..ce6cd960f74 100644 --- a/vortex-array/src/arrays/list/compute/mod.rs +++ b/vortex-array/src/arrays/list/compute/mod.rs @@ -37,7 +37,10 @@ mod tests { let array = ListArray::try_new(elements.into_array(), offsets.into_array(), validity).unwrap(); - test_mask_conformance(&array.into_array()); + test_mask_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] @@ -48,7 +51,10 @@ mod tests { let array = ListArray::try_new(elements.into_array(), offsets.into_array(), validity).unwrap(); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[rstest] diff --git a/vortex-array/src/arrays/list/compute/take.rs b/vortex-array/src/arrays/list/compute/take.rs index 2c6089e0c58..14e89d4c238 100644 --- a/vortex-array/src/arrays/list/compute/take.rs +++ b/vortex-array/src/arrays/list/compute/take.rs @@ -212,12 +212,11 @@ mod test { use vortex_buffer::buffer; use crate::IntoArray as _; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::BoolArray; use crate::arrays::ListArray; + use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; use crate::compute::conformance::take::test_take_conformance; use crate::dtype::DType; @@ -228,6 +227,7 @@ mod test { #[test] fn nullable_take() { + let mut ctx = array_session().create_execution_ctx(); let list = ListArray::try_new( buffer![0i32, 5, 3, 4].into_array(), buffer![0, 2, 3, 4, 4].into_array(), @@ -249,8 +249,7 @@ mod test { ) ); - #[expect(deprecated)] - let result = result.to_listview(); + let result = result.execute::(&mut ctx).unwrap(); assert_eq!(result.len(), 4); @@ -332,6 +331,7 @@ mod test { #[test] fn non_nullable_take() { + let mut ctx = array_session().create_execution_ctx(); let list = ListArray::try_new( buffer![0i32, 5, 3, 4].into_array(), buffer![0, 2, 3, 3, 4].into_array(), @@ -352,8 +352,7 @@ mod test { ) ); - #[expect(deprecated)] - let result = result.to_listview(); + let result = result.execute::(&mut ctx).unwrap(); assert_eq!(result.len(), 3); @@ -466,11 +465,15 @@ mod test { Validity::NonNullable, ).unwrap())] fn test_take_list_conformance(#[case] list: ListArray) { - test_take_conformance(&list.into_array()); + test_take_conformance( + &list.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] fn test_u64_offset_accumulation_non_nullable() { + let mut ctx = array_session().create_execution_ctx(); let elements = buffer![0i32; 200].into_array(); let offsets = buffer![0u8, 200].into_array(); let list = ListArray::try_new(elements, offsets, Validity::NonNullable) @@ -483,8 +486,7 @@ mod test { assert_eq!(result.len(), 2); - #[expect(deprecated)] - let result_view = result.to_listview(); + let result_view = result.execute::(&mut ctx).unwrap(); assert_eq!(result_view.len(), 2); assert!( result_view @@ -500,6 +502,7 @@ mod test { #[test] fn test_u64_offset_accumulation_nullable() { + let mut ctx = array_session().create_execution_ctx(); let elements = buffer![0i32; 150].into_array(); let offsets = buffer![0u8, 150, 150].into_array(); let validity = BoolArray::from_iter(vec![true, false]).into_array(); @@ -513,8 +516,7 @@ mod test { assert_eq!(result.len(), 3); - #[expect(deprecated)] - let result_view = result.to_listview(); + let result_view = result.execute::(&mut ctx).unwrap(); assert_eq!(result_view.len(), 3); assert!( result_view diff --git a/vortex-array/src/arrays/list/test_harness.rs b/vortex-array/src/arrays/list/test_harness.rs index 42351484c19..ba2c9d335f1 100644 --- a/vortex-array/src/arrays/list/test_harness.rs +++ b/vortex-array/src/arrays/list/test_harness.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use itertools::Itertools; use vortex_error::VortexResult; -use crate::ArrayRef; use crate::arrays::ListArray; use crate::builders::ArrayBuilder; use crate::builders::ListBuilder; @@ -21,7 +20,7 @@ impl ListArray { pub fn from_iter_slow( iter: I, dtype: Arc, - ) -> VortexResult + ) -> VortexResult where I::Item: IntoIterator, ::Item: Into, @@ -42,13 +41,13 @@ impl ListArray { ); builder.append_value(elem.as_list())? } - Ok(builder.finish()) + Ok(builder.finish_into_list()) } pub fn from_iter_opt_slow>, T>( iter: I, dtype: Arc, - ) -> VortexResult + ) -> VortexResult where T: IntoIterator, T::Item: Into, @@ -73,6 +72,6 @@ impl ListArray { builder.append_null() } } - Ok(builder.finish()) + Ok(builder.finish_into_list()) } } diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index c71adadd8a0..faa91c459bc 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -34,6 +34,8 @@ use crate::arrays::list::array::SLOT_NAMES; use crate::arrays::list::compute::rules::PARENT_RULES; use crate::arrays::listview::list_view_from_list; use crate::buffer::BufferHandle; +use crate::builders::ArrayBuilder; +use crate::builders::ListBuilder; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; @@ -190,6 +192,17 @@ impl VTable for List { list_view_from_list(array, ctx)?.into_array(), )) } + + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let Some(builder) = builder.as_any_mut().downcast_mut::>() else { + vortex_bail!("append_to_builder for List requires a ListBuilder"); + }; + builder.append_list_array(&array.into_owned().into_array(), ctx) + } } #[derive(Clone, Debug)] diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index 1d6297ba7c1..95e7ce46c92 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -19,8 +19,6 @@ use crate::ArrayRef; use crate::ArraySlots; use crate::ExecutionCtx; use crate::LEGACY_SESSION; -#[expect(deprecated)] -use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::aggregate_fn::NumericalAggregateOpts; use crate::aggregate_fn::fns::min_max::min_max; @@ -259,10 +257,12 @@ impl ListViewData { // Skip host-only validation when offsets/sizes are not host-resident. if offsets.is_host() && sizes.is_host() { - #[expect(deprecated)] - let offsets_primitive = offsets.to_primitive(); - #[expect(deprecated)] - let sizes_primitive = sizes.to_primitive(); + let offsets_primitive = offsets + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())?; + let sizes_primitive = sizes + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())?; // Offsets and sizes are non-negative; reinterpret to unsigned to dispatch over 4 widths // each (4x4 instead of 8x8). This is a read-only validation, so result types are moot. let offsets_primitive = @@ -427,14 +427,6 @@ pub trait ListViewArrayExt: TypedArrayRef { self.elements().slice(offset..offset + size) } - fn verify_is_zero_copy_to_list(&self) -> bool { - #[expect(deprecated)] - let offsets_primitive = self.offsets().to_primitive(); - #[expect(deprecated)] - let sizes_primitive = self.sizes().to_primitive(); - validate_zctl(self.elements(), offsets_primitive, sizes_primitive).is_ok() - } - /// Returns a [`Mask`] of length `elements.len()` where each bit is set iff that /// position in `elements` is referenced by at least one view. Caller must ensure `elements` /// is non-empty. @@ -648,10 +640,16 @@ impl Array { /// See [`ListViewData::with_zero_copy_to_list`]. pub unsafe fn with_zero_copy_to_list(self, is_zctl: bool) -> Self { if cfg!(debug_assertions) && is_zctl { - #[expect(deprecated)] - let offsets_primitive = self.offsets().to_primitive(); - #[expect(deprecated)] - let sizes_primitive = self.sizes().to_primitive(); + let offsets_primitive = self + .offsets() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("offsets must canonicalize to primitive"); + let sizes_primitive = self + .sizes() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("sizes must canonicalize to primitive"); validate_zctl(self.elements(), offsets_primitive, sizes_primitive) .vortex_expect("Failed to validate zero-copy to list flag"); } diff --git a/vortex-array/src/arrays/listview/mod.rs b/vortex-array/src/arrays/listview/mod.rs index 80b2540a897..b240854c6cf 100644 --- a/vortex-array/src/arrays/listview/mod.rs +++ b/vortex-array/src/arrays/listview/mod.rs @@ -12,7 +12,7 @@ pub(crate) mod compute; mod vtable; pub use vtable::ListView; -pub(crate) fn initialize(session: &vortex_session::VortexSession) { +pub(crate) fn initialize(session: &VortexSession) { vtable::initialize(session); } @@ -25,6 +25,7 @@ mod rebuild; pub use rebuild::DEFAULT_REBUILD_DENSITY_THRESHOLD; pub use rebuild::DEFAULT_TRIM_ELEMENTS_THRESHOLD; pub use rebuild::ListViewRebuildMode; +use vortex_session::VortexSession; #[cfg(test)] mod tests; diff --git a/vortex-array/src/arrays/listview/rebuild.rs b/vortex-array/src/arrays/listview/rebuild.rs index bae2c4f3fb9..342c9eccf19 100644 --- a/vortex-array/src/arrays/listview/rebuild.rs +++ b/vortex-array/src/arrays/listview/rebuild.rs @@ -253,7 +253,7 @@ impl ListViewArray { } /// Rebuilds elements list-by-list: canonicalize elements upfront, then for each list `slice` - /// the relevant range and `extend_from_array` into a typed builder. + /// the relevant range and `append_to_builder` into a typed builder. fn rebuild_list_by_list( &self, ctx: &mut ExecutionCtx, @@ -317,7 +317,9 @@ impl ListViewArray { new_offsets.push(n_elements); new_sizes.push(size); - new_elements_builder.extend_from_array(&elements_canonical.slice(start..stop)?); + elements_canonical + .slice(start..stop)? + .append_to_builder(new_elements_builder.as_mut(), ctx)?; n_elements += num_traits::cast(size).vortex_expect("Cast failed"); } diff --git a/vortex-array/src/arrays/listview/tests/basic.rs b/vortex-array/src/arrays/listview/tests/basic.rs index 3d4d6db284d..f3082757ca0 100644 --- a/vortex-array/src/arrays/listview/tests/basic.rs +++ b/vortex-array/src/arrays/listview/tests/basic.rs @@ -401,9 +401,15 @@ fn test_verify_is_zero_copy_to_list() { let listview = ListViewArray::new(elements, offsets, sizes, Validity::NonNullable); - // Should return true since offsets are sorted and no overlaps exist. - assert!(listview.verify_is_zero_copy_to_list()); + // Marking as zero-copy-to-list validates the components and should succeed since offsets + // are sorted and no overlaps exist. + let zctl = unsafe { listview.with_zero_copy_to_list(true) }; + assert!(zctl.is_zero_copy_to_list()); +} +#[test] +#[should_panic(expected = "Zero-copy-to-list requires views to be non-overlapping and ordered")] +fn test_verify_is_zero_copy_to_list_overlapping() { // Create a ListView that is NOT zero-copyable to List due to overlapping views. // Logical lists: [[1,2], [2,3,4], [3,4]] let elements = buffer![1i32, 2, 3, 4, 5].into_array(); @@ -412,8 +418,10 @@ fn test_verify_is_zero_copy_to_list() { let listview = ListViewArray::new(elements, offsets, sizes, Validity::NonNullable); - // Should return false due to overlapping list views. - assert!(!listview.verify_is_zero_copy_to_list()); + // Validation should reject overlapping list views. + unsafe { + let _zctl = listview.with_zero_copy_to_list(true); + } } #[test] @@ -460,9 +468,6 @@ fn test_validate_monotonic_ends_correct_nulls() { let listview = ListViewArray::new(elements, offsets, sizes, validity); // Should be valid as zero-copy-to-list - this should NOT panic - let zctl_listview = unsafe { listview.clone().with_zero_copy_to_list(true) }; + let zctl_listview = unsafe { listview.with_zero_copy_to_list(true) }; assert!(zctl_listview.is_zero_copy_to_list()); - - // verify_is_zero_copy_to_list should also return true - assert!(listview.verify_is_zero_copy_to_list()); } diff --git a/vortex-array/src/arrays/listview/tests/filter.rs b/vortex-array/src/arrays/listview/tests/filter.rs index 25cc0de4aa8..e8cc7636b30 100644 --- a/vortex-array/src/arrays/listview/tests/filter.rs +++ b/vortex-array/src/arrays/listview/tests/filter.rs @@ -33,7 +33,7 @@ static SESSION: LazyLock = LazyLock::new(crate::array_session); #[case::overlapping(create_overlapping_listview())] #[case::large(create_large_listview())] fn test_filter_listview_conformance(#[case] listview: ListViewArray) { - test_filter_conformance(&listview.into_array()); + test_filter_conformance(&listview.into_array(), &mut SESSION.create_execution_ctx()); } #[test] diff --git a/vortex-array/src/arrays/listview/tests/operations.rs b/vortex-array/src/arrays/listview/tests/operations.rs index f49680a854e..51f03b1abeb 100644 --- a/vortex-array/src/arrays/listview/tests/operations.rs +++ b/vortex-array/src/arrays/listview/tests/operations.rs @@ -13,8 +13,6 @@ use super::common::create_large_listview; use super::common::create_nullable_listview; use crate::ArrayRef; use crate::IntoArray; -#[expect(deprecated)] -use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::is_constant::is_constant; use crate::array_session; @@ -258,8 +256,9 @@ fn test_cast_numeric_types(#[case] from_ptype: PType, #[case] to_ptype: PType) { let result = listview.cast(target_dtype.clone()).unwrap(); assert_eq!(result.dtype(), &target_dtype); - #[expect(deprecated)] - let result_list = result.to_listview(); + let result_list = result + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); assert!( result_list.len() == 3 || result_list.len() == 2, "Expected 2 or 3 lists" @@ -295,8 +294,9 @@ fn test_cast_with_nulls() { let result = listview.cast(target_dtype.clone()).unwrap(); assert_eq!(result.dtype(), &target_dtype); - #[expect(deprecated)] - let result_list = result.to_listview(); + let result_list = result + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); assert!( result_list .is_valid(0, &mut array_session().create_execution_ctx()) @@ -346,8 +346,9 @@ fn test_cast_special_patterns(#[case] expected_sizes: Vec, #[case] list_c }; let result = listview.cast(target_dtype).unwrap(); - #[expect(deprecated)] - let result_list = result.to_listview(); + let result_list = result + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); assert_eq!(result_list.len(), list_count); @@ -379,8 +380,9 @@ fn test_cast_large_dataset() { ); let result = listview.cast(target_dtype).unwrap(); - #[expect(deprecated)] - let result_list = result.to_listview(); + let result_list = result + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); assert_eq!(result_list.len(), 20); for i in 0..20 { @@ -622,7 +624,10 @@ fn test_constant_repeated_same_lists() { #[case::nullable(create_nullable_listview())] #[case::large(create_large_listview())] fn test_mask_listview_conformance(#[case] listview: ListViewArray) { - test_mask_conformance(&listview.into_array()); + test_mask_conformance( + &listview.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] @@ -644,8 +649,9 @@ fn test_mask_preserves_structure() { let result = listview.mask((!&selection).into_array()).unwrap(); assert_eq!(result.len(), 4); // Length is preserved. - #[expect(deprecated)] - let result_list = result.to_listview(); + let result_list = result + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); // Check validity: true in selection means null. assert!( @@ -698,8 +704,9 @@ fn test_mask_with_existing_nulls() { // Mask additional elements. let selection = Mask::from_iter([false, true, true]); let result = listview.mask((!&selection).into_array()).unwrap(); - #[expect(deprecated)] - let result_list = result.to_listview(); + let result_list = result + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); // Check combined validity: assert!( @@ -731,8 +738,9 @@ fn test_mask_with_gaps() { let selection = Mask::from_iter([true, false, false]); let result = listview.mask((!&selection).into_array()).unwrap(); - #[expect(deprecated)] - let result_list = result.to_listview(); + let result_list = result + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); assert_eq!(result_list.len(), 3); assert!( @@ -776,8 +784,9 @@ fn test_mask_constant_arrays() { let selection = Mask::from_iter([false, true, false]); let result = const_list.mask((!&selection).into_array()).unwrap(); - #[expect(deprecated)] - let result_list = result.to_listview(); + let result_list = result + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); assert_eq!(result_list.len(), 3); assert!( diff --git a/vortex-array/src/arrays/listview/tests/take.rs b/vortex-array/src/arrays/listview/tests/take.rs index 2a68f711d30..2af397e262f 100644 --- a/vortex-array/src/arrays/listview/tests/take.rs +++ b/vortex-array/src/arrays/listview/tests/take.rs @@ -32,7 +32,7 @@ static SESSION: LazyLock = LazyLock::new(crate::array_session); #[case::overlapping(create_overlapping_listview())] #[case::large(create_large_listview())] fn test_take_listview_conformance(#[case] listview: ListViewArray) { - test_take_conformance(&listview.into_array()); + test_take_conformance(&listview.into_array(), &mut SESSION.create_execution_ctx()); } // ListView-specific tests that aren't covered by conformance. diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index ba5ae42b3ad..95944bb99fe 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -33,6 +33,8 @@ use crate::arrays::listview::array::SIZES_SLOT; use crate::arrays::listview::array::SLOT_NAMES; use crate::arrays::listview::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; +use crate::builders::ArrayBuilder; +use crate::builders::ListViewBuilder; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; @@ -215,6 +217,21 @@ impl VTable for ListView { Ok(ExecutionResult::done(array)) } + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + // `builder_with_capacity` always produces a `ListViewBuilder` for `DType::List`. + let Some(builder) = builder + .as_any_mut() + .downcast_mut::>() + else { + vortex_bail!("append_to_builder for ListView requires a ListViewBuilder"); + }; + builder.append_listview_array(&array.into_owned(), ctx) + } + fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, diff --git a/vortex-array/src/arrays/masked/compute/filter.rs b/vortex-array/src/arrays/masked/compute/filter.rs index 9ea6b168105..287a996a12b 100644 --- a/vortex-array/src/arrays/masked/compute/filter.rs +++ b/vortex-array/src/arrays/masked/compute/filter.rs @@ -33,6 +33,8 @@ mod tests { use rstest::rstest; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::MaskedArray; use crate::arrays::PrimitiveArray; use crate::compute::conformance::filter::test_filter_conformance; @@ -58,6 +60,9 @@ mod tests { ).unwrap() )] fn test_filter_masked_conformance(#[case] array: MaskedArray) { - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/masked/compute/mask.rs b/vortex-array/src/arrays/masked/compute/mask.rs index 514d0aeaafc..e830f15bfa0 100644 --- a/vortex-array/src/arrays/masked/compute/mask.rs +++ b/vortex-array/src/arrays/masked/compute/mask.rs @@ -34,6 +34,8 @@ mod tests { use rstest::rstest; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::MaskedArray; use crate::arrays::PrimitiveArray; use crate::compute::conformance::mask::test_mask_conformance; @@ -59,6 +61,9 @@ mod tests { ).unwrap() )] fn test_mask_masked_conformance(#[case] array: MaskedArray) { - test_mask_conformance(&array.into_array()); + test_mask_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/masked/compute/take.rs b/vortex-array/src/arrays/masked/compute/take.rs index 1eff686b4d9..49ad0fc9fe6 100644 --- a/vortex-array/src/arrays/masked/compute/take.rs +++ b/vortex-array/src/arrays/masked/compute/take.rs @@ -44,6 +44,8 @@ mod tests { use rstest::rstest; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::MaskedArray; use crate::arrays::PrimitiveArray; use crate::compute::conformance::take::test_take_conformance; @@ -69,6 +71,9 @@ mod tests { ).unwrap() )] fn test_take_masked_conformance(#[case] array: MaskedArray) { - test_take_conformance(&array.into_array()); + test_take_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/masked/tests.rs b/vortex-array/src/arrays/masked/tests.rs index 60b045d521d..842599bfa52 100644 --- a/vortex-array/src/arrays/masked/tests.rs +++ b/vortex-array/src/arrays/masked/tests.rs @@ -8,8 +8,6 @@ use vortex_error::VortexResult; use super::*; use crate::Canonical; use crate::IntoArray; -#[expect(deprecated)] -use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::PrimitiveArray; @@ -63,11 +61,13 @@ fn test_masked_child_with_validity() { let array = MaskedArray::try_new(child, Validity::from_iter([true, false, true, false, true])).unwrap(); - #[expect(deprecated)] - let prim = array.as_array().to_primitive(); - // Positions where validity is false should be null in masked_child. let mut ctx = array_session().create_execution_ctx(); + let prim = array + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(prim.valid_count(&mut ctx).unwrap(), 3); assert!( prim.is_valid(0, &mut array_session().create_execution_ctx()) diff --git a/vortex-array/src/arrays/null/compute/cast.rs b/vortex-array/src/arrays/null/compute/cast.rs index a2527e5fb73..c1058156127 100644 --- a/vortex-array/src/arrays/null/compute/cast.rs +++ b/vortex-array/src/arrays/null/compute/cast.rs @@ -90,6 +90,9 @@ mod tests { #[case(NullArray::new(100))] #[case(NullArray::new(0))] fn test_cast_null_conformance(#[case] array: NullArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/null/compute/mod.rs b/vortex-array/src/arrays/null/compute/mod.rs index 5e756da27d0..73a23fe7a8d 100644 --- a/vortex-array/src/arrays/null/compute/mod.rs +++ b/vortex-array/src/arrays/null/compute/mod.rs @@ -15,8 +15,6 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::NullArray; @@ -29,8 +27,11 @@ mod test { #[test] fn test_slice_nulls() { let nulls = NullArray::new(10); - #[expect(deprecated)] - let sliced = nulls.slice(0..4).unwrap().to_null(); + let sliced = nulls + .slice(0..4) + .unwrap() + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); assert_eq!(sliced.len(), 4); let sliced_arr = sliced.as_array(); @@ -50,11 +51,11 @@ mod test { #[test] fn test_take_nulls() { let nulls = NullArray::new(10); - #[expect(deprecated)] let taken = nulls .take(buffer![0u64, 2, 4, 6, 8].into_array()) .unwrap() - .to_null(); + .execute::(&mut array_session().create_execution_ctx()) + .unwrap(); assert_eq!(taken.len(), 5); let taken_arr = taken.as_array(); @@ -81,21 +82,42 @@ mod test { #[test] fn test_filter_null_array() { - test_filter_conformance(&NullArray::new(5).into_array()); - test_filter_conformance(&NullArray::new(1).into_array()); - test_filter_conformance(&NullArray::new(10).into_array()); + test_filter_conformance( + &NullArray::new(5).into_array(), + &mut array_session().create_execution_ctx(), + ); + test_filter_conformance( + &NullArray::new(1).into_array(), + &mut array_session().create_execution_ctx(), + ); + test_filter_conformance( + &NullArray::new(10).into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] fn test_mask_null_array() { - test_mask_conformance(&NullArray::new(5).into_array()); + test_mask_conformance( + &NullArray::new(5).into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] fn test_take_null_array_conformance() { - test_take_conformance(&NullArray::new(5).into_array()); - test_take_conformance(&NullArray::new(1).into_array()); - test_take_conformance(&NullArray::new(10).into_array()); + test_take_conformance( + &NullArray::new(5).into_array(), + &mut array_session().create_execution_ctx(), + ); + test_take_conformance( + &NullArray::new(1).into_array(), + &mut array_session().create_execution_ctx(), + ); + test_take_conformance( + &NullArray::new(10).into_array(), + &mut array_session().create_execution_ctx(), + ); } #[rstest] diff --git a/vortex-array/src/arrays/null/compute/take.rs b/vortex-array/src/arrays/null/compute/take.rs index 5352ae67653..bd4837e82aa 100644 --- a/vortex-array/src/arrays/null/compute/take.rs +++ b/vortex-array/src/arrays/null/compute/take.rs @@ -6,11 +6,12 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::IntoArray; -#[expect(deprecated)] -use crate::ToCanonical as _; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::array::ArrayView; use crate::arrays::Null; use crate::arrays::NullArray; +use crate::arrays::PrimitiveArray; use crate::arrays::dict::TakeReduce; use crate::arrays::dict::TakeReduceAdaptor; use crate::match_each_integer_ptype; @@ -19,8 +20,9 @@ use crate::optimizer::rules::ParentRuleSet; impl TakeReduce for Null { #[expect(clippy::cast_possible_truncation)] fn take(array: ArrayView<'_, Null>, indices: &ArrayRef) -> VortexResult> { - #[expect(deprecated)] - let indices = indices.to_primitive(); + let indices = indices + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())?; // Enforce all indices are valid match_each_integer_ptype!(indices.ptype(), |T| { diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 78fe191b6a0..ce32cc89342 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_error::VortexResult; +use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; @@ -20,6 +21,8 @@ use crate::array::VTable; use crate::array::ValidityVTable; use crate::arrays::null::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; +use crate::builders::ArrayBuilder; +use crate::builders::NullBuilder; use crate::dtype::DType; use crate::scalar::Scalar; use crate::serde::ArrayChildren; @@ -109,6 +112,18 @@ impl VTable for Null { fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } + + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + _ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + vortex_bail!("append_to_builder for Null requires a NullBuilder"); + }; + builder.append_nulls(array.len()); + Ok(()) + } } /// A array where all values are null. diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index dbcb4c220dc..67183d1cdf5 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -201,8 +201,7 @@ impl VTable for Patched { .clone() .execute::(ctx)? .into_array(); - builder.extend_from_array(&canonical); - return Ok(()); + return canonical.append_to_builder(builder, ctx); } let ptype = dtype.as_ptype(); diff --git a/vortex-array/src/arrays/primitive/array/patch.rs b/vortex-array/src/arrays/primitive/array/patch.rs index d183552b66f..d4b4899077d 100644 --- a/vortex-array/src/arrays/primitive/array/patch.rs +++ b/vortex-array/src/arrays/primitive/array/patch.rs @@ -135,8 +135,6 @@ mod tests { use vortex_buffer::buffer; use super::*; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::assert_arrays_eq; @@ -178,8 +176,7 @@ mod tests { let mut ctx = array_session().create_execution_ctx(); let input = PrimitiveArray::new(buffer![2u32; 10], Validity::AllValid); let sliced = input.slice(2..8).unwrap(); - #[expect(deprecated)] - let sliced_primitive = sliced.to_primitive(); + let sliced_primitive = sliced.execute::(&mut ctx).unwrap(); assert_arrays_eq!( sliced_primitive, PrimitiveArray::new(buffer![2u32; 6], Validity::AllValid), diff --git a/vortex-array/src/arrays/primitive/array/top_value.rs b/vortex-array/src/arrays/primitive/array/top_value.rs index 7ae45c89405..a6233eb89ee 100644 --- a/vortex-array/src/arrays/primitive/array/top_value.rs +++ b/vortex-array/src/arrays/primitive/array/top_value.rs @@ -10,8 +10,7 @@ use vortex_mask::AllOr; use vortex_mask::Mask; use vortex_utils::aliases::hash_map::HashMap; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; +use crate::ExecutionCtx; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::NativeValue; use crate::dtype::NativePType; @@ -20,7 +19,7 @@ use crate::scalar::PValue; impl PrimitiveArray { /// Compute most common present value of this array - pub fn top_value(&self) -> VortexResult> { + pub fn top_value(&self, ctx: &mut ExecutionCtx) -> VortexResult> { if self.is_empty() { return Ok(None); } @@ -32,10 +31,9 @@ impl PrimitiveArray { match_each_native_ptype!(self.ptype(), |P| { let (top, count) = typed_top_value( self.as_slice::

(), - self.as_ref().validity()?.execute_mask( - self.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?, + self.as_ref() + .validity()? + .execute_mask(self.as_ref().len(), ctx)?, ); Ok(Some((top.into(), count))) }) diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index 36b95b4c554..09defa442b8 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -277,8 +277,6 @@ mod test { use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::Nullability; @@ -291,18 +289,21 @@ mod test { let arr = buffer![0u32, 10, 200].into_array(); // cast from u32 to u8 - #[expect(deprecated)] - let p = arr.cast(PType::U8.into()).unwrap().to_primitive(); + let p = arr + .cast(PType::U8.into()) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(p, PrimitiveArray::from_iter([0u8, 10, 200]), &mut ctx); assert!(matches!(p.validity(), Ok(Validity::NonNullable))); // to nullable - #[expect(deprecated)] let p = p .into_array() .cast(DType::Primitive(PType::U8, Nullability::Nullable)) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, PrimitiveArray::new(buffer![0u8, 10, 200], Validity::AllValid), @@ -311,22 +312,22 @@ mod test { assert!(matches!(p.validity(), Ok(Validity::AllValid))); // back to non-nullable - #[expect(deprecated)] let p = p .into_array() .cast(DType::Primitive(PType::U8, Nullability::NonNullable)) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(p, PrimitiveArray::from_iter([0u8, 10, 200]), &mut ctx); assert!(matches!(p.validity(), Ok(Validity::NonNullable))); // to nullable u32 - #[expect(deprecated)] let p = p .into_array() .cast(DType::Primitive(PType::U32, Nullability::Nullable)) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, PrimitiveArray::new(buffer![0u32, 10, 200], Validity::AllValid), @@ -335,12 +336,12 @@ mod test { assert!(matches!(p.validity(), Ok(Validity::AllValid))); // to non-nullable u8 - #[expect(deprecated)] let p = p .into_array() .cast(DType::Primitive(PType::U8, Nullability::NonNullable)) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(p, PrimitiveArray::from_iter([0u8, 10, 200]), &mut ctx); assert!(matches!(p.validity(), Ok(Validity::NonNullable))); } @@ -349,8 +350,11 @@ mod test { fn cast_u32_f32() { let mut ctx = array_session().create_execution_ctx(); let arr = buffer![0u32, 10, 200].into_array(); - #[expect(deprecated)] - let u8arr = arr.cast(PType::F32.into()).unwrap().to_primitive(); + let u8arr = arr + .cast(PType::F32.into()) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( u8arr, PrimitiveArray::from_iter([0.0f32, 10., 200.]), @@ -394,12 +398,12 @@ mod test { buffer![-1i32, 0, 10], Validity::from_iter([false, true, true]), ); - #[expect(deprecated)] let p = arr .into_array() .cast(DType::Primitive(PType::U32, Nullability::Nullable)) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, PrimitiveArray::from_option_iter([None, Some(0u32), Some(10)]), @@ -426,8 +430,10 @@ mod test { let src = PrimitiveArray::from_iter([0u32, 10, 100]); let src_ptr = src.as_slice::().as_ptr(); - #[expect(deprecated)] - let dst = src.into_array().cast(PType::I32.into())?.to_primitive(); + let dst = src + .into_array() + .cast(PType::I32.into())? + .execute::(&mut ctx)?; let dst_ptr = dst.as_slice::().as_ptr(); // Zero-copy: the data pointer should be identical. @@ -453,12 +459,12 @@ mod test { /// touching the buffer contents. #[test] fn cast_same_width_all_null() -> vortex_error::VortexResult<()> { + let mut ctx = array_session().create_execution_ctx(); let arr = PrimitiveArray::new(buffer![0xFFu8, 0xFF], Validity::AllInvalid); - #[expect(deprecated)] let casted = arr .into_array() .cast(DType::Primitive(PType::I8, Nullability::Nullable))? - .to_primitive(); + .execute::(&mut ctx)?; assert_eq!(casted.len(), 2); assert!(matches!(casted.validity(), Ok(Validity::AllInvalid))); Ok(()) @@ -475,11 +481,10 @@ mod test { buffer![u32::MAX, 0u32, 42u32], Validity::from_iter([false, true, true]), ); - #[expect(deprecated)] let casted = arr .into_array() .cast(DType::Primitive(PType::I32, Nullability::Nullable))? - .to_primitive(); + .execute::(&mut ctx)?; assert_arrays_eq!( casted, PrimitiveArray::from_option_iter([None, Some(0i32), Some(42)]), @@ -495,11 +500,10 @@ mod test { buffer![1000u32, 10u32, 42u32], Validity::from_iter([false, true, true]), ); - #[expect(deprecated)] let casted = arr .into_array() .cast(DType::Primitive(PType::U8, Nullability::Nullable))? - .to_primitive(); + .execute::(&mut ctx)?; assert_arrays_eq!( casted, PrimitiveArray::from_option_iter([None, Some(10u8), Some(42)]), @@ -525,6 +529,6 @@ mod test { #[case(PrimitiveArray::from_option_iter([Some(1i32), None, Some(-100), Some(0), None]).into_array())] #[case(buffer![42u32].into_array())] fn test_cast_primitive_conformance(#[case] array: ArrayRef) { - test_cast_conformance(&array); + test_cast_conformance(&array, &mut array_session().create_execution_ctx()); } } diff --git a/vortex-array/src/arrays/primitive/compute/fill_null.rs b/vortex-array/src/arrays/primitive/compute/fill_null.rs index d73442a150e..e8b0d48b322 100644 --- a/vortex-array/src/arrays/primitive/compute/fill_null.rs +++ b/vortex-array/src/arrays/primitive/compute/fill_null.rs @@ -57,8 +57,6 @@ mod test { use crate::arrays::primitive::compute::fill_null::BoolArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::scalar::Scalar; use crate::validity::Validity; @@ -66,12 +64,12 @@ mod test { fn fill_null_leading_none() { let mut ctx = array_session().create_execution_ctx(); let arr = PrimitiveArray::from_option_iter([None, Some(8u8), None, Some(10), None]); - #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::from(42u8)) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, PrimitiveArray::from_iter([42u8, 8, 42, 10, 42]), @@ -95,12 +93,12 @@ mod test { let mut ctx = array_session().create_execution_ctx(); let arr = PrimitiveArray::from_option_iter([Option::::None, None, None, None, None]); - #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::from(255u8)) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, PrimitiveArray::from_iter([255u8, 255, 255, 255, 255]), @@ -126,12 +124,12 @@ mod test { buffer![8u8, 10, 12, 14, 16], Validity::Array(BoolArray::from_iter([true, true, true, true, true]).into_array()), ); - #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::from(255u8)) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, PrimitiveArray::from_iter([8u8, 10, 12, 14, 16]), @@ -154,8 +152,11 @@ mod test { fn fill_null_non_nullable() { let mut ctx = array_session().create_execution_ctx(); let arr = buffer![8u8, 10, 12, 14, 16].into_array(); - #[expect(deprecated)] - let p = arr.fill_null(Scalar::from(255u8)).unwrap().to_primitive(); + let p = arr + .fill_null(Scalar::from(255u8)) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( p, PrimitiveArray::from_iter([8u8, 10, 12, 14, 16]), diff --git a/vortex-array/src/arrays/primitive/compute/mask.rs b/vortex-array/src/arrays/primitive/compute/mask.rs index 6768efd6fbf..023310b3807 100644 --- a/vortex-array/src/arrays/primitive/compute/mask.rs +++ b/vortex-array/src/arrays/primitive/compute/mask.rs @@ -30,6 +30,8 @@ mod test { use rstest::rstest; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::PrimitiveArray; use crate::compute::conformance::mask::test_mask_conformance; @@ -42,6 +44,9 @@ mod test { #[case(PrimitiveArray::from_iter([0.1f32, 0.2, 0.3, 0.4, 0.5]))] #[case(PrimitiveArray::from_option_iter([Some(1.1f64), None, Some(2.2), Some(3.3), None]))] fn test_mask_primitive_conformance(#[case] array: PrimitiveArray) { - test_mask_conformance(&array.into_array()); + test_mask_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/primitive/compute/take/mod.rs b/vortex-array/src/arrays/primitive/compute/take/mod.rs index 9294659708e..dd562281608 100644 --- a/vortex-array/src/arrays/primitive/compute/take/mod.rs +++ b/vortex-array/src/arrays/primitive/compute/take/mod.rs @@ -170,6 +170,7 @@ mod test { #[test] fn test_take_with_null_indices() { + let mut ctx = array_session().create_execution_ctx(); let values = PrimitiveArray::new( buffer![1i32, 2, 3, 4, 5], Validity::Array(BoolArray::from_iter([true, true, false, false, true]).into_array()), @@ -180,23 +181,17 @@ mod test { ); let actual = values.take(indices.into_array()).unwrap(); assert_eq!( - actual - .execute_scalar(0, &mut array_session().create_execution_ctx()) - .vortex_expect("no fail"), + actual.execute_scalar(0, &mut ctx).vortex_expect("no fail"), Scalar::from(Some(1)) ); // position 3 is null assert_eq!( - actual - .execute_scalar(1, &mut array_session().create_execution_ctx()) - .vortex_expect("no fail"), + actual.execute_scalar(1, &mut ctx).vortex_expect("no fail"), Scalar::null_native::() ); // the third index is null assert_eq!( - actual - .execute_scalar(2, &mut array_session().create_execution_ctx()) - .vortex_expect("no fail"), + actual.execute_scalar(2, &mut ctx).vortex_expect("no fail"), Scalar::null_native::() ); } @@ -213,7 +208,10 @@ mod test { ))] #[case(PrimitiveArray::from_option_iter([Some(1), None, Some(3), Some(4), None]))] fn test_take_primitive_conformance(#[case] array: PrimitiveArray) { - test_take_conformance(&array.into_array()); + test_take_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/primitive/tests.rs b/vortex-array/src/arrays/primitive/tests.rs index f623479c7b0..6e0758c9621 100644 --- a/vortex-array/src/arrays/primitive/tests.rs +++ b/vortex-array/src/arrays/primitive/tests.rs @@ -5,6 +5,8 @@ use vortex_buffer::buffer; use crate::ArrayRef; use crate::IntoArray; +use crate::VortexSessionExecute; +use crate::array_session; use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; use crate::compute::conformance::filter::test_filter_conformance; @@ -36,12 +38,15 @@ fn test_search_sorted_primitive( fn test_mask_primitive_array() { test_mask_conformance( &PrimitiveArray::new(buffer![0, 1, 2, 3, 4], Validity::NonNullable).into_array(), + &mut array_session().create_execution_ctx(), ); test_mask_conformance( &PrimitiveArray::new(buffer![0, 1, 2, 3, 4], Validity::AllValid).into_array(), + &mut array_session().create_execution_ctx(), ); test_mask_conformance( &PrimitiveArray::new(buffer![0, 1, 2, 3, 4], Validity::AllInvalid).into_array(), + &mut array_session().create_execution_ctx(), ); test_mask_conformance( &PrimitiveArray::new( @@ -49,6 +54,7 @@ fn test_mask_primitive_array() { Validity::Array(BoolArray::from_iter([true, false, true, false, true]).into_array()), ) .into_array(), + &mut array_session().create_execution_ctx(), ); } @@ -57,20 +63,25 @@ fn test_filter_primitive_array() { // Test various sizes test_filter_conformance( &PrimitiveArray::new(buffer![42i32], Validity::NonNullable).into_array(), + &mut array_session().create_execution_ctx(), ); test_filter_conformance( &PrimitiveArray::new(buffer![0, 1], Validity::NonNullable).into_array(), + &mut array_session().create_execution_ctx(), ); test_filter_conformance( &PrimitiveArray::new(buffer![0, 1, 2, 3, 4], Validity::NonNullable).into_array(), + &mut array_session().create_execution_ctx(), ); test_filter_conformance( &PrimitiveArray::new(buffer![0, 1, 2, 3, 4, 5, 6, 7], Validity::NonNullable).into_array(), + &mut array_session().create_execution_ctx(), ); // Test with validity test_filter_conformance( &PrimitiveArray::new(buffer![0, 1, 2, 3, 4], Validity::AllValid).into_array(), + &mut array_session().create_execution_ctx(), ); test_filter_conformance( &PrimitiveArray::new( @@ -80,5 +91,6 @@ fn test_filter_primitive_array() { ), ) .into_array(), + &mut array_session().create_execution_ctx(), ); } diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index b5eba2cb1a0..39aa0e32f81 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -201,8 +201,7 @@ impl VTable for Primitive { } }); - builder.extend_from_array(array.as_ref()); - Ok(()) + vortex_bail!("append_to_builder for Primitive requires a matching PrimitiveBuilder"); } fn reduce_parent( diff --git a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs index 2ac376155e3..dfeb732be35 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs @@ -2,13 +2,16 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_error::VortexResult; +use vortex_error::vortex_bail; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::ConstantArray; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::vtable::ArrayExpr; use crate::arrays::scalar_fn::vtable::FakeEq; @@ -24,30 +27,32 @@ use crate::validity::Validity; /// Execute an expression tree recursively. /// /// This assumes all leaf expressions are either ArrayExpr (wrapping actual arrays) or Literals. -fn execute_expr(expr: &Expression, row_count: usize) -> VortexResult { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - +fn execute_expr( + expr: &Expression, + row_count: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult { // Handle Root expression - this should not happen in validity expressions if expr.is::() { - vortex_error::vortex_bail!("Root expression cannot be executed in validity context"); + vortex_bail!("Root expression cannot be executed in validity context"); } // Handle Literal expression - create a constant array if expr.is::() { let scalar = expr.as_::(); - return Ok(crate::arrays::ConstantArray::new(scalar.clone(), row_count).into_array()); + return Ok(ConstantArray::new(scalar.clone(), row_count).into_array()); } // Recursively execute child expressions to get input arrays let inputs: Vec = expr .children() .iter() - .map(|child| execute_expr(child, row_count)) + .map(|child| execute_expr(child, row_count, ctx)) .collect::>()?; let args = VecExecutionArgs::new(inputs, row_count); - Ok(expr.scalar_fn().execute(&args, &mut ctx)?.into_array()) + Ok(expr.scalar_fn().execute(&args, ctx)?.into_array()) } impl ValidityVTable for ScalarFn { @@ -68,7 +73,12 @@ impl ValidityVTable for ScalarFn { let expr = Expression::try_new(array.scalar_fn().clone(), inputs)?; let validity_expr = array.scalar_fn().validity(&expr)?; + let ctx = &mut LEGACY_SESSION.create_execution_ctx(); // Execute the validity expression. All leaves are ArrayExpr nodes. - Ok(Validity::Array(execute_expr(&validity_expr, array.len())?)) + Ok(Validity::Array(execute_expr( + &validity_expr, + array.len(), + ctx, + )?)) } } diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index ac14983f0e2..7a2539a8740 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -189,7 +189,7 @@ mod tests { #[case(create_nested_struct())] #[case(create_simple_struct())] fn test_cast_struct_conformance(#[case] array: StructArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] diff --git a/vortex-array/src/arrays/struct_/compute/mod.rs b/vortex-array/src/arrays/struct_/compute/mod.rs index c1539a638bf..4b9a4e396fd 100644 --- a/vortex-array/src/arrays/struct_/compute/mod.rs +++ b/vortex-array/src/arrays/struct_/compute/mod.rs @@ -117,6 +117,7 @@ mod tests { &StructArray::try_new(FieldNames::empty(), vec![], 5, Validity::NonNullable) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } @@ -151,6 +152,7 @@ mod tests { ) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } @@ -270,6 +272,7 @@ mod tests { &StructArray::try_new(FieldNames::empty(), vec![], 5, Validity::NonNullable) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } @@ -286,6 +289,7 @@ mod tests { &StructArray::try_new(["xs", "ys"].into(), vec![xs, ys], 5, Validity::NonNullable) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } @@ -307,6 +311,7 @@ mod tests { ) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } @@ -335,6 +340,7 @@ mod tests { ) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } @@ -347,6 +353,7 @@ mod tests { &StructArray::try_new(["xs", "ys"].into(), vec![xs, ys], 1, Validity::NonNullable) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } @@ -370,6 +377,7 @@ mod tests { ) .unwrap() .into_array(), + &mut array_session().create_execution_ctx(), ); } diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index df9b1170b02..5488f91baa7 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -22,6 +22,8 @@ use crate::arrays::struct_::array::VALIDITY_SLOT; use crate::arrays::struct_::array::make_struct_slots; use crate::arrays::struct_::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; +use crate::builders::ArrayBuilder; +use crate::builders::StructBuilder; use crate::dtype::DType; use crate::serde::ArrayChildren; use crate::validity::Validity; @@ -188,6 +190,17 @@ impl VTable for Struct { Ok(ExecutionResult::done(array)) } + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + vortex_bail!("append_to_builder for Struct requires a StructBuilder"); + }; + builder.append_struct_array(&array.into_owned(), ctx) + } + fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, diff --git a/vortex-array/src/arrays/varbin/compute/cast.rs b/vortex-array/src/arrays/varbin/compute/cast.rs index 5983886ea30..b0c4325c774 100644 --- a/vortex-array/src/arrays/varbin/compute/cast.rs +++ b/vortex-array/src/arrays/varbin/compute/cast.rs @@ -130,6 +130,6 @@ mod tests { #[case(VarBinArray::from_iter(vec![Some(b"test".as_slice()), None], DType::Binary(Nullability::Nullable)))] #[case(VarBinArray::from_iter(vec![Some("single")], DType::Utf8(Nullability::NonNullable)))] fn test_cast_varbin_conformance(#[case] array: VarBinArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index ede3c409d94..42876a92aca 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -159,10 +159,9 @@ mod test { use vortex_buffer::ByteBuffer; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; + use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::VarBinArray; use crate::arrays::VarBinViewArray; @@ -175,11 +174,11 @@ mod test { #[test] fn test_binary_compare() { + let mut ctx = array_session().create_execution_ctx(); let array = VarBinArray::from_iter( [Some(b"abc".to_vec()), None, Some(b"def".to_vec())], DType::Binary(Nullability::Nullable), ); - #[expect(deprecated)] let result = array .into_array() .binary( @@ -191,17 +190,15 @@ mod test { Operator::Eq, ) .unwrap() - .to_bool(); + .execute::(&mut ctx) + .unwrap(); assert_eq!( &result .as_ref() .validity() .unwrap() - .execute_mask( - result.as_ref().len(), - &mut array_session().create_execution_ctx() - ) + .execute_mask(result.as_ref().len(), &mut ctx) .unwrap() .to_bit_buffer(), &BitBuffer::from_iter([true, false, true]) @@ -214,6 +211,7 @@ mod test { #[test] fn varbinview_compare() { + let mut ctx = array_session().create_execution_ctx(); let array = VarBinArray::from_iter( [Some(b"abc".to_vec()), None, Some(b"def".to_vec())], DType::Binary(Nullability::Nullable), @@ -222,22 +220,19 @@ mod test { [None, None, Some(b"def".to_vec())], DType::Binary(Nullability::Nullable), ); - #[expect(deprecated)] let result = array .into_array() .binary(vbv.into_array(), Operator::Eq) .unwrap() - .to_bool(); + .execute::(&mut ctx) + .unwrap(); assert_eq!( result .as_ref() .validity() .unwrap() - .execute_mask( - result.as_ref().len(), - &mut array_session().create_execution_ctx() - ) + .execute_mask(result.as_ref().len(), &mut ctx) .unwrap() .to_bit_buffer(), BitBuffer::from_iter([false, false, true]) diff --git a/vortex-array/src/arrays/varbin/compute/filter.rs b/vortex-array/src/arrays/varbin/compute/filter.rs index ab969cf3def..400beb9aa0a 100644 --- a/vortex-array/src/arrays/varbin/compute/filter.rs +++ b/vortex-array/src/arrays/varbin/compute/filter.rs @@ -360,12 +360,18 @@ mod test { vec!["hello", "world", "filter", "good", "bye"], DType::Utf8(NonNullable), ); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); let array = VarBinArray::from_iter( vec![Some("hello"), None, Some("filter"), Some("good"), None], DType::Utf8(Nullable), ); - test_filter_conformance(&array.into_array()); + test_filter_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/varbin/compute/mask.rs b/vortex-array/src/arrays/varbin/compute/mask.rs index 1c5567adea7..19236cee47b 100644 --- a/vortex-array/src/arrays/varbin/compute/mask.rs +++ b/vortex-array/src/arrays/varbin/compute/mask.rs @@ -29,6 +29,8 @@ impl MaskReduce for VarBin { #[cfg(test)] mod test { use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::VarBinArray; use crate::compute::conformance::mask::test_mask_conformance; use crate::dtype::DType; @@ -40,12 +42,18 @@ mod test { vec!["hello", "world", "filter", "good", "bye"], DType::Utf8(Nullability::NonNullable), ); - test_mask_conformance(&array.into_array()); + test_mask_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); let array = VarBinArray::from_iter( vec![Some("hello"), None, Some("filter"), Some("good"), None], DType::Utf8(Nullability::Nullable), ); - test_mask_conformance(&array.into_array()); + test_mask_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/varbin/compute/take.rs b/vortex-array/src/arrays/varbin/compute/take.rs index 74e0bd6af9d..6a6454d0603 100644 --- a/vortex-array/src/arrays/varbin/compute/take.rs +++ b/vortex-array/src/arrays/varbin/compute/take.rs @@ -303,7 +303,10 @@ mod tests { ))] #[case(VarBinArray::from_iter(["single"].map(Some), DType::Utf8(Nullability::NonNullable)))] fn test_take_varbin_conformance(#[case] array: VarBinArray) { - test_take_conformance(&array.into_array()); + test_take_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } #[test] diff --git a/vortex-array/src/arrays/varbin/vtable/canonical.rs b/vortex-array/src/arrays/varbin/vtable/canonical.rs index 4865df5e1a3..fca9671e3b4 100644 --- a/vortex-array/src/arrays/varbin/vtable/canonical.rs +++ b/vortex-array/src/arrays/varbin/vtable/canonical.rs @@ -53,8 +53,6 @@ mod tests { use crate::arrays::VarBinViewArray; use crate::arrays::varbin::builder::VarBinBuilder; use crate::assert_arrays_eq; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; @@ -73,8 +71,8 @@ mod tests { let varbin = varbin.slice(1..4).unwrap(); - #[expect(deprecated)] - let canonical = varbin.to_varbinview(); + let mut ctx = array_session().create_execution_ctx(); + let canonical = varbin.execute::(&mut ctx).unwrap(); assert_eq!(canonical.dtype(), &dtype); assert!( @@ -98,8 +96,11 @@ mod tests { fn test_canonical_varbin_unsliced(#[case] dtype: DType) { let mut ctx = array_session().create_execution_ctx(); let varbin = VarBinArray::from_iter_nonnull(["foo", "bar", "baz"], dtype.clone()); - #[expect(deprecated)] - let canonical = varbin.as_array().to_varbinview(); + let canonical = varbin + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected = match dtype { DType::Utf8(_) => VarBinViewArray::from_iter_str(["foo", "bar", "baz"]), _ => VarBinViewArray::from_iter_bin(["foo", "bar", "baz"]), @@ -112,8 +113,12 @@ mod tests { fn test_canonical_varbin_empty() { let varbin = VarBinArray::from_iter_nonnull([] as [&str; 0], DType::Utf8(Nullability::NonNullable)); - #[expect(deprecated)] - let canonical = varbin.as_array().to_varbinview(); + let mut ctx = array_session().create_execution_ctx(); + let canonical = varbin + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(canonical.len(), 0); } } diff --git a/vortex-array/src/arrays/varbinview/compact.rs b/vortex-array/src/arrays/varbinview/compact.rs index fc3f3477ffc..2aba21ae448 100644 --- a/vortex-array/src/arrays/varbinview/compact.rs +++ b/vortex-array/src/arrays/varbinview/compact.rs @@ -10,12 +10,9 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; -use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; +use crate::ExecutionCtx; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::Ref; -use crate::builders::ArrayBuilder; use crate::builders::VarBinViewBuilder; const DEFAULT_COMPACTION_THRESHOLD: f64 = 0.5; @@ -30,16 +27,16 @@ impl VarBinViewArray { /// that are no longer visible. We detect when there is wasted space in any of the buffers, and if /// so, will aggressively compact all visible outlined string data into new buffers while keeping /// well-utilized buffers unchanged. - pub fn compact_buffers(&self) -> VortexResult { + pub fn compact_buffers(&self, ctx: &mut ExecutionCtx) -> VortexResult { // If there is nothing to be gained by compaction, return the original array untouched. - if !self.should_compact()? { + if !self.should_compact(ctx)? { return Ok(self.clone()); } - self.compact_with_threshold(DEFAULT_COMPACTION_THRESHOLD) + self.compact_with_threshold(DEFAULT_COMPACTION_THRESHOLD, ctx) } - fn should_compact(&self) -> VortexResult { + fn should_compact(&self, ctx: &mut ExecutionCtx) -> VortexResult { let nbuffers = self.data_buffers().len(); // If the array is entirely inlined strings, do not attempt to compact. @@ -62,21 +59,22 @@ impl VarBinViewArray { return Ok(false); } - let bytes_referenced: u64 = self.count_referenced_bytes()?; + let bytes_referenced: u64 = self.count_referenced_bytes(ctx)?; Ok((bytes_referenced as f64 / buffer_total_bytes as f64) < DEFAULT_COMPACTION_THRESHOLD) } /// Iterates over all valid, non-inlined views, calling the provided /// closure for each one. #[inline(always)] - fn iter_valid_views(&self, mut f: F) -> VortexResult<()> + fn iter_valid_views(&self, ctx: &mut ExecutionCtx, mut f: F) -> VortexResult<()> where F: FnMut(&Ref), { - match self.as_ref().validity()?.execute_mask( - self.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )? { + match self + .as_ref() + .validity()? + .execute_mask(self.as_ref().len(), ctx)? + { Mask::AllTrue(_) => { for &view in self.views().iter() { if !view.is_inlined() { @@ -98,13 +96,16 @@ impl VarBinViewArray { /// Count the number of bytes addressed by the views, not including null /// values or any inlined strings. - fn count_referenced_bytes(&self) -> VortexResult { + fn count_referenced_bytes(&self, ctx: &mut ExecutionCtx) -> VortexResult { let mut total = 0u64; - self.iter_valid_views(|view| total += view.size as u64)?; + self.iter_valid_views(ctx, |view| total += view.size as u64)?; Ok(total) } - pub(crate) fn buffer_utilizations(&self) -> VortexResult> { + pub(crate) fn buffer_utilizations( + &self, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { let mut utilizations: Vec = self .data_buffers() .iter() @@ -114,7 +115,7 @@ impl VarBinViewArray { }) .collect(); - self.iter_valid_views(|view| { + self.iter_valid_views(ctx, |view| { utilizations[view.buffer_index as usize].add(view.offset, view.size); })?; @@ -138,13 +139,14 @@ impl VarBinViewArray { pub fn compact_with_threshold( &self, buffer_utilization_threshold: f64, // [0, 1] + ctx: &mut ExecutionCtx, ) -> VortexResult { let mut builder = VarBinViewBuilder::with_compaction( self.dtype().clone(), self.len(), buffer_utilization_threshold, ); - builder.extend_from_array(&self.clone().into_array()); + builder.append_varbinview_array(self, ctx)?; Ok(builder.finish_into_varbinview()) } } @@ -227,14 +229,12 @@ mod tests { // Take only the first and last elements (indices 0 and 4) let indices = buffer![0u32, 4u32].into_array(); let taken = original.take(indices).unwrap(); - let taken = taken - .execute::(&mut array_session().create_execution_ctx()) - .unwrap(); + let taken = taken.execute::(&mut ctx).unwrap(); // The taken array should still have the same number of buffers assert_eq!(taken.data_buffers().len(), original_buffers); // Now optimize the taken array - let optimized_array = taken.compact_buffers().unwrap(); + let optimized_array = taken.compact_buffers(&mut ctx).unwrap(); // The optimized array should have compacted buffers // Since both remaining strings are short, they should be inlined @@ -272,7 +272,7 @@ mod tests { .execute::(&mut array_session().create_execution_ctx()) .unwrap(); - let optimized_array = taken_array.compact_with_threshold(1.0).unwrap(); + let optimized_array = taken_array.compact_with_threshold(1.0, &mut ctx).unwrap(); // The optimized array should have exactly 1 buffer (consolidated) assert_eq!(optimized_array.data_buffers().len(), 1); @@ -295,7 +295,7 @@ mod tests { assert_eq!(original.data_buffers().len(), 0); // Optimize should return the same array - let optimized_array = original.compact_buffers().unwrap(); + let optimized_array = original.compact_buffers(&mut ctx).unwrap(); assert_eq!(optimized_array.data_buffers().len(), 0); @@ -315,7 +315,7 @@ mod tests { assert_eq!(original.buffer(0).len(), str1.len() + str2.len()); // Optimize should return the same array (no change needed) - let optimized_array = original.compact_buffers().unwrap(); + let optimized_array = original.compact_buffers(&mut ctx).unwrap(); assert_eq!(optimized_array.data_buffers().len(), 1); @@ -341,7 +341,7 @@ mod tests { .execute::(&mut array_session().create_execution_ctx()) .unwrap(); // Compact with threshold=0 (should not compact) - let compacted = taken.compact_with_threshold(0.0).unwrap(); + let compacted = taken.compact_with_threshold(0.0, &mut ctx).unwrap(); // Should still have the same number of buffers as the taken array assert_eq!(compacted.data_buffers().len(), taken.data_buffers().len()); @@ -370,7 +370,7 @@ mod tests { let original_buffers = taken.data_buffers().len(); // Compact with threshold=1.0 (aggressive compaction) - let compacted = taken.compact_with_threshold(1.0).unwrap(); + let compacted = taken.compact_with_threshold(1.0, &mut ctx).unwrap(); // Should have compacted buffers assert!(compacted.data_buffers().len() <= original_buffers); @@ -393,7 +393,7 @@ mod tests { assert_eq!(original.data_buffers().len(), 1); // Compact with high threshold - let compacted = original.compact_with_threshold(0.8).unwrap(); + let compacted = original.compact_with_threshold(0.8, &mut ctx).unwrap(); // Well-utilized buffer should be preserved assert_eq!(compacted.data_buffers().len(), 1); @@ -425,7 +425,7 @@ mod tests { .unwrap(); // Compact with moderate threshold - let compacted = taken.compact_with_threshold(0.7).unwrap(); + let compacted = taken.compact_with_threshold(0.7, &mut ctx).unwrap(); let expected = VarBinViewArray::from_iter( [0, 2, 4, 6, 8].map(|i| Some(strings[i].as_str())), @@ -451,12 +451,12 @@ mod tests { .execute::(&mut array_session().create_execution_ctx()) .unwrap(); // Get buffer stats before compaction - let utils_before = taken.buffer_utilizations().unwrap(); + let utils_before = taken.buffer_utilizations(&mut ctx).unwrap(); let original_buffer_count = taken.data_buffers().len(); // Compact with a threshold that should trigger slicing // The range utilization should be high even if overall utilization is low - let compacted = taken.compact_with_threshold(0.8).unwrap(); + let compacted = taken.compact_with_threshold(0.8, &mut ctx).unwrap(); // After compaction, we should still have buffers (sliced, not rewritten) assert!( @@ -497,9 +497,13 @@ mod tests { #[case] expected_bytes: u64, #[case] expected_utils: &[f64], ) { - assert_eq!(arr.count_referenced_bytes().unwrap(), expected_bytes); + let mut ctx = array_session().create_execution_ctx(); + assert_eq!( + arr.count_referenced_bytes(&mut ctx).unwrap(), + expected_bytes + ); let utils: Vec = arr - .buffer_utilizations() + .buffer_utilizations(&mut ctx) .unwrap() .iter() .map(|u| u.overall_utilization()) diff --git a/vortex-array/src/arrays/varbinview/compute/cast.rs b/vortex-array/src/arrays/varbinview/compute/cast.rs index 476c74e7309..660a7842ed9 100644 --- a/vortex-array/src/arrays/varbinview/compute/cast.rs +++ b/vortex-array/src/arrays/varbinview/compute/cast.rs @@ -135,6 +135,6 @@ mod tests { #[case(VarBinViewArray::from_iter(vec![Some("single")], DType::Utf8(Nullability::NonNullable)))] #[case(VarBinViewArray::from_iter(vec![Some("very long string that exceeds the inline size to test view functionality with multiple buffers")], DType::Utf8(Nullability::NonNullable)))] fn test_cast_varbinview_conformance(#[case] array: VarBinViewArray) { - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/vortex-array/src/arrays/varbinview/compute/mask.rs b/vortex-array/src/arrays/varbinview/compute/mask.rs index 6172c42ab7d..e470dceba32 100644 --- a/vortex-array/src/arrays/varbinview/compute/mask.rs +++ b/vortex-array/src/arrays/varbinview/compute/mask.rs @@ -33,6 +33,8 @@ impl MaskReduce for VarBinView { #[cfg(test)] mod tests { use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::VarBinViewArray; use crate::compute::conformance::mask::test_mask_conformance; @@ -40,6 +42,7 @@ mod tests { fn take_mask_var_bin_view_array() { test_mask_conformance( &VarBinViewArray::from_iter_str(["one", "two", "three", "four", "five"]).into_array(), + &mut array_session().create_execution_ctx(), ); test_mask_conformance( @@ -51,6 +54,7 @@ mod tests { Some("five"), ]) .into_array(), + &mut array_session().create_execution_ctx(), ); } } diff --git a/vortex-array/src/arrays/varbinview/compute/mod.rs b/vortex-array/src/arrays/varbinview/compute/mod.rs index 0358ad01460..e25b6eafb53 100644 --- a/vortex-array/src/arrays/varbinview/compute/mod.rs +++ b/vortex-array/src/arrays/varbinview/compute/mod.rs @@ -15,8 +15,6 @@ mod tests { use crate::IntoArray; use crate::arrays::VarBinViewArray; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; #[test] fn take_nullable() -> VortexResult<()> { let arr = VarBinViewArray::from_iter_nullable_str([ @@ -32,8 +30,7 @@ mod tests { assert!(taken.dtype().is_nullable()); let mut ctx = array_session().create_execution_ctx(); - #[expect(deprecated)] - let taken = taken.to_varbinview(); + let taken = taken.execute::(&mut ctx)?; let mask = taken.validity()?.execute_mask(taken.len(), &mut ctx)?; let result = (0..taken.len()) .map(|i| { diff --git a/vortex-array/src/arrays/varbinview/compute/take.rs b/vortex-array/src/arrays/varbinview/compute/take.rs index 1b7435729f1..ce10abda3d0 100644 --- a/vortex-array/src/arrays/varbinview/compute/take.rs +++ b/vortex-array/src/arrays/varbinview/compute/take.rs @@ -173,6 +173,9 @@ mod tests { ))] #[case(VarBinViewArray::from_iter(["single"].map(Some), DType::Utf8(NonNullable)))] fn test_take_varbinview_conformance(#[case] array: VarBinViewArray) { - test_take_conformance(&array.into_array()); + test_take_conformance( + &array.into_array(), + &mut array_session().create_execution_ctx(), + ); } } diff --git a/vortex-array/src/arrays/varbinview/compute/zip.rs b/vortex-array/src/arrays/varbinview/compute/zip.rs index 75e30808821..9f2b4e1ecf9 100644 --- a/vortex-array/src/arrays/varbinview/compute/zip.rs +++ b/vortex-array/src/arrays/varbinview/compute/zip.rs @@ -217,8 +217,6 @@ mod tests { use crate::array_session; use crate::arrays::VarBinViewArray; use crate::builtins::ArrayBuiltins; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; @@ -250,12 +248,12 @@ mod tests { let mask = Mask::from_iter([true, false, true, false, false, true]); - #[expect(deprecated)] + let mut ctx = array_session().create_execution_ctx(); let zipped = mask .clone() .into_array() .zip(a.into_array(), b.into_array())? - .to_varbinview(); + .execute::(&mut ctx)?; let mut ctx = array_session().create_execution_ctx(); let validity_mask = zipped.validity()?.execute_mask(zipped.len(), &mut ctx)?; diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 386625adcf1..6af44097824 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -227,12 +227,10 @@ impl VTable for VarBinView { builder: &mut dyn ArrayBuilder, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - if let Some(builder) = builder.as_any_mut().downcast_mut::() { - return builder.append_varbinview_array(&array.into_owned(), ctx); - } - - builder.extend_from_array(array.as_ref()); - Ok(()) + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + vortex_bail!("append_to_builder for VarBinView requires a VarBinViewBuilder"); + }; + builder.append_varbinview_array(&array.into_owned(), ctx) } fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { diff --git a/vortex-array/src/arrow/executor/byte.rs b/vortex-array/src/arrow/executor/byte.rs index 2db9d3e6494..83c1c681284 100644 --- a/vortex-array/src/arrow/executor/byte.rs +++ b/vortex-array/src/arrow/executor/byte.rs @@ -79,15 +79,15 @@ mod tests { use arrow_array::Array; use arrow_array::cast::AsArray; use arrow_schema::DataType; + use arrow_schema::Field; use rstest::rstest; use vortex_error::VortexResult; use vortex_mask::Mask; use crate::IntoArray; - use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::array_session; - use crate::arrow::ArrowArrayExecutor; + use crate::arrow::ArrowSessionExt; use crate::arrow::executor::byte::VarBinViewArray; use crate::dtype::DType; use crate::dtype::Nullability; @@ -122,10 +122,12 @@ mod tests { #[case] vortex_array: VarBinViewArray, #[case] target_dtype: DataType, ) { - let mut ctx = array_session().create_execution_ctx(); - let arrow = vortex_array - .into_array() - .execute_arrow(Some(&target_dtype), &mut ctx) + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + let field = Field::new("test_field", target_dtype.clone(), true); + let arrow = session + .arrow() + .execute_arrow(vortex_array.into_array(), Some(&field), &mut ctx) .unwrap(); assert_eq!(arrow.data_type(), &target_dtype); @@ -169,10 +171,12 @@ mod tests { vortex_dtype, ); - let mut ctx = array_session().create_execution_ctx(); - let arrow = vortex_array - .into_array() - .execute_arrow(Some(&target_dtype), &mut ctx) + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + let field = Field::new("test_field", target_dtype.clone(), true); + let arrow = session + .arrow() + .execute_arrow(vortex_array.into_array(), Some(&field), &mut ctx) .unwrap(); assert_eq!(arrow.data_type(), &target_dtype); @@ -192,10 +196,11 @@ mod tests { .into_array() .filter(Mask::from_iter([true, false, false]))?; - let arrow = filtered.execute_arrow( - Some(&DataType::Utf8View), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let session = array_session(); + let mut ctx = session.create_execution_ctx(); + let arrow = session + .arrow() + .execute_arrow(filtered.into_array(), None, &mut ctx)?; assert_eq!(arrow.as_string_view().value(0), "selected"); assert!( diff --git a/vortex-array/src/arrow/executor/byte_view.rs b/vortex-array/src/arrow/executor/byte_view.rs index 89f37681220..4e17b7f2376 100644 --- a/vortex-array/src/arrow/executor/byte_view.rs +++ b/vortex-array/src/arrow/executor/byte_view.rs @@ -47,7 +47,7 @@ pub fn execute_varbinview_to_arrow( array: &VarBinViewArray, ctx: &mut ExecutionCtx, ) -> VortexResult { - let compacted = array.compact_buffers()?; + let compacted = array.compact_buffers(ctx)?; canonical_varbinview_to_arrow::(&compacted, ctx) } diff --git a/vortex-array/src/builders/bool.rs b/vortex-array/src/builders/bool.rs index f26cbf04b57..73d2089e221 100644 --- a/vortex-array/src/builders/bool.rs +++ b/vortex-array/src/builders/bool.rs @@ -5,7 +5,6 @@ use std::any::Any; use std::mem; use vortex_buffer::BitBufferMut; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_mask::Mask; @@ -13,16 +12,12 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -127,13 +122,6 @@ impl ArrayBuilder for BoolBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let bool_array = array.to_bool(); - self.append_bool_array(&bool_array, &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to append bool array"); - } - fn reserve_exact(&mut self, additional: usize) { self.inner.reserve(additional); self.nulls.reserve_exact(additional); @@ -147,7 +135,7 @@ impl ArrayBuilder for BoolBuilder { self.finish_into_bool().into_array() } - fn finish_into_canonical(&mut self) -> Canonical { + fn finish_into_canonical(&mut self, _ctx: &mut ExecutionCtx) -> Canonical { Canonical::Bool(self.finish_into_bool()) } } @@ -170,8 +158,6 @@ mod tests { use crate::builders::BoolBuilder; use crate::builders::bool::BoolArray; use crate::builders::builder_with_capacity; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -205,10 +191,8 @@ mod tests { .clone() .append_to_builder(builder.as_mut(), &mut ctx)?; - #[expect(deprecated)] - let canon_into = builder.finish().to_bool(); - #[expect(deprecated)] - let into_canon = chunk.to_bool(); + let canon_into = builder.finish().execute::(&mut ctx)?; + let into_canon = chunk.clone().execute::(&mut ctx)?; assert!(canon_into.validity()?.mask_eq( &into_canon.validity()?, diff --git a/vortex-array/src/builders/decimal.rs b/vortex-array/src/builders/decimal.rs index aa777051e39..c08dfbc4d87 100644 --- a/vortex-array/src/builders/decimal.rs +++ b/vortex-array/src/builders/decimal.rs @@ -12,11 +12,8 @@ use vortex_error::vortex_panic; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -#[expect(deprecated)] -use crate::ToCanonical as _; -use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; @@ -129,6 +126,28 @@ impl DecimalBuilder { self.nulls.append_n_non_nulls(n); } + /// Appends the values of a canonical [`DecimalArray`] to the builder, coercing the physical + /// storage type to the builder's type as needed. + pub(crate) fn append_decimal_array( + &mut self, + array: &DecimalArray, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + match_each_decimal_value_type!(array.values_type(), |D| { + // Extends the values buffer from another buffer of type D where D can be coerced to the + // builder type. + self.values.extend(array.buffer::().iter().copied()); + }); + + self.nulls.append_validity_mask( + &array + .as_ref() + .validity()? + .execute_mask(array.as_ref().len(), ctx)?, + ); + Ok(()) + } + /// Finishes the builder directly into a [`DecimalArray`]. pub fn finish_into_decimal(&mut self) -> DecimalArray { let validity = self.nulls.finish_with_nullability(self.dtype.nullability()); @@ -195,30 +214,6 @@ impl ArrayBuilder for DecimalBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let decimal_array = array.to_decimal(); - - match_each_decimal_value_type!(decimal_array.values_type(), |D| { - // Extends the values buffer from another buffer of type D where D can be coerced to the - // builder type. - self.values - .extend(decimal_array.buffer::().iter().copied()); - }); - - self.nulls.append_validity_mask( - &decimal_array - .as_ref() - .validity() - .vortex_expect("validity_mask") - .execute_mask( - decimal_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("Failed to compute validity mask"), - ); - } - fn reserve_exact(&mut self, additional: usize) { self.values.reserve(additional); self.nulls.reserve_exact(additional); @@ -232,7 +227,7 @@ impl ArrayBuilder for DecimalBuilder { self.finish_into_decimal().into_array() } - fn finish_into_canonical(&mut self) -> Canonical { + fn finish_into_canonical(&mut self, _ctx: &mut ExecutionCtx) -> Canonical { Canonical::Decimal(self.finish_into_decimal()) } } @@ -328,7 +323,8 @@ mod tests { let i8s = i8s.finish(); let mut i128s = DecimalBuilder::new::(DecimalDType::new(2, 1), false.into()); - i128s.extend_from_array(&i8s); + i8s.append_to_builder(&mut i128s, &mut array_session().create_execution_ctx()) + .unwrap(); let i128s = i128s.finish(); for i in 0..i8s.len() { diff --git a/vortex-array/src/builders/extension.rs b/vortex-array/src/builders/extension.rs index 61d5fe8d7b0..aa8e1b76aa2 100644 --- a/vortex-array/src/builders/extension.rs +++ b/vortex-array/src/builders/extension.rs @@ -8,6 +8,7 @@ use vortex_error::vortex_ensure; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::ExtensionArray; use crate::arrays::extension::ExtensionArrayExt; @@ -15,8 +16,6 @@ use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::builder_with_capacity; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; use crate::scalar::ExtScalar; @@ -47,6 +46,18 @@ impl ExtensionBuilder { self.storage.append_scalar(&value.to_storage_scalar()) } + /// Appends the values of a canonical [`ExtensionArray`] to the builder by appending its + /// storage array to the underlying storage builder. + pub(crate) fn append_extension_array( + &mut self, + array: &ExtensionArray, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + array + .storage_array() + .append_to_builder(self.storage.as_mut(), ctx) + } + /// Finishes the builder directly into a [`ExtensionArray`]. pub fn finish_into_extension(&mut self) -> ExtensionArray { let storage = self.storage.finish(); @@ -101,12 +112,6 @@ impl ArrayBuilder for ExtensionBuilder { self.append_value(scalar.as_extension()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let ext_array = array.to_extension(); - self.storage.extend_from_array(ext_array.storage_array()) - } - fn reserve_exact(&mut self, capacity: usize) { self.storage.reserve_exact(capacity) } @@ -119,7 +124,7 @@ impl ArrayBuilder for ExtensionBuilder { self.finish_into_extension().into_array() } - fn finish_into_canonical(&mut self) -> Canonical { + fn finish_into_canonical(&mut self, _ctx: &mut ExecutionCtx) -> Canonical { Canonical::Extension(self.finish_into_extension()) } } diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index 9129d8cb644..9150d37b50b 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -14,8 +14,6 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::FixedSizeListArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builders::ArrayBuilder; @@ -23,8 +21,6 @@ use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::builders::builder_with_capacity; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::ListScalar; @@ -108,6 +104,25 @@ impl FixedSizeListBuilder { Ok(()) } + /// Appends the values of a canonical [`FixedSizeListArray`] to the builder, recursing into the + /// elements builder. + pub(crate) fn append_fixed_size_list_array( + &mut self, + array: &FixedSizeListArray, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + if array.is_empty() { + return Ok(()); + } + + array + .elements() + .append_to_builder(self.elements_builder.as_mut(), ctx)?; + self.nulls + .append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?); + Ok(()) + } + /// Appends a fixed-size list `value` to the builder. /// /// Note that a [`ListScalar`] can represent both a [`ListArray`] scalar **and** a @@ -242,23 +257,6 @@ impl ArrayBuilder for FixedSizeListBuilder { /// This will increase the capacity if extending with this `array` would go past the original /// capacity. - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let fsl = array.to_fixed_size_list(); - if fsl.is_empty() { - return; - } - - self.elements_builder.extend_from_array(fsl.elements()); - self.nulls.append_validity_mask( - &array - .validity() - .vortex_expect("validity_mask in extend_from_array_unchecked") - .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to compute validity mask"), - ); - } - fn reserve_exact(&mut self, additional: usize) { self.elements_builder .reserve_exact(additional * self.list_size() as usize); @@ -273,7 +271,7 @@ impl ArrayBuilder for FixedSizeListBuilder { self.finish_into_fixed_size_list().into_array() } - fn finish_into_canonical(&mut self) -> Canonical { + fn finish_into_canonical(&mut self, _ctx: &mut ExecutionCtx) -> Canonical { Canonical::FixedSizeList(self.finish_into_fixed_size_list()) } } @@ -287,8 +285,6 @@ mod tests { use super::FixedSizeListBuilder; use crate::IntoArray as _; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::PrimitiveArray; @@ -342,8 +338,8 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 2); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let mut ctx = array_session().create_execution_ctx(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.elements().len(), 6); assert_eq!(fsl_array.list_size(), 3); } @@ -366,8 +362,8 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 100); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let mut ctx = array_session().create_execution_ctx(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.list_size(), 0); // The elements array should be empty since list_size is 0. assert_eq!(fsl_array.elements().len(), 0); @@ -396,8 +392,8 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 100); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let mut ctx = array_session().create_execution_ctx(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.list_size(), 0); assert_eq!(fsl_array.elements().len(), 0); } @@ -426,8 +422,8 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 5); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let mut ctx = array_session().create_execution_ctx(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.elements().len(), 10); } @@ -440,8 +436,8 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 0); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let mut ctx = array_session().create_execution_ctx(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.list_size(), 100000000); assert_eq!(fsl_array.elements().len(), 0); } @@ -474,8 +470,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 3); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert!( fsl_array .validity() @@ -538,8 +533,8 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 2); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let mut ctx = array_session().create_execution_ctx(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.elements().len(), 6); } @@ -553,14 +548,17 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 5); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let mut ctx = array_session().create_execution_ctx(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.list_size(), 3); assert_eq!(fsl_array.elements().len(), 15); // Check that all elements are zeros. - #[expect(deprecated)] - let elements_array = fsl_array.elements().to_primitive(); + let elements_array = fsl_array + .elements() + .clone() + .execute::(&mut ctx) + .unwrap(); let elements = elements_array.as_slice::(); assert!(elements.iter().all(|&x| x == 0)); } @@ -579,8 +577,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 3); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.list_size(), 2); // Check that all lists are null. @@ -611,8 +608,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 1); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.list_size(), 2); // Check that all lists are null. @@ -637,8 +633,8 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 1000); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let mut ctx = array_session().create_execution_ctx(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.list_size(), 0); assert_eq!(fsl_array.elements().len(), 0); } @@ -684,14 +680,17 @@ mod tests { let mut builder = FixedSizeListBuilder::with_capacity(dtype, 2, Nullable, 0); let source_array = source.into_array(); - builder.extend_from_array(&source_array); - builder.extend_from_array(&source_array); + source_array + .append_to_builder(&mut builder, &mut ctx) + .unwrap(); + source_array + .append_to_builder(&mut builder, &mut ctx) + .unwrap(); let fsl = builder.finish(); assert_eq!(fsl.len(), 6); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.elements().len(), 12); // Check validity pattern is repeated. @@ -761,14 +760,19 @@ mod tests { let mut builder = FixedSizeListBuilder::with_capacity(dtype, 0, Nullable, 0); - builder.extend_from_array(&source1.into_array()); - builder.extend_from_array(&source2.into_array()); + source1 + .into_array() + .append_to_builder(&mut builder, &mut ctx) + .unwrap(); + source2 + .into_array() + .append_to_builder(&mut builder, &mut ctx) + .unwrap(); let fsl = builder.finish(); assert_eq!(fsl.len(), 5); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.list_size(), 0); assert_eq!(fsl_array.elements().len(), 0); @@ -812,6 +816,7 @@ mod tests { #[test] fn test_extend_empty_array() { + let mut ctx = array_session().create_execution_ctx(); let dtype: Arc = Arc::new(I32.into()); // Create an empty source array. @@ -838,7 +843,10 @@ mod tests { .unwrap(); // Extend with empty array (should be no-op). - builder.extend_from_array(&source.into_array()); + source + .into_array() + .append_to_builder(&mut builder, &mut ctx) + .unwrap(); let fsl = builder.finish(); assert_eq!(fsl.len(), 1); @@ -876,13 +884,15 @@ mod tests { Validity::AllValid, 1, ); - builder.extend_from_array(&source.into_array()); + source + .into_array() + .append_to_builder(&mut builder, &mut ctx) + .unwrap(); let fsl = builder.finish(); assert_eq!(fsl.len(), 6); - #[expect(deprecated)] - let fsl_array = fsl.to_fixed_size_list(); + let fsl_array = fsl.execute::(&mut ctx).unwrap(); assert_eq!(fsl_array.elements().len(), 12); // Check validity. @@ -927,7 +937,7 @@ mod tests { .vortex_expect("fixed-size-list validity should be derivable") .execute_is_valid(5, &mut ctx) .unwrap() - ); // extend_from_array + ); } #[test] @@ -1043,8 +1053,11 @@ mod tests { assert_eq!(fsl.list_size(), 3); // Verify elements array: [1, 2, 3, 10, 11, 12, 4, 5, 6, 20, 21, 22]. - #[expect(deprecated)] - let elements = fsl.elements().to_primitive(); + let elements = fsl + .elements() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!( elements.as_slice::(), &[1, 2, 3, 10, 11, 12, 4, 5, 6, 20, 21, 22] diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index 12c74ba53a4..71098796d00 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -15,17 +15,15 @@ use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::ListArray; +use crate::arrays::ListViewArray; +use crate::arrays::PrimitiveArray; use crate::arrays::listview::ListViewArrayExt; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::builders::PrimitiveBuilder; use crate::builders::builder_with_capacity; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::IntegerPType; use crate::dtype::Nullability; @@ -170,87 +168,40 @@ impl ListBuilder { element_dtype } -} - -impl ArrayBuilder for ListBuilder { - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - - fn dtype(&self) -> &DType { - &self.dtype - } - fn len(&self) -> usize { - self.nulls.len() - } - - fn append_zeros(&mut self, n: usize) { - let curr_len = self.elements_builder.len(); - for _ in 0..n { - self.offsets_builder.append_value( - O::from_usize(curr_len).vortex_expect("Failed to convert from usize to "), - ) - } - self.nulls.append_n_non_nulls(n); - } - - unsafe fn append_nulls_unchecked(&mut self, n: usize) { - let curr_len = self.elements_builder.len(); - for _ in 0..n { - // A list with a null element is can be a list with a zero-span offset and a validity - // bit set - self.offsets_builder.append_value( - O::from_usize(curr_len).vortex_expect("Failed to convert from usize to "), - ) - } - self.nulls.append_n_nulls(n); - } - - fn append_scalar(&mut self, scalar: &Scalar) -> VortexResult<()> { - vortex_ensure!( - scalar.dtype() == self.dtype(), - "ListBuilder expected scalar with dtype {}, got {}", - self.dtype(), - scalar.dtype() - ); - - self.append_value(scalar.as_list()) - } - - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let list = array.to_listview(); + /// Appends the values of a list-typed `array` to the builder, canonicalizing to a + /// [`ListViewArray`] and converting into the `ListArray` (`n + 1` offsets) layout. + /// + /// [`ListBuilder`] is not the canonical builder for [`DType::List`] (that is + /// [`ListViewBuilder`](crate::builders::ListViewBuilder)), so no encoding dispatches into it via + /// `append_to_builder`; this helper exists for direct callers such as tests. + pub(crate) fn append_list_array( + &mut self, + array: &ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let list = array.clone().execute::(ctx)?; if list.is_empty() { - return; + return Ok(()); } // Append validity information. - self.nulls.append_validity_mask( - &array - .validity() - .vortex_expect("validity_mask in extend_from_array_unchecked") - .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to compute validity mask"), - ); + self.nulls + .append_validity_mask(&list.validity()?.execute_mask(list.len(), ctx)?); // Note that `ListViewArray` has `n` offsets and sizes, not `n+1` offsets like `ListArray`. let elements = list.elements(); - #[expect(deprecated)] - let offsets = list.offsets().to_primitive(); - #[expect(deprecated)] - let sizes = list.sizes().to_primitive(); + let offsets = list.offsets().clone().execute::(ctx)?; + let sizes = list.sizes().clone().execute::(ctx)?; fn extend_inner( builder: &mut ListBuilder, new_elements: &ArrayRef, new_offsets: &[OffsetType], new_sizes: &[SizeType], - ) where + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> + where O: IntegerPType, OffsetType: IntegerPType, SizeType: IntegerPType, @@ -271,7 +222,7 @@ impl ArrayBuilder for ListBuilder { let list_elements = new_elements .slice(offset..offset + size) .vortex_expect("list builder slice"); - builder.elements_builder.extend_from_array(&list_elements); + list_elements.append_to_builder(builder.elements_builder.as_mut(), ctx)?; curr_offset += size; } @@ -284,6 +235,7 @@ impl ArrayBuilder for ListBuilder { // SAFETY: We have initialized all `num_lists` values, and since the `offsets` array is // non-nullable, we are done. unsafe { offsets_range.finish() }; + Ok(()) } match_each_integer_ptype!(offsets.ptype(), |OffsetType| { @@ -293,9 +245,62 @@ impl ArrayBuilder for ListBuilder { elements, offsets.as_slice::(), sizes.as_slice::(), - ) + ctx, + )? }) - }) + }); + Ok(()) + } +} + +impl ArrayBuilder for ListBuilder { + fn as_any(&self) -> &dyn Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + + fn dtype(&self) -> &DType { + &self.dtype + } + + fn len(&self) -> usize { + self.nulls.len() + } + + fn append_zeros(&mut self, n: usize) { + let curr_len = self.elements_builder.len(); + for _ in 0..n { + self.offsets_builder.append_value( + O::from_usize(curr_len).vortex_expect("Failed to convert from usize to "), + ) + } + self.nulls.append_n_non_nulls(n); + } + + unsafe fn append_nulls_unchecked(&mut self, n: usize) { + let curr_len = self.elements_builder.len(); + for _ in 0..n { + // A list with a null element is can be a list with a zero-span offset and a validity + // bit set + self.offsets_builder.append_value( + O::from_usize(curr_len).vortex_expect("Failed to convert from usize to "), + ) + } + self.nulls.append_n_nulls(n); + } + + fn append_scalar(&mut self, scalar: &Scalar) -> VortexResult<()> { + vortex_ensure!( + scalar.dtype() == self.dtype(), + "ListBuilder expected scalar with dtype {}, got {}", + self.dtype(), + scalar.dtype() + ); + + self.append_value(scalar.as_list()) } fn reserve_exact(&mut self, additional: usize) { @@ -312,9 +317,11 @@ impl ArrayBuilder for ListBuilder { self.finish_into_list().into_array() } - fn finish_into_canonical(&mut self) -> Canonical { - #[expect(deprecated)] - let listview = self.finish_into_list().into_array().to_listview(); + fn finish_into_canonical(&mut self, ctx: &mut ExecutionCtx) -> Canonical { + let listview = self + .finish() + .execute::(ctx) + .vortex_expect("list builder should canonicalize to listview"); Canonical::List(listview) } } @@ -329,10 +336,9 @@ mod tests { use vortex_error::VortexExpect; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::array_session; use crate::arrays::ChunkedArray; + use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; use crate::arrays::list::ListArrayExt; use crate::arrays::listview::ListViewArrayExt; @@ -387,8 +393,8 @@ mod tests { let list = builder.finish(); assert_eq!(list.len(), 2); - #[expect(deprecated)] - let list_array = list.to_listview(); + let mut ctx = array_session().create_execution_ctx(); + let list_array = list.execute::(&mut ctx).unwrap(); assert_eq!(list_array.list_elements_at(0).unwrap().len(), 3); assert_eq!(list_array.list_elements_at(1).unwrap().len(), 3); @@ -440,8 +446,8 @@ mod tests { let list = builder.finish(); assert_eq!(list.len(), 3); - #[expect(deprecated)] - let list_array = list.to_listview(); + let mut ctx = array_session().create_execution_ctx(); + let list_array = list.execute::(&mut ctx).unwrap(); assert_eq!(list_array.list_elements_at(0).unwrap().len(), 3); assert_eq!(list_array.list_elements_at(1).unwrap().len(), 0); @@ -453,18 +459,22 @@ mod tests { [Some(vec![0, 1, 2]), None, Some(vec![4, 5])], Arc::new(I32.into()), ) - .unwrap(); + .unwrap() + .into_array(); assert_eq!(list.len(), 3); let mut ctx = array_session().create_execution_ctx(); let mut builder = ListBuilder::::with_capacity(Arc::new(I32.into()), Nullable, 18, 9); - builder.extend_from_array(&list); - builder.extend_from_array(&list); - builder.extend_from_array(&list.slice(0..0).unwrap()); - builder.extend_from_array(&list.slice(1..3).unwrap()); + builder.append_list_array(&list, &mut ctx).unwrap(); + builder.append_list_array(&list, &mut ctx).unwrap(); + builder + .append_list_array(&list.slice(0..0).unwrap(), &mut ctx) + .unwrap(); + builder + .append_list_array(&list.slice(1..3).unwrap(), &mut ctx) + .unwrap(); - #[expect(deprecated)] let expected = ListArray::from_iter_opt_slow::( [ Some(vec![0, 1, 2]), @@ -479,9 +489,11 @@ mod tests { Arc::new(DType::Primitive(I32, NonNullable)), ) .unwrap() - .to_listview(); + .into_array() + .execute::(&mut ctx) + .unwrap(); - let actual = builder.finish_into_canonical().into_listview(); + let actual = builder.finish_into_canonical(&mut ctx).into_listview(); assert_arrays_eq!(actual.elements(), expected.elements(), &mut ctx); @@ -539,8 +551,13 @@ mod tests { DType::List(Arc::new(DType::Primitive(I32, NonNullable)), NonNullable), ); - #[expect(deprecated)] - let canon_values = chunked_list.unwrap().as_array().to_listview(); + let mut ctx = array_session().create_execution_ctx(); + let canon_values = chunked_list + .unwrap() + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!( one_trailing_unused_element diff --git a/vortex-array/src/builders/listview.rs b/vortex-array/src/builders/listview.rs index cb131048339..35289a41813 100644 --- a/vortex-array/src/builders/listview.rs +++ b/vortex-array/src/builders/listview.rs @@ -21,8 +21,6 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::array::IntoArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -209,7 +207,7 @@ impl ListViewBuilder { // amount. // - We checked on construction that the sizes type fits into the offsets. // - In every method that adds values to this builder (`append_value`, `append_scalar`, and - // `extend_from_array_unchecked`), we checked that `offset + size` does not overflow. + // `append_listview_array`), we checked that `offset + size` does not overflow. // - We constructed everything in a way that builds the `ListViewArray` similar to the shape // of a `ListArray`, so we know the resulting array is zero-copyable to a `ListArray`. unsafe { @@ -227,6 +225,64 @@ impl ListViewBuilder { element_dtype } + + /// Appends the values of a canonical [`ListViewArray`] to the builder, recursing into the + /// elements builder. + pub(crate) fn append_listview_array( + &mut self, + array: &ListViewArray, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + if array.is_empty() { + return Ok(()); + } + + // Normalize to an exact zero-copy-to-list layout and then bulk append. This avoids the + // very expensive scalar_at-per-list path for overlapping / out-of-order list views. + let listview = array.rebuild(ListViewRebuildMode::MakeExact, ctx)?; + debug_assert!(listview.is_zero_copy_to_list()); + + self.nulls + .append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?); + + // Bulk append the new elements (which should have no gaps or overlaps). + let old_elements_len = self.elements_builder.len(); + self.elements_builder + .reserve_exact(listview.elements().len()); + listview + .elements() + .append_to_builder(self.elements_builder.as_mut(), ctx)?; + let new_elements_len = self.elements_builder.len(); + + // Reserve enough space for the new views. + let extend_length = listview.len(); + self.sizes_builder.reserve_exact(extend_length); + self.offsets_builder.reserve_exact(extend_length); + + // The incoming sizes might have a different type than the builder, so we need to cast. + let cast_sizes = listview + .sizes() + .clone() + .cast(self.sizes_builder.dtype().clone())?; + cast_sizes.append_to_builder(&mut self.sizes_builder, ctx)?; + + // Now we need to adjust all of the offsets by adding the current number of elements in the + // builder. + let uninit_range = self.offsets_builder.uninit_range(extend_length); + + // This should be cheap because we didn't compress after rebuilding. + let new_offsets = listview.offsets().clone().execute::(ctx)?; + + match_each_integer_ptype!(new_offsets.ptype(), |A| { + adjust_and_extend_offsets::( + uninit_range, + new_offsets, + old_elements_len, + new_elements_len, + ); + }); + Ok(()) + } } impl ArrayBuilder for ListViewBuilder { @@ -296,80 +352,6 @@ impl ArrayBuilder for ListViewBuilder { self.append_value(list_scalar) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - // TODO: The `ArrayBuilder` trait does not thread an `ExecutionCtx` through its extend - // methods, so we are forced to mint a fresh `LEGACY_SESSION` context here on every call - // (which for chunked input means once per chunk). Once the trait carries a `&mut - // ExecutionCtx`, the caller's session should be reused instead. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - - let listview = array - .clone() - .execute::(&mut ctx) - .vortex_expect("failed to execute array into ListViewArray in extend_from_array"); - if listview.is_empty() { - return; - } - - // Normalize to an exact zero-copy-to-list layout and then bulk append. This avoids the - // very expensive scalar_at-per-list path for overlapping / out-of-order list views. - let listview = listview - .rebuild(ListViewRebuildMode::MakeExact, &mut ctx) - .vortex_expect("ListViewArray::rebuild(MakeExact) failed in extend_from_array"); - debug_assert!(listview.is_zero_copy_to_list()); - - self.nulls.append_validity_mask( - &array - .validity() - .vortex_expect("validity_mask in extend_from_array_unchecked") - .execute_mask(array.len(), &mut ctx) - .vortex_expect("Failed to compute validity mask"), - ); - - // Bulk append the new elements (which should have no gaps or overlaps). - let old_elements_len = self.elements_builder.len(); - self.elements_builder - .reserve_exact(listview.elements().len()); - self.elements_builder.extend_from_array(listview.elements()); - let new_elements_len = self.elements_builder.len(); - - // Reserve enough space for the new views. - let extend_length = listview.len(); - self.sizes_builder.reserve_exact(extend_length); - self.offsets_builder.reserve_exact(extend_length); - - // The incoming sizes might have a different type than the builder, so we need to cast. - let cast_sizes = listview - .sizes() - .clone() - .cast(self.sizes_builder.dtype().clone()) - .vortex_expect( - "was somehow unable to cast the new sizes to the type of the builder sizes", - ); - self.sizes_builder.extend_from_array(&cast_sizes); - - // Now we need to adjust all of the offsets by adding the current number of elements in the - // builder. - - let uninit_range = self.offsets_builder.uninit_range(extend_length); - - // This should be cheap because we didn't compress after rebuilding. - let new_offsets = listview - .offsets() - .clone() - .execute::(&mut ctx) - .vortex_expect("failed to execute list view offsets into a PrimitiveArray"); - - match_each_integer_ptype!(new_offsets.ptype(), |A| { - adjust_and_extend_offsets::( - uninit_range, - new_offsets, - old_elements_len, - new_elements_len, - ); - }) - } - fn reserve_exact(&mut self, capacity: usize) { self.elements_builder.reserve_exact(capacity * 2); self.offsets_builder.reserve_exact(capacity); @@ -385,15 +367,15 @@ impl ArrayBuilder for ListViewBuilder { self.finish_into_listview().into_array() } - fn finish_into_canonical(&mut self) -> Canonical { + fn finish_into_canonical(&mut self, _ctx: &mut ExecutionCtx) -> Canonical { Canonical::List(self.finish_into_listview()) } } /// Given new offsets, adds them to the `UninitRange` after adding the `old_elements_len` to each /// offset. -fn adjust_and_extend_offsets<'a, O: IntegerPType, A: IntegerPType>( - mut uninit_range: UninitRange<'a, O>, +fn adjust_and_extend_offsets( + mut uninit_range: UninitRange, new_offsets: PrimitiveArray, old_elements_len: usize, new_elements_len: usize, @@ -674,7 +656,11 @@ mod tests { .unwrap(); // Extend from the ListArray. - builder.extend_from_array(&source.into_array()); + let source = source + .into_array() + .execute::(&mut ctx) + .unwrap(); + builder.append_listview_array(&source, &mut ctx).unwrap(); // Extend from empty array (should be no-op). let empty_source = ListArray::from_iter_opt_slow::>( @@ -682,7 +668,13 @@ mod tests { Arc::new(I32.into()), ) .unwrap(); - builder.extend_from_array(&empty_source.into_array()); + let empty_source = empty_source + .into_array() + .execute::(&mut ctx) + .unwrap(); + builder + .append_listview_array(&empty_source, &mut ctx) + .unwrap(); let listview = builder.finish_into_listview(); assert_eq!(listview.len(), 4); @@ -740,7 +732,7 @@ mod tests { let mut builder = ListViewBuilder::::with_capacity(Arc::clone(&dtype), Nullable, 0, 0); - builder.extend_from_array(&source.into_array()); + builder.append_listview_array(&source, &mut ctx).unwrap(); let listview = builder.finish_into_listview(); assert_eq!(listview.len(), 3); diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs index 48a347aa3d5..e94cdd8c6dd 100644 --- a/vortex-array/src/builders/mod.rs +++ b/vortex-array/src/builders/mod.rs @@ -34,10 +34,10 @@ use std::any::Any; use std::sync::Arc; use vortex_error::VortexResult; -use vortex_error::vortex_panic; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::canonical::Canonical; use crate::dtype::DType; use crate::match_each_decimal_value_type; @@ -153,30 +153,6 @@ pub trait ArrayBuilder: Send { /// A generic function to append a scalar to the builder. fn append_scalar(&mut self, scalar: &Scalar) -> VortexResult<()>; - /// The inner part of `extend_from_array`. - /// - /// # Safety - /// - /// The array that must have an equal [`DType`] to the array builder's `DType` (with nullability - /// superset semantics). - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef); - - /// Extends the array with the provided array, canonicalizing if necessary. - /// - /// Implementors must validate that the passed in [`ArrayRef`] has the correct [`DType`]. - fn extend_from_array(&mut self, array: &ArrayRef) { - if !self.dtype().eq_with_nullability_superset(array.dtype()) { - vortex_panic!( - "tried to extend a builder with `DType` {} with an array with `DType {}", - self.dtype(), - array.dtype() - ); - } - - // SAFETY: We checked that the array had a valid `DType` above. - unsafe { self.extend_from_array_unchecked(array) } - } - /// Allocate space for extra `additional` items fn reserve_exact(&mut self, additional: usize); @@ -214,7 +190,7 @@ pub trait ArrayBuilder: Send { /// This method provides a default implementation that creates an [`ArrayRef`] via `finish` and /// then converts it to canonical form. Specific builders can override this with optimized /// implementations that avoid the intermediate [`ArrayRef`] creation. - fn finish_into_canonical(&mut self) -> Canonical; + fn finish_into_canonical(&mut self, ctx: &mut ExecutionCtx) -> Canonical; } /// Construct a new canonical builder for the given [`DType`]. diff --git a/vortex-array/src/builders/null.rs b/vortex-array/src/builders/null.rs index 45b6e7ed974..28eab14dd7a 100644 --- a/vortex-array/src/builders/null.rs +++ b/vortex-array/src/builders/null.rs @@ -8,6 +8,7 @@ use vortex_error::vortex_ensure; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::NullArray; use crate::builders::ArrayBuilder; @@ -69,10 +70,6 @@ impl ArrayBuilder for NullBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - self.append_nulls(array.len()); - } - fn reserve_exact(&mut self, _additional: usize) {} unsafe fn set_validity_unchecked(&mut self, _validity: Mask) {} @@ -81,7 +78,7 @@ impl ArrayBuilder for NullBuilder { NullArray::new(self.length).into_array() } - fn finish_into_canonical(&mut self) -> Canonical { + fn finish_into_canonical(&mut self, _ctx: &mut ExecutionCtx) -> Canonical { Canonical::Null(NullArray::new(self.length)) } } diff --git a/vortex-array/src/builders/primitive.rs b/vortex-array/src/builders/primitive.rs index e7e8d3d7512..4ce1aafe1f7 100644 --- a/vortex-array/src/builders/primitive.rs +++ b/vortex-array/src/builders/primitive.rs @@ -13,15 +13,11 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; @@ -200,14 +196,6 @@ impl ArrayBuilder for PrimitiveBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let array = array.to_primitive(); - - self.append_primitive_array(&array, &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to append primitive array"); - } - fn reserve_exact(&mut self, additional: usize) { self.values.reserve(additional); self.nulls.reserve_exact(additional); @@ -221,7 +209,7 @@ impl ArrayBuilder for PrimitiveBuilder { self.finish_into_primitive().into_array() } - fn finish_into_canonical(&mut self) -> Canonical { + fn finish_into_canonical(&mut self, _ctx: &mut ExecutionCtx) -> Canonical { Canonical::Primitive(self.finish_into_primitive()) } } diff --git a/vortex-array/src/builders/struct_.rs b/vortex-array/src/builders/struct_.rs index 016d42508ee..da08e262760 100644 --- a/vortex-array/src/builders/struct_.rs +++ b/vortex-array/src/builders/struct_.rs @@ -12,9 +12,8 @@ use vortex_error::vortex_panic; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::StructArray; use crate::arrays::struct_::StructArrayExt; use crate::builders::ArrayBuilder; @@ -22,8 +21,6 @@ use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::builders::builder_with_capacity; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::StructFields; @@ -122,6 +119,25 @@ impl StructBuilder { struct_fields } + + /// Appends the values of a canonical [`StructArray`] to the builder, recursing into each + /// field's builder. + pub(crate) fn append_struct_array( + &mut self, + array: &StructArray, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + for (field, builder) in array + .iter_unmasked_fields() + .zip_eq(self.builders.iter_mut()) + { + field.append_to_builder(builder.as_mut(), ctx)?; + } + + self.nulls + .append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?); + Ok(()) + } } impl ArrayBuilder for StructBuilder { @@ -168,26 +184,6 @@ impl ArrayBuilder for StructBuilder { self.append_value(scalar.as_struct()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let array = array.to_struct(); - - for (a, builder) in array - .iter_unmasked_fields() - .zip_eq(self.builders.iter_mut()) - { - builder.extend_from_array(a); - } - - self.nulls.append_validity_mask( - &array - .validity() - .vortex_expect("validity_mask") - .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to compute validity mask"), - ); - } - fn reserve_exact(&mut self, capacity: usize) { self.builders.iter_mut().for_each(|builder| { builder.reserve_exact(capacity); @@ -203,7 +199,7 @@ impl ArrayBuilder for StructBuilder { self.finish_into_struct().into_array() } - fn finish_into_canonical(&mut self) -> Canonical { + fn finish_into_canonical(&mut self, _ctx: &mut ExecutionCtx) -> Canonical { Canonical::Struct(self.finish_into_struct()) } } diff --git a/vortex-array/src/builders/tests.rs b/vortex-array/src/builders/tests.rs index 02babab4a7d..138e6941def 100644 --- a/vortex-array/src/builders/tests.rs +++ b/vortex-array/src/builders/tests.rs @@ -8,7 +8,10 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; +use crate::Canonical; +use crate::ExecutionCtx; use crate::VortexSessionExecute; +use crate::array::IntoArray; use crate::array_session; use crate::builders::ArrayBuilder; use crate::builders::builder_with_capacity; @@ -223,12 +226,10 @@ fn test_append_defaults_behavior(#[case] dtype: DType, #[case] should_be_null: b /// Helper function that fills two builders with the same values and compares the results /// of `to_canonical()` vs `finish().to_canonical()`. -fn compare_to_canonical_methods(dtype: &DType, mut fill_builder: F) +fn compare_to_canonical_methods(dtype: &DType, ctx: &mut ExecutionCtx, mut fill_builder: F) where F: FnMut(&mut dyn ArrayBuilder), { - use crate::IntoArray; - // Create two identical builders. let mut builder1 = builder_with_capacity(dtype, 10); let mut builder2 = builder_with_capacity(dtype, 10); @@ -238,11 +239,10 @@ where fill_builder(builder2.as_mut()); // Get canonical arrays using both methods. - let canonical_direct = builder1.finish_into_canonical(); - #[expect(deprecated)] + let canonical_direct = builder1.finish_into_canonical(ctx); let canonical_indirect = builder2 .finish() - .to_canonical() + .execute::(ctx) .vortex_expect("to_canonical failed"); // Convert both to arrays for comparison. @@ -254,12 +254,8 @@ where // Compare each element. for i in 0..array_direct.len() { - let scalar_direct = array_direct - .execute_scalar(i, &mut array_session().create_execution_ctx()) - .unwrap(); - let scalar_indirect = array_indirect - .execute_scalar(i, &mut array_session().create_execution_ctx()) - .unwrap(); + let scalar_direct = array_direct.execute_scalar(i, ctx).unwrap(); + let scalar_indirect = array_indirect.execute_scalar(i, ctx).unwrap(); assert_eq!( scalar_direct, scalar_indirect, @@ -271,8 +267,9 @@ where #[test] fn test_to_canonical_bool() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Bool(Nullability::NonNullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for i in 0..5 { let value = Scalar::bool(i % 2 == 0, Nullability::NonNullable); builder.append_scalar(&value).unwrap(); @@ -282,8 +279,9 @@ fn test_to_canonical_bool() { #[test] fn test_to_canonical_bool_nullable() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Bool(Nullability::Nullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for i in 0..5 { let value = Scalar::bool(i % 2 == 0, Nullability::Nullable); builder.append_scalar(&value).unwrap(); @@ -294,8 +292,9 @@ fn test_to_canonical_bool_nullable() { #[test] fn test_to_canonical_i32() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Primitive(PType::I32, Nullability::NonNullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for i in 0..5 { let value = Scalar::primitive(i, Nullability::NonNullable); builder.append_scalar(&value).unwrap(); @@ -305,8 +304,9 @@ fn test_to_canonical_i32() { #[test] fn test_to_canonical_i32_nullable() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Primitive(PType::I32, Nullability::Nullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for i in 0..5 { let value = Scalar::primitive(i, Nullability::Nullable); builder.append_scalar(&value).unwrap(); @@ -317,8 +317,9 @@ fn test_to_canonical_i32_nullable() { #[test] fn test_to_canonical_f64() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Primitive(PType::F64, Nullability::NonNullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for i in 0..5 { let value = Scalar::primitive(i as f64, Nullability::NonNullable); builder.append_scalar(&value).unwrap(); @@ -328,8 +329,9 @@ fn test_to_canonical_f64() { #[test] fn test_to_canonical_utf8() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Utf8(Nullability::NonNullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { let values = ["hello", "world", "test", "data", "vortex"]; for value in &values { let scalar = Scalar::utf8(*value, Nullability::NonNullable); @@ -340,8 +342,9 @@ fn test_to_canonical_utf8() { #[test] fn test_to_canonical_utf8_nullable() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Utf8(Nullability::Nullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { let values = ["hello", "world", "test"]; for value in &values { let scalar = Scalar::utf8(*value, Nullability::Nullable); @@ -353,8 +356,9 @@ fn test_to_canonical_utf8_nullable() { #[test] fn test_to_canonical_binary() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Binary(Nullability::NonNullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { let values = [b"hello", b"world", b"vortx", b"bytes", b"tests"]; for value in &values { let scalar = Scalar::binary(value.to_vec(), Nullability::NonNullable); @@ -365,6 +369,7 @@ fn test_to_canonical_binary() { #[test] fn test_to_canonical_struct() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Struct( StructFields::from_iter([ ("a", DType::Primitive(PType::I32, Nullability::NonNullable)), @@ -372,7 +377,7 @@ fn test_to_canonical_struct() { ]), Nullability::NonNullable, ); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for _ in 0..3 { let value = Scalar::default_value(&dtype); builder.append_scalar(&value).unwrap(); @@ -382,9 +387,10 @@ fn test_to_canonical_struct() { #[test] fn test_to_canonical_extension() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Extension(Timestamp::new(TimeUnit::Milliseconds, Nullability::NonNullable).erased()); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { let ext_dtype = match &dtype { DType::Extension(ext) => ext.clone(), _ => unreachable!(), @@ -399,16 +405,18 @@ fn test_to_canonical_extension() { #[test] fn test_to_canonical_null() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Null; - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { builder.append_nulls(5); }); } #[test] fn test_to_canonical_decimal() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Decimal(DecimalDType::new(10, 2), Nullability::NonNullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for _ in 0..5 { let value = Scalar::default_value(&dtype); builder.append_scalar(&value).unwrap(); @@ -418,8 +426,9 @@ fn test_to_canonical_decimal() { #[test] fn test_to_canonical_i8() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Primitive(PType::I8, Nullability::NonNullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for i in 0..5i8 { let value = Scalar::primitive(i, Nullability::NonNullable); builder.append_scalar(&value).unwrap(); @@ -429,8 +438,9 @@ fn test_to_canonical_i8() { #[test] fn test_to_canonical_u64() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Primitive(PType::U64, Nullability::NonNullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for i in 0..5 { let value = Scalar::primitive(i as u64, Nullability::NonNullable); builder.append_scalar(&value).unwrap(); @@ -440,8 +450,9 @@ fn test_to_canonical_u64() { #[test] fn test_to_canonical_f32() { + let mut ctx = array_session().create_execution_ctx(); let dtype = DType::Primitive(PType::F32, Nullability::NonNullable); - compare_to_canonical_methods(&dtype, |builder| { + compare_to_canonical_methods(&dtype, &mut ctx, |builder| { for i in 0..5 { let value = Scalar::primitive(i as f32, Nullability::NonNullable); builder.append_scalar(&value).unwrap(); diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index 3aabfd87c31..1d530a384ef 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -22,8 +22,6 @@ use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::VarBinViewArrayExt; use crate::arrays::varbinview::build_views::BinaryView; @@ -31,8 +29,6 @@ use crate::arrays::varbinview::compact::BufferUtilization; use crate::builders::ArrayBuilder; use crate::builders::LazyBitBufferBuilder; use crate::canonical::Canonical; -#[expect(deprecated)] -use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::scalar::Scalar; @@ -280,6 +276,7 @@ impl VarBinViewBuilder { .extend_from_compaction(BuffersWithOffsets::from_array( array, self.compaction_threshold, + ctx, )); match view_adjustment { @@ -372,13 +369,6 @@ impl ArrayBuilder for VarBinViewBuilder { Ok(()) } - unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { - #[expect(deprecated)] - let array = array.to_varbinview(); - self.append_varbinview_array(&array, &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("Failed to append varbinview array"); - } - fn reserve_exact(&mut self, additional: usize) { self.views_builder.reserve(additional); self.nulls.reserve_exact(additional); @@ -392,7 +382,7 @@ impl ArrayBuilder for VarBinViewBuilder { self.finish_into_varbinview().into_array() } - fn finish_into_canonical(&mut self) -> Canonical { + fn finish_into_canonical(&mut self, _ctx: &mut ExecutionCtx) -> Canonical { Canonical::VarBinView(self.finish_into_varbinview()) } } @@ -636,7 +626,11 @@ enum BuffersWithOffsets { } impl BuffersWithOffsets { - pub fn from_array(array: &VarBinViewArray, compaction_threshold: f64) -> Self { + pub fn from_array( + array: &VarBinViewArray, + compaction_threshold: f64, + ctx: &mut ExecutionCtx, + ) -> Self { if compaction_threshold == 0.0 { return Self::AllKept { buffers: Arc::from( @@ -652,7 +646,7 @@ impl BuffersWithOffsets { } let buffer_utilizations = array - .buffer_utilizations() + .buffer_utilizations(ctx) .vortex_expect("buffer_utilizations in BuffersWithOffsets::from_array"); let mut has_rewrite = false; let mut has_nonzero_offset = false; @@ -923,11 +917,11 @@ mod tests { let mut builder = VarBinViewBuilder::with_capacity(DType::Utf8(Nullability::Nullable), 10); builder.append_value("Hello1"); - builder.extend_from_array(&array); + array.append_to_builder(&mut builder, &mut ctx).unwrap(); builder.append_nulls(2); builder.append_value("Hello3"); - let actual = builder.finish_into_canonical(); + let actual = builder.finish_into_canonical(&mut ctx); let expected = >::from_iter([ Some("Hello1"), None, diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index 28953a93e4e..cda03098efa 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -262,7 +262,7 @@ impl Canonical { /// and copy operations. pub fn compact(&self, ctx: &mut ExecutionCtx) -> VortexResult { match self { - Canonical::VarBinView(array) => Ok(Canonical::VarBinView(array.compact_buffers()?)), + Canonical::VarBinView(array) => Ok(Canonical::VarBinView(array.compact_buffers(ctx)?)), Canonical::List(array) => Ok(Canonical::List( array.rebuild(ListViewRebuildMode::TrimElements, ctx)?, )), diff --git a/vortex-array/src/compute/conformance/cast.rs b/vortex-array/src/compute/conformance/cast.rs index c823b2a778a..25bebaef26a 100644 --- a/vortex-array/src/compute/conformance/cast.rs +++ b/vortex-array/src/compute/conformance/cast.rs @@ -6,13 +6,12 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::RecursiveCanonical; -use crate::VortexSessionExecute; use crate::aggregate_fn::NumericalAggregateOpts; use crate::aggregate_fn::fns::min_max::MinMaxResult; use crate::aggregate_fn::fns::min_max::min_max; -use crate::array_session; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::Nullability; @@ -20,10 +19,14 @@ use crate::dtype::PType; use crate::scalar::Scalar; /// Cast and force execution via `to_canonical`, returning the canonical array. -fn cast_and_execute(array: &ArrayRef, dtype: DType) -> VortexResult { +fn cast_and_execute( + array: &ArrayRef, + dtype: DType, + ctx: &mut ExecutionCtx, +) -> VortexResult { Ok(array .cast(dtype)? - .execute::(&mut array_session().create_execution_ctx())? + .execute::(ctx)? .0 .into_array()) } @@ -37,18 +40,18 @@ fn cast_and_execute(array: &ArrayRef, dtype: DType) -> VortexResult { /// - Casting with nullability changes /// - Casting between string types (Utf8/Binary) /// - Edge cases like overflow behavior -pub fn test_cast_conformance(array: &ArrayRef) { +pub fn test_cast_conformance(array: &ArrayRef, ctx: &mut ExecutionCtx) { let dtype = array.dtype(); // Always test identity cast and nullability changes - test_cast_identity(array); + test_cast_identity(array, ctx); - test_cast_to_non_nullable(array); - test_cast_to_nullable(array); + test_cast_to_non_nullable(array, ctx); + test_cast_to_nullable(array, ctx); // Test based on the specific DType match dtype { - DType::Null => test_cast_from_null(array), + DType::Null => test_cast_from_null(array, ctx), DType::Primitive(ptype, ..) => match ptype { PType::U8 | PType::U16 @@ -57,16 +60,16 @@ pub fn test_cast_conformance(array: &ArrayRef) { | PType::I8 | PType::I16 | PType::I32 - | PType::I64 => test_cast_to_integral_types(array), - PType::F16 | PType::F32 | PType::F64 => test_cast_from_floating_point_types(array), + | PType::I64 => test_cast_to_integral_types(array, ctx), + PType::F16 | PType::F32 | PType::F64 => test_cast_from_floating_point_types(array, ctx), }, _ => {} } } -fn test_cast_identity(array: &ArrayRef) { +fn test_cast_identity(array: &ArrayRef, ctx: &mut ExecutionCtx) { // Casting to the same type should be a no-op - let result = cast_and_execute(&array.clone(), array.dtype().clone()) + let result = cast_and_execute(&array.clone(), array.dtype().clone(), ctx) .vortex_expect("cast should succeed in conformance test"); assert_eq!(result.len(), array.len()); assert_eq!(result.dtype(), array.dtype()); @@ -75,18 +78,18 @@ fn test_cast_identity(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } } -fn test_cast_from_null(array: &ArrayRef) { +fn test_cast_from_null(array: &ArrayRef, ctx: &mut ExecutionCtx) { // Null can be cast to itself - let result = cast_and_execute(&array.clone(), DType::Null) + let result = cast_and_execute(&array.clone(), DType::Null, ctx) .vortex_expect("cast should succeed in conformance test"); assert_eq!(result.len(), array.len()); assert_eq!(result.dtype(), &DType::Null); @@ -101,7 +104,7 @@ fn test_cast_from_null(array: &ArrayRef) { ]; for dtype in nullable_types { - let result = cast_and_execute(&array.clone(), dtype.clone()) + let result = cast_and_execute(&array.clone(), dtype.clone(), ctx) .vortex_expect("cast should succeed in conformance test"); assert_eq!(result.len(), array.len()); assert_eq!(result.dtype(), &dtype); @@ -110,7 +113,7 @@ fn test_cast_from_null(array: &ArrayRef) { for i in 0..array.len().min(10) { assert!( result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") .is_null() ); @@ -124,17 +127,17 @@ fn test_cast_from_null(array: &ArrayRef) { ]; for dtype in non_nullable_types { - assert!(cast_and_execute(&array.clone(), dtype.clone()).is_err()); + assert!(cast_and_execute(&array.clone(), dtype.clone(), ctx).is_err()); } } -fn test_cast_to_non_nullable(array: &ArrayRef) { +fn test_cast_to_non_nullable(array: &ArrayRef, ctx: &mut ExecutionCtx) { if array - .invalid_count(&mut array_session().create_execution_ctx()) + .invalid_count(ctx) .vortex_expect("invalid_count should succeed in conformance test") == 0 { - let non_nullable = cast_and_execute(&array.clone(), array.dtype().as_nonnullable()) + let non_nullable = cast_and_execute(&array.clone(), array.dtype().as_nonnullable(), ctx) .vortex_expect("arrays without nulls can cast to non-nullable"); assert_eq!(non_nullable.dtype(), &array.dtype().as_nonnullable()); assert_eq!(non_nullable.len(), array.len()); @@ -142,15 +145,15 @@ fn test_cast_to_non_nullable(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), non_nullable - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } - let back_to_nullable = cast_and_execute(&non_nullable, array.dtype().clone()) + let back_to_nullable = cast_and_execute(&non_nullable, array.dtype().clone(), ctx) .vortex_expect("non-nullable arrays can cast to nullable"); assert_eq!(back_to_nullable.dtype(), array.dtype()); assert_eq!(back_to_nullable.len(), array.len()); @@ -158,10 +161,10 @@ fn test_cast_to_non_nullable(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), back_to_nullable - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -171,7 +174,7 @@ fn test_cast_to_non_nullable(array: &ArrayRef) { // array can be casted to DType::Null. return; } - cast_and_execute(&array.clone(), array.dtype().as_nonnullable()) + cast_and_execute(&array.clone(), array.dtype().as_nonnullable(), ctx) .err() .unwrap_or_else(|| { vortex_panic!( @@ -182,8 +185,8 @@ fn test_cast_to_non_nullable(array: &ArrayRef) { } } -fn test_cast_to_nullable(array: &ArrayRef) { - let nullable = cast_and_execute(&array.clone(), array.dtype().as_nullable()) +fn test_cast_to_nullable(array: &ArrayRef, ctx: &mut ExecutionCtx) { + let nullable = cast_and_execute(&array.clone(), array.dtype().as_nullable(), ctx) .vortex_expect("arrays without nulls can cast to nullable"); assert_eq!(nullable.dtype(), &array.dtype().as_nullable()); assert_eq!(nullable.len(), array.len()); @@ -191,15 +194,15 @@ fn test_cast_to_nullable(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), nullable - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } - let back = cast_and_execute(&nullable, array.dtype().clone()) + let back = cast_and_execute(&nullable, array.dtype().clone(), ctx) .vortex_expect("casting to nullable and back should be a no-op"); assert_eq!(back.dtype(), array.dtype()); assert_eq!(back.len(), array.len()); @@ -207,38 +210,43 @@ fn test_cast_to_nullable(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), - back.execute_scalar(i, &mut array_session().create_execution_ctx()) + back.execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } } -fn test_cast_from_floating_point_types(array: &ArrayRef) { +fn test_cast_from_floating_point_types(array: &ArrayRef, ctx: &mut ExecutionCtx) { let ptype = array.as_primitive_typed().ptype(); - test_cast_to_primitive(array, PType::I8, false); - test_cast_to_primitive(array, PType::U8, false); - test_cast_to_primitive(array, PType::I16, false); - test_cast_to_primitive(array, PType::U16, false); - test_cast_to_primitive(array, PType::I32, false); - test_cast_to_primitive(array, PType::U32, false); - test_cast_to_primitive(array, PType::I64, false); - test_cast_to_primitive(array, PType::U64, false); - test_cast_to_primitive(array, PType::F16, matches!(ptype, PType::F16)); - test_cast_to_primitive(array, PType::F32, matches!(ptype, PType::F16 | PType::F32)); - test_cast_to_primitive(array, PType::F64, true); + test_cast_to_primitive(array, PType::I8, false, ctx); + test_cast_to_primitive(array, PType::U8, false, ctx); + test_cast_to_primitive(array, PType::I16, false, ctx); + test_cast_to_primitive(array, PType::U16, false, ctx); + test_cast_to_primitive(array, PType::I32, false, ctx); + test_cast_to_primitive(array, PType::U32, false, ctx); + test_cast_to_primitive(array, PType::I64, false, ctx); + test_cast_to_primitive(array, PType::U64, false, ctx); + test_cast_to_primitive(array, PType::F16, matches!(ptype, PType::F16), ctx); + test_cast_to_primitive( + array, + PType::F32, + matches!(ptype, PType::F16 | PType::F32), + ctx, + ); + test_cast_to_primitive(array, PType::F64, true, ctx); } -fn test_cast_to_integral_types(array: &ArrayRef) { - test_cast_to_primitive(array, PType::I8, true); - test_cast_to_primitive(array, PType::U8, true); - test_cast_to_primitive(array, PType::I16, true); - test_cast_to_primitive(array, PType::U16, true); - test_cast_to_primitive(array, PType::I32, true); - test_cast_to_primitive(array, PType::U32, true); - test_cast_to_primitive(array, PType::I64, true); - test_cast_to_primitive(array, PType::U64, true); +fn test_cast_to_integral_types(array: &ArrayRef, ctx: &mut ExecutionCtx) { + test_cast_to_primitive(array, PType::I8, true, ctx); + test_cast_to_primitive(array, PType::U8, true, ctx); + test_cast_to_primitive(array, PType::I16, true, ctx); + test_cast_to_primitive(array, PType::U16, true, ctx); + test_cast_to_primitive(array, PType::I32, true, ctx); + test_cast_to_primitive(array, PType::U32, true, ctx); + test_cast_to_primitive(array, PType::I64, true, ctx); + test_cast_to_primitive(array, PType::U64, true, ctx); } /// Does this scalar fit in this type? @@ -247,9 +255,13 @@ fn fits(value: &Scalar, ptype: PType) -> bool { value.cast(&dtype).is_ok() } -fn test_cast_to_primitive(array: &ArrayRef, target_ptype: PType, test_round_trip: bool) { - let mut ctx = array_session().create_execution_ctx(); - let maybe_min_max = min_max(array, &mut ctx, NumericalAggregateOpts::default()) +fn test_cast_to_primitive( + array: &ArrayRef, + target_ptype: PType, + test_round_trip: bool, + ctx: &mut ExecutionCtx, +) { + let maybe_min_max = min_max(array, ctx, NumericalAggregateOpts::default()) .vortex_expect("cast should succeed in conformance test"); if let Some(MinMaxResult { min, max }) = maybe_min_max @@ -258,6 +270,7 @@ fn test_cast_to_primitive(array: &ArrayRef, target_ptype: PType, test_round_trip cast_and_execute( &array.clone(), DType::Primitive(target_ptype, array.dtype().nullability()), + ctx, ) .err() .unwrap_or_else(|| { @@ -277,6 +290,7 @@ fn test_cast_to_primitive(array: &ArrayRef, target_ptype: PType, test_round_trip let casted = cast_and_execute( &array.clone(), DType::Primitive(target_ptype, array.dtype().nullability()), + ctx, ) .unwrap_or_else(|e| { vortex_panic!( @@ -289,20 +303,20 @@ fn test_cast_to_primitive(array: &ArrayRef, target_ptype: PType, test_round_trip array .validity() .vortex_expect("validity_mask should succeed in conformance test") - .execute_mask(array.len(), &mut array_session().create_execution_ctx()) + .execute_mask(array.len(), ctx) .vortex_expect("Failed to compute validity mask"), casted .validity() .vortex_expect("validity_mask should succeed in conformance test") - .execute_mask(casted.len(), &mut array_session().create_execution_ctx()) + .execute_mask(casted.len(), ctx) .vortex_expect("Failed to compute validity mask") ); for i in 0..array.len().min(10) { let original = array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"); let casted = casted - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( original @@ -325,10 +339,15 @@ fn test_cast_to_primitive(array: &ArrayRef, target_ptype: PType, test_round_trip #[cfg(test)] mod tests { + use std::sync::LazyLock; + use vortex_buffer::buffer; + use vortex_session::VortexSession; use super::*; use crate::IntoArray; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::BoolArray; use crate::arrays::ListArray; use crate::arrays::NullArray; @@ -339,40 +358,42 @@ mod tests { use crate::dtype::FieldNames; use crate::dtype::Nullability; + static SESSION: LazyLock = LazyLock::new(array_session); + #[test] fn test_cast_conformance_u32() { let array = buffer![0u32, 100, 200, 65535, 1000000].into_array(); - test_cast_conformance(&array); + test_cast_conformance(&array, &mut SESSION.create_execution_ctx()); } #[test] fn test_cast_conformance_i32() { let array = buffer![-100i32, -1, 0, 1, 100].into_array(); - test_cast_conformance(&array); + test_cast_conformance(&array, &mut SESSION.create_execution_ctx()); } #[test] fn test_cast_conformance_f32() { let array = buffer![0.0f32, 1.5, -2.5, 100.0, 1e6].into_array(); - test_cast_conformance(&array); + test_cast_conformance(&array, &mut SESSION.create_execution_ctx()); } #[test] fn test_cast_conformance_nullable() { let array = PrimitiveArray::from_option_iter([Some(1u8), None, Some(255), Some(0), None]); - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] fn test_cast_conformance_bool() { let array = BoolArray::from_iter(vec![true, false, true, false]); - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] fn test_cast_conformance_null() { let array = NullArray::new(5); - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -381,7 +402,7 @@ mod tests { vec![Some("hello"), None, Some("world")], DType::Utf8(Nullability::Nullable), ); - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -390,7 +411,7 @@ mod tests { vec![Some(b"data".as_slice()), None, Some(b"bytes".as_slice())], DType::Binary(Nullability::Nullable), ); - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -407,7 +428,7 @@ mod tests { let array = StructArray::try_new(names, vec![a, b], 3, crate::validity::Validity::NonNullable) .unwrap(); - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } #[test] @@ -417,6 +438,6 @@ mod tests { let array = ListArray::try_new(data, offsets, crate::validity::Validity::NonNullable).unwrap(); - test_cast_conformance(&array.into_array()); + test_cast_conformance(&array.into_array(), &mut SESSION.create_execution_ctx()); } } diff --git a/vortex-array/src/compute/conformance/filter.rs b/vortex-array/src/compute/conformance/filter.rs index 01bfb3bb2fa..5c78e0d70ad 100644 --- a/vortex-array/src/compute/conformance/filter.rs +++ b/vortex-array/src/compute/conformance/filter.rs @@ -5,9 +5,9 @@ use vortex_error::VortexExpect; use vortex_mask::Mask; use crate::ArrayRef; +use crate::Canonical; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::VortexSessionExecute; -use crate::array_session; use crate::assert_arrays_eq; use crate::dtype::DType; @@ -18,16 +18,16 @@ pub const LARGE_SIZE: usize = 1024; /// Test filter compute function with various array sizes and patterns. /// The input array can be of any length. -pub fn test_filter_conformance(array: &ArrayRef) { +pub fn test_filter_conformance(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); // Test with arrays of any size if len > 0 { - test_all_filter(array); + test_all_filter(array, ctx); test_none_filter(array); - test_selective_filter(array); - test_single_element_filter(array); - test_alternating_pattern_filter(array); + test_selective_filter(array, ctx); + test_single_element_filter(array, ctx); + test_alternating_pattern_filter(array, ctx); test_runs_pattern_filter(array); test_sparse_true_filter(array); test_sparse_false_filter(array); @@ -61,14 +61,13 @@ pub fn create_runs_pattern(len: usize, run_length: usize) -> Vec { } /// Tests that filtering with an all-true mask returns all elements unchanged -fn test_all_filter(array: &ArrayRef) { - let mut ctx = array_session().create_execution_ctx(); +fn test_all_filter(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); let mask = Mask::new_true(len); let filtered = array .filter(mask) .vortex_expect("filter should succeed in conformance test"); - assert_arrays_eq!(filtered, array, &mut ctx); + assert_arrays_eq!(filtered, array, ctx); } /// Tests that filtering with an all-false mask returns an empty array with the same dtype @@ -82,7 +81,7 @@ fn test_none_filter(array: &ArrayRef) { assert_eq!(filtered.dtype(), array.dtype()); } -fn test_selective_filter(array: &ArrayRef) { +fn test_selective_filter(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); if len < 2 { return; // Skip for very small arrays @@ -101,10 +100,10 @@ fn test_selective_filter(array: &ArrayRef) { for (filtered_idx, i) in (0..len).step_by(2).enumerate() { assert_eq!( filtered - .execute_scalar(filtered_idx, &mut array_session().create_execution_ctx()) + .execute_scalar(filtered_idx, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -121,24 +120,24 @@ fn test_selective_filter(array: &ArrayRef) { assert_eq!(filtered.len(), 2); assert_eq!( filtered - .execute_scalar(0, &mut array_session().create_execution_ctx()) + .execute_scalar(0, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(0, &mut array_session().create_execution_ctx()) + .execute_scalar(0, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); assert_eq!( filtered - .execute_scalar(1, &mut array_session().create_execution_ctx()) + .execute_scalar(1, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(len - 1, &mut array_session().create_execution_ctx()) + .execute_scalar(len - 1, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } } -fn test_single_element_filter(array: &ArrayRef) { +fn test_single_element_filter(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); if len == 0 { return; @@ -154,10 +153,10 @@ fn test_single_element_filter(array: &ArrayRef) { assert_eq!(filtered.len(), 1); assert_eq!( filtered - .execute_scalar(0, &mut array_session().create_execution_ctx()) + .execute_scalar(0, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(0, &mut array_session().create_execution_ctx()) + .execute_scalar(0, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); @@ -172,18 +171,16 @@ fn test_single_element_filter(array: &ArrayRef) { assert_eq!(filtered.len(), 1); assert_eq!( filtered - .execute_scalar(0, &mut array_session().create_execution_ctx()) + .execute_scalar(0, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(len - 1, &mut array_session().create_execution_ctx()) + .execute_scalar(len - 1, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } } fn test_empty_array_filter(dtype: &DType) { - use crate::Canonical; - let empty_array = Canonical::empty(dtype).into_array(); let empty_mask = Mask::new_false(0); let filtered = empty_array @@ -221,7 +218,7 @@ fn test_mismatched_lengths(array: &ArrayRef) { } /// Tests filtering with alternating true/false pattern -fn test_alternating_pattern_filter(array: &ArrayRef) { +fn test_alternating_pattern_filter(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); let pattern = create_alternating_pattern(len); let expected_count = pattern.iter().filter(|&&v| v).count(); @@ -238,10 +235,10 @@ fn test_alternating_pattern_filter(array: &ArrayRef) { if keep { assert_eq!( filtered - .execute_scalar(filtered_idx, &mut array_session().create_execution_ctx()) + .execute_scalar(filtered_idx, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); filtered_idx += 1; diff --git a/vortex-array/src/compute/conformance/mask.rs b/vortex-array/src/compute/conformance/mask.rs index 7a002662d38..50930ba2876 100644 --- a/vortex-array/src/compute/conformance/mask.rs +++ b/vortex-array/src/compute/conformance/mask.rs @@ -5,38 +5,38 @@ use vortex_error::VortexExpect; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; -use crate::VortexSessionExecute; -use crate::array_session; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; +use crate::validity::Validity; /// Test mask compute function with various array sizes and patterns. /// The mask operation sets elements to null where the mask is true. -pub fn test_mask_conformance(array: &ArrayRef) { +pub fn test_mask_conformance(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); if len > 0 { - test_heterogenous_mask(array); - test_empty_mask(array); - test_full_mask(array); - test_alternating_mask(array); - test_sparse_mask(array); - test_single_element_mask(array); + test_heterogenous_mask(array, ctx); + test_empty_mask(array, ctx); + test_full_mask(array, ctx); + test_alternating_mask(array, ctx); + test_sparse_mask(array, ctx); + test_single_element_mask(array, ctx); } if len >= 5 { - test_double_mask(array); + test_double_mask(array, ctx); } if len > 0 { - test_nullable_mask_input(array); + test_nullable_mask_input(array, ctx); } } /// Tests masking with a heterogeneous pattern -fn test_heterogenous_mask(array: &ArrayRef) { +fn test_heterogenous_mask(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); // Create a pattern where roughly half the values are masked @@ -54,16 +54,16 @@ fn test_heterogenous_mask(array: &ArrayRef) { if masked_out { assert!( !masked - .is_valid(i, &mut array_session().create_execution_ctx()) + .is_valid(i, ctx) .vortex_expect("is_valid should succeed in conformance test") ); } else { assert_eq!( masked - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -72,7 +72,7 @@ fn test_heterogenous_mask(array: &ArrayRef) { } /// Tests that an empty mask (all false) preserves all elements -fn test_empty_mask(array: &ArrayRef) { +fn test_empty_mask(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); let all_unmasked = vec![false; len]; let mask_array = Mask::from_iter(all_unmasked); @@ -87,10 +87,10 @@ fn test_empty_mask(array: &ArrayRef) { for i in 0..len { assert_eq!( masked - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -98,7 +98,7 @@ fn test_empty_mask(array: &ArrayRef) { } /// Tests that a full mask (all true) makes all elements null -fn test_full_mask(array: &ArrayRef) { +fn test_full_mask(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); let all_masked = vec![true; len]; let mask_array = Mask::from_iter(all_masked); @@ -113,14 +113,14 @@ fn test_full_mask(array: &ArrayRef) { for i in 0..len { assert!( !masked - .is_valid(i, &mut array_session().create_execution_ctx()) + .is_valid(i, ctx) .vortex_expect("is_valid should succeed in conformance test") ); } } /// Tests alternating mask pattern -fn test_alternating_mask(array: &ArrayRef) { +fn test_alternating_mask(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); let pattern: Vec = (0..len).map(|i| i % 2 == 0).collect(); let mask_array = Mask::from_iter(pattern); @@ -135,16 +135,16 @@ fn test_alternating_mask(array: &ArrayRef) { if i % 2 == 0 { assert!( !masked - .is_valid(i, &mut array_session().create_execution_ctx()) + .is_valid(i, ctx) .vortex_expect("is_valid should succeed in conformance test") ); } else { assert_eq!( masked - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -153,7 +153,7 @@ fn test_alternating_mask(array: &ArrayRef) { } /// Tests sparse mask (only a few elements masked) -fn test_sparse_mask(array: &ArrayRef) { +fn test_sparse_mask(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); if len < 10 { return; // Skip for small arrays @@ -173,7 +173,7 @@ fn test_sparse_mask(array: &ArrayRef) { let valid_count = (0..len) .filter(|&i| { masked - .is_valid(i, &mut array_session().create_execution_ctx()) + .is_valid(i, ctx) .vortex_expect("is_valid should succeed in conformance test") }) .count(); @@ -185,7 +185,7 @@ fn test_sparse_mask(array: &ArrayRef) { .filter(|&i| { pattern[i] || !array - .is_valid(i, &mut array_session().create_execution_ctx()) + .is_valid(i, ctx) .vortex_expect("is_valid should succeed in conformance test") }) .count(); @@ -194,7 +194,7 @@ fn test_sparse_mask(array: &ArrayRef) { } /// Tests masking a single element -fn test_single_element_mask(array: &ArrayRef) { +fn test_single_element_mask(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); // Mask only the first element @@ -208,17 +208,17 @@ fn test_single_element_mask(array: &ArrayRef) { .vortex_expect("mask should succeed in conformance test"); assert!( !masked - .is_valid(0, &mut array_session().create_execution_ctx()) + .is_valid(0, ctx) .vortex_expect("is_valid should succeed in conformance test") ); for i in 1..len { assert_eq!( masked - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -226,7 +226,7 @@ fn test_single_element_mask(array: &ArrayRef) { } /// Tests double masking operations -fn test_double_mask(array: &ArrayRef) { +fn test_double_mask(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); // Create two different mask patterns @@ -249,16 +249,16 @@ fn test_double_mask(array: &ArrayRef) { if mask1_pattern[i] || mask2_pattern[i] { assert!( !double_masked - .is_valid(i, &mut array_session().create_execution_ctx()) + .is_valid(i, ctx) .vortex_expect("is_valid should succeed in conformance test") ); } else { assert_eq!( double_masked - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -267,7 +267,7 @@ fn test_double_mask(array: &ArrayRef) { } /// Tests masking with nullable mask (nulls treated as false) -fn test_nullable_mask_input(array: &ArrayRef) { +fn test_nullable_mask_input(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); if len < 3 { return; // Skip for very small arrays @@ -278,11 +278,10 @@ fn test_nullable_mask_input(array: &ArrayRef) { let validity_values: Vec = (0..len).map(|i| i % 3 != 0).collect(); let bool_array = BoolArray::from_iter(bool_values.clone()); - let validity = crate::validity::Validity::from_iter(validity_values.clone()); + let validity = Validity::from_iter(validity_values.clone()); let nullable_mask = BoolArray::new(bool_array.to_bit_buffer(), validity); - let mask_array = - nullable_mask.to_mask_fill_null_false(&mut array_session().create_execution_ctx()); + let mask_array = nullable_mask.to_mask_fill_null_false(ctx); let masked = array .clone() .mask((!&mask_array).into_array()) @@ -293,16 +292,16 @@ fn test_nullable_mask_input(array: &ArrayRef) { if bool_values[i] && validity_values[i] { assert!( !masked - .is_valid(i, &mut array_session().create_execution_ctx()) + .is_valid(i, ctx) .vortex_expect("is_valid should succeed in conformance test") ); } else { assert_eq!( masked - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); diff --git a/vortex-array/src/compute/conformance/take.rs b/vortex-array/src/compute/conformance/take.rs index babaf392eb9..47bc8f1bd82 100644 --- a/vortex-array/src/compute/conformance/take.rs +++ b/vortex-array/src/compute/conformance/take.rs @@ -6,9 +6,8 @@ use vortex_error::VortexExpect; use crate::ArrayRef; use crate::Canonical; +use crate::ExecutionCtx; use crate::IntoArray as _; -use crate::VortexSessionExecute; -use crate::array_session; use crate::arrays::PrimitiveArray; use crate::dtype::Nullability; @@ -21,39 +20,39 @@ use crate::dtype::Nullability; /// - Taking with out-of-bounds indices (should panic) /// - Taking with nullable indices /// - Edge cases like empty arrays -pub fn test_take_conformance(array: &ArrayRef) { +pub fn test_take_conformance(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); if len > 0 { - test_take_all(array); + test_take_all(array, ctx); test_take_none(array); - test_take_selective(array); - test_take_first_and_last(array); - test_take_with_nullable_indices(array); - test_take_repeated_indices(array); + test_take_selective(array, ctx); + test_take_first_and_last(array, ctx); + test_take_with_nullable_indices(array, ctx); + test_take_repeated_indices(array, ctx); } test_empty_indices(array); // Additional edge cases for non-empty arrays if len > 0 { - test_take_reverse(array); - test_take_single_middle(array); + test_take_reverse(array, ctx); + test_take_single_middle(array, ctx); } if len > 3 { - test_take_random_unsorted(array); - test_take_contiguous_range(array); - test_take_mixed_repeated(array); + test_take_random_unsorted(array, ctx); + test_take_contiguous_range(array, ctx); + test_take_mixed_repeated(array, ctx); } // Test for larger arrays if len >= 1024 { - test_take_large_indices(array); + test_take_large_indices(array, ctx); } } -fn test_take_all(array: &ArrayRef) { +fn test_take_all(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); let indices = PrimitiveArray::from_iter(0..len as u64); let result = array @@ -64,13 +63,13 @@ fn test_take_all(array: &ArrayRef) { assert_eq!(result.dtype(), array.dtype()); // Verify elements match - #[expect(deprecated)] let array_canonical = array - .to_canonical() + .clone() + .execute::(ctx) .vortex_expect("to_canonical failed on array"); - #[expect(deprecated)] let result_canonical = result - .to_canonical() + .clone() + .execute::(ctx) .vortex_expect("to_canonical failed on result"); match (array_canonical, result_canonical) { (Canonical::Primitive(orig_prim), Canonical::Primitive(result_prim)) => { @@ -84,10 +83,10 @@ fn test_take_all(array: &ArrayRef) { for i in 0..len { assert_eq!( array - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -106,7 +105,7 @@ fn test_take_none(array: &ArrayRef) { } #[expect(clippy::cast_possible_truncation)] -fn test_take_selective(array: &ArrayRef) { +fn test_take_selective(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); // Take every other element @@ -123,19 +122,16 @@ fn test_take_selective(array: &ArrayRef) { for (result_idx, &original_idx) in indices.iter().enumerate() { assert_eq!( array - .execute_scalar( - original_idx as usize, - &mut array_session().create_execution_ctx() - ) + .execute_scalar(original_idx as usize, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(result_idx, &mut array_session().create_execution_ctx()) + .execute_scalar(result_idx, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } } -fn test_take_first_and_last(array: &ArrayRef) { +fn test_take_first_and_last(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); let indices = PrimitiveArray::from_iter([0u64, (len - 1) as u64]); let result = array @@ -145,24 +141,24 @@ fn test_take_first_and_last(array: &ArrayRef) { assert_eq!(result.len(), 2); assert_eq!( array - .execute_scalar(0, &mut array_session().create_execution_ctx()) + .execute_scalar(0, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(0, &mut array_session().create_execution_ctx()) + .execute_scalar(0, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); assert_eq!( array - .execute_scalar(len - 1, &mut array_session().create_execution_ctx()) + .execute_scalar(len - 1, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(1, &mut array_session().create_execution_ctx()) + .execute_scalar(1, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } #[expect(clippy::cast_possible_truncation)] -fn test_take_with_nullable_indices(array: &ArrayRef) { +fn test_take_with_nullable_indices(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); // Create indices with some null values @@ -190,17 +186,17 @@ fn test_take_with_nullable_indices(array: &ArrayRef) { match idx_opt { Some(idx) => { let expected = array - .execute_scalar(*idx as usize, &mut array_session().create_execution_ctx()) + .execute_scalar(*idx as usize, ctx) .vortex_expect("scalar_at should succeed in conformance test"); let actual = result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!(expected, actual); } None => { assert!( result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") .is_null() ); @@ -209,7 +205,7 @@ fn test_take_with_nullable_indices(array: &ArrayRef) { } } -fn test_take_repeated_indices(array: &ArrayRef) { +fn test_take_repeated_indices(array: &ArrayRef, ctx: &mut ExecutionCtx) { if array.is_empty() { return; } @@ -222,12 +218,12 @@ fn test_take_repeated_indices(array: &ArrayRef) { assert_eq!(result.len(), 3); let first_elem = array - .execute_scalar(0, &mut array_session().create_execution_ctx()) + .execute_scalar(0, ctx) .vortex_expect("scalar_at should succeed in conformance test"); for i in 0..3 { assert_eq!( result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), first_elem ); @@ -244,7 +240,7 @@ fn test_empty_indices(array: &ArrayRef) { assert_eq!(result.dtype(), array.dtype()); } -fn test_take_reverse(array: &ArrayRef) { +fn test_take_reverse(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); // Take elements in reverse order let indices = PrimitiveArray::from_iter((0..len as u64).rev()); @@ -258,16 +254,16 @@ fn test_take_reverse(array: &ArrayRef) { for i in 0..len { assert_eq!( array - .execute_scalar(len - 1 - i, &mut array_session().create_execution_ctx()) + .execute_scalar(len - 1 - i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } } -fn test_take_single_middle(array: &ArrayRef) { +fn test_take_single_middle(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); let middle_idx = len / 2; @@ -279,20 +275,20 @@ fn test_take_single_middle(array: &ArrayRef) { assert_eq!(result.len(), 1); assert_eq!( array - .execute_scalar(middle_idx, &mut array_session().create_execution_ctx()) + .execute_scalar(middle_idx, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(0, &mut array_session().create_execution_ctx()) + .execute_scalar(0, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } #[expect(clippy::cast_possible_truncation)] -fn test_take_random_unsorted(array: &ArrayRef) { +fn test_take_random_unsorted(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); // Create a pseudo-random but deterministic pattern - let mut indices = Vec::new(); + let mut indices = Vec::with_capacity(len.min(10)); let mut idx = 1u64; for _ in 0..len.min(10) { indices.push((idx * 7 + 3) % len as u64); @@ -310,16 +306,16 @@ fn test_take_random_unsorted(array: &ArrayRef) { for (i, &idx) in indices.iter().enumerate() { assert_eq!( array - .execute_scalar(idx as usize, &mut array_session().create_execution_ctx()) + .execute_scalar(idx as usize, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } } -fn test_take_contiguous_range(array: &ArrayRef) { +fn test_take_contiguous_range(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); let start = len / 4; let end = len / 2; @@ -336,17 +332,17 @@ fn test_take_contiguous_range(array: &ArrayRef) { for i in 0..(end - start) { assert_eq!( array - .execute_scalar(start + i, &mut array_session().create_execution_ctx()) + .execute_scalar(start + i, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } } #[expect(clippy::cast_possible_truncation)] -fn test_take_mixed_repeated(array: &ArrayRef) { +fn test_take_mixed_repeated(array: &ArrayRef, ctx: &mut ExecutionCtx) { let len = array.len(); // Create pattern with some repeated indices @@ -372,17 +368,17 @@ fn test_take_mixed_repeated(array: &ArrayRef) { for (i, &idx) in indices.iter().enumerate() { assert_eq!( array - .execute_scalar(idx as usize, &mut array_session().create_execution_ctx()) + .execute_scalar(idx as usize, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } } #[expect(clippy::cast_possible_truncation)] -fn test_take_large_indices(array: &ArrayRef) { +fn test_take_large_indices(array: &ArrayRef, ctx: &mut ExecutionCtx) { // Test with a large number of indices to stress test performance let len = array.len(); let num_indices = 10000.min(len * 3); @@ -404,10 +400,10 @@ fn test_take_large_indices(array: &ArrayRef) { let expected_idx = indices[i] as usize; assert_eq!( array - .execute_scalar(expected_idx, &mut array_session().create_execution_ctx()) + .execute_scalar(expected_idx, ctx) .vortex_expect("scalar_at should succeed in conformance test"), result - .execute_scalar(i, &mut array_session().create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at should succeed in conformance test") ); } diff --git a/vortex-array/src/display/mod.rs b/vortex-array/src/display/mod.rs index 0134a28558c..cb23663d84f 100644 --- a/vortex-array/src/display/mod.rs +++ b/vortex-array/src/display/mod.rs @@ -6,6 +6,7 @@ mod extractors; mod tree_display; use std::fmt::Display; +use std::iter::repeat_n; pub use extractor::IndentedFormatter; pub use extractor::TreeContext; @@ -544,10 +545,7 @@ impl ArrayRef { "{opening_brace}{}{closing_brace}", (0..limit.saturating_sub(3)) .map(fmt_scalar) - .chain(std::iter::repeat_n( - "...".to_string(), - is_truncated as usize - )) + .chain(repeat_n("...".to_string(), is_truncated as usize)) .chain((self.len().saturating_sub(3)..self.len()).map(fmt_scalar)) .format(sep) ) diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 6f3801b8aa0..0a47087b839 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -496,9 +496,9 @@ impl Executable for ArrayRef { /// Execute `array` into the given `builder`. /// /// This uses the encoding's [`crate::array::VTable::append_to_builder`] implementation. Most -/// encodings use the default path of `execute::` followed by `builder.extend_from_array`, -/// while encodings like `Chunked` can override that to append child-by-child without materializing -/// the entire parent. +/// encodings use the default path of `execute::` followed by re-dispatching +/// `append_to_builder` on the canonical array, while encodings like `Chunked` can override that to +/// append child-by-child without materializing the entire parent. /// /// The builder must have a [`DType`] that is a nullability-superset of `array.dtype()`. pub fn execute_into_builder( diff --git a/vortex-array/src/expr/exprs.rs b/vortex-array/src/expr/exprs.rs index ec12a321552..2bdd5346e09 100644 --- a/vortex-array/src/expr/exprs.rs +++ b/vortex-array/src/expr/exprs.rs @@ -201,15 +201,17 @@ pub fn nested_case_when( /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray}; /// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::{IntoArray, ToCanonical}; +/// # use vortex_array::IntoArray; +/// # use vortex_array::{VortexSessionExecute, array_session}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; /// # use vortex_array::expr::{eq, root, lit}; /// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); /// let result = xs.into_array().apply(&eq(root(), lit(3))).unwrap(); +/// let mut ctx = array_session().create_execution_ctx(); /// /// assert_eq!( -/// result.to_bool().to_bit_buffer(), +/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), /// BoolArray::from_iter(vec![false, false, true]).to_bit_buffer(), /// ); /// ``` @@ -226,15 +228,17 @@ pub fn eq(lhs: Expression, rhs: Expression) -> Expression { /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray}; /// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::{ IntoArray, ToCanonical}; +/// # use vortex_array::IntoArray; +/// # use vortex_array::{VortexSessionExecute, array_session}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; /// # use vortex_array::expr::{root, lit, not_eq}; /// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); /// let result = xs.into_array().apply(¬_eq(root(), lit(3))).unwrap(); +/// let mut ctx = array_session().create_execution_ctx(); /// /// assert_eq!( -/// result.to_bool().to_bit_buffer(), +/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), /// BoolArray::from_iter(vec![true, true, false]).to_bit_buffer(), /// ); /// ``` @@ -251,15 +255,17 @@ pub fn not_eq(lhs: Expression, rhs: Expression) -> Expression { /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; /// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::{IntoArray, ToCanonical}; +/// # use vortex_array::IntoArray; +/// # use vortex_array::{VortexSessionExecute, array_session}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; /// # use vortex_array::expr::{gt_eq, root, lit}; /// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); /// let result = xs.into_array().apply(>_eq(root(), lit(3))).unwrap(); +/// let mut ctx = array_session().create_execution_ctx(); /// /// assert_eq!( -/// result.to_bool().to_bit_buffer(), +/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), /// BoolArray::from_iter(vec![false, false, true]).to_bit_buffer(), /// ); /// ``` @@ -276,15 +282,17 @@ pub fn gt_eq(lhs: Expression, rhs: Expression) -> Expression { /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; /// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::{IntoArray, ToCanonical}; +/// # use vortex_array::IntoArray; +/// # use vortex_array::{VortexSessionExecute, array_session}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; /// # use vortex_array::expr::{gt, root, lit}; /// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); /// let result = xs.into_array().apply(>(root(), lit(2))).unwrap(); +/// let mut ctx = array_session().create_execution_ctx(); /// /// assert_eq!( -/// result.to_bool().to_bit_buffer(), +/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), /// BoolArray::from_iter(vec![false, false, true]).to_bit_buffer(), /// ); /// ``` @@ -301,15 +309,17 @@ pub fn gt(lhs: Expression, rhs: Expression) -> Expression { /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; /// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::{IntoArray, ToCanonical}; +/// # use vortex_array::IntoArray; +/// # use vortex_array::{VortexSessionExecute, array_session}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; /// # use vortex_array::expr::{root, lit, lt_eq}; /// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); /// let result = xs.into_array().apply(<_eq(root(), lit(2))).unwrap(); +/// let mut ctx = array_session().create_execution_ctx(); /// /// assert_eq!( -/// result.to_bool().to_bit_buffer(), +/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), /// BoolArray::from_iter(vec![true, true, false]).to_bit_buffer(), /// ); /// ``` @@ -326,15 +336,17 @@ pub fn lt_eq(lhs: Expression, rhs: Expression) -> Expression { /// ``` /// # use vortex_array::arrays::{BoolArray, PrimitiveArray }; /// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::{IntoArray, ToCanonical}; +/// # use vortex_array::IntoArray; +/// # use vortex_array::{VortexSessionExecute, array_session}; /// # use vortex_array::validity::Validity; /// # use vortex_buffer::buffer; /// # use vortex_array::expr::{root, lit, lt}; /// let xs = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); /// let result = xs.into_array().apply(<(root(), lit(3))).unwrap(); +/// let mut ctx = array_session().create_execution_ctx(); /// /// assert_eq!( -/// result.to_bool().to_bit_buffer(), +/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), /// BoolArray::from_iter(vec![true, true, false]).to_bit_buffer(), /// ); /// ``` @@ -351,13 +363,15 @@ pub fn lt(lhs: Expression, rhs: Expression) -> Expression { /// ``` /// # use vortex_array::arrays::BoolArray; /// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::{IntoArray, ToCanonical}; +/// # use vortex_array::IntoArray; +/// # use vortex_array::{VortexSessionExecute, array_session}; /// # use vortex_array::expr::{root, lit, or}; /// let xs = BoolArray::from_iter(vec![true, false, true]); /// let result = xs.into_array().apply(&or(root(), lit(false))).unwrap(); +/// let mut ctx = array_session().create_execution_ctx(); /// /// assert_eq!( -/// result.to_bool().to_bit_buffer(), +/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), /// BoolArray::from_iter(vec![true, false, true]).to_bit_buffer(), /// ); /// ``` @@ -387,13 +401,15 @@ where /// ``` /// # use vortex_array::arrays::BoolArray; /// # use vortex_array::arrays::bool::BoolArrayExt; -/// # use vortex_array::{IntoArray, ToCanonical}; +/// # use vortex_array::IntoArray; +/// # use vortex_array::{VortexSessionExecute, array_session}; /// # use vortex_array::expr::{and, root, lit}; /// let xs = BoolArray::from_iter(vec![true, false, true]).into_array(); /// let result = xs.apply(&and(root(), lit(true))).unwrap(); +/// let mut ctx = array_session().create_execution_ctx(); /// /// assert_eq!( -/// result.to_bool().to_bit_buffer(), +/// result.execute::(&mut ctx).unwrap().to_bit_buffer(), /// BoolArray::from_iter(vec![true, false, true]).to_bit_buffer(), /// ); /// ``` diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index a9952a7e99b..5c73aa95d13 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -1210,8 +1210,6 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::assert_arrays_eq; @@ -1269,10 +1267,16 @@ mod test { ) .unwrap() .unwrap(); - #[expect(deprecated)] - let primitive_values = taken.values().to_primitive(); - #[expect(deprecated)] - let primitive_indices = taken.indices().to_primitive(); + let primitive_values = taken + .values() + .clone() + .execute::(&mut ctx) + .unwrap(); + let primitive_indices = taken + .indices() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(taken.array_len(), 2); assert_arrays_eq!( primitive_values, @@ -1316,8 +1320,11 @@ mod test { .unwrap() .unwrap(); - #[expect(deprecated)] - let primitive_values = taken.values().to_primitive(); + let primitive_values = taken + .values() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(taken.array_len(), 2); assert_arrays_eq!( primitive_values, @@ -1629,8 +1636,11 @@ mod test { ); // Values should be the null and 300 - #[expect(deprecated)] - let masked_values = masked.values().to_primitive(); + let masked_values = masked + .values() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(masked_values.len(), 2); assert!(!masked_values.is_valid(0, &mut ctx).unwrap()); // the null value at index 5 assert!(masked_values.is_valid(1, &mut ctx).unwrap()); // the 300 value at index 8 @@ -1825,8 +1835,11 @@ mod test { ) .unwrap(); - #[expect(deprecated)] - let values = patches.values().to_primitive(); + let values = patches + .values() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!( i32::try_from(&values.execute_scalar(0, &mut ctx).unwrap()).unwrap(), 100i32 diff --git a/vortex-array/src/scalar_fn/fns/binary/boolean.rs b/vortex-array/src/scalar_fn/fns/binary/boolean.rs index b7f838c6d13..ebdd8f4ded0 100644 --- a/vortex-array/src/scalar_fn/fns/binary/boolean.rs +++ b/vortex-array/src/scalar_fn/fns/binary/boolean.rs @@ -763,8 +763,6 @@ mod tests { use crate::arrays::ConstantArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -864,9 +862,9 @@ mod tests { BoolArray::from_iter([Some(true), Some(true), Some(false), Some(false)]).into_array(), )] fn test_or(#[case] lhs: ArrayRef, #[case] rhs: ArrayRef) { + let mut ctx = array_session().create_execution_ctx(); let r = lhs.binary(rhs, Operator::Or).unwrap(); - #[expect(deprecated)] - let r = r.to_bool().into_array(); + let r = r.execute::(&mut ctx).unwrap().into_array(); let v0 = r .execute_scalar(0, &mut array_session().create_execution_ctx()) @@ -905,11 +903,12 @@ mod tests { BoolArray::from_iter([Some(true), Some(true), Some(false), Some(false)]).into_array(), )] fn test_and(#[case] lhs: ArrayRef, #[case] rhs: ArrayRef) { - #[expect(deprecated)] + let mut ctx = array_session().create_execution_ctx(); let r = lhs .binary(rhs, Operator::And) .unwrap() - .to_bool() + .execute::(&mut ctx) + .unwrap() .into_array(); let v0 = r diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs index 5185e513e70..edf530cb75f 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs @@ -400,6 +400,7 @@ mod tests { use rstest::rstest; use vortex_buffer::BitBuffer; use vortex_buffer::Buffer; + use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_session::VortexSession; @@ -410,8 +411,6 @@ mod tests { use crate::arrays::ListArray; use crate::arrays::VarBinArray; use crate::assert_arrays_eq; - #[expect(deprecated)] - use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType::I32; @@ -666,15 +665,14 @@ mod tests { // -- Tests migrated from compute/list_contains.rs -- fn nonnull_strings(values: Vec>) -> ArrayRef { - #[expect(deprecated)] - let result = ListArray::from_iter_slow::( - values, - Arc::new(DType::Utf8(Nullability::NonNullable)), - ) - .unwrap() - .to_listview() - .into_array(); - result + let mut ctx = array_session().create_execution_ctx(); + + ListArray::from_iter_slow::(values, Arc::new(DType::Utf8(Nullability::NonNullable))) + .unwrap() + .into_array() + .execute::(&mut ctx) + .vortex_expect("failed to convert to listview") + .into_array() } fn null_strings(values: Vec>>) -> ArrayRef { @@ -693,13 +691,15 @@ mod tests { let elements = VarBinArray::from_iter(elements, DType::Utf8(Nullability::Nullable)).into_array(); - #[expect(deprecated)] - let result = ListArray::try_new(elements, offsets, Validity::NonNullable) + let mut ctx = array_session().create_execution_ctx(); + + ListArray::try_new(elements, offsets, Validity::NonNullable) .unwrap() .as_array() - .to_listview() - .into_array(); - result + .clone() + .execute::(&mut ctx) + .vortex_expect("failed to convert to listview") + .into_array() } fn bool_array(values: Vec, validity: Validity) -> BoolArray { diff --git a/vortex-array/src/scalar_fn/fns/merge.rs b/vortex-array/src/scalar_fn/fns/merge.rs index c9f5783de27..5ef7d2113b6 100644 --- a/vortex-array/src/scalar_fn/fns/merge.rs +++ b/vortex-array/src/scalar_fn/fns/merge.rs @@ -277,8 +277,6 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::PrimitiveArray; @@ -300,21 +298,27 @@ mod tests { use crate::scalar_fn::fns::pack::Pack; fn primitive_field(array: &ArrayRef, field_path: &[&str]) -> VortexResult { + let mut ctx = array_session().create_execution_ctx(); let mut field_path = field_path.iter(); let Some(field) = field_path.next() else { vortex_bail!("empty field path"); }; - #[expect(deprecated)] - let mut array = array.to_struct().unmasked_field_by_name(field)?.clone(); + let mut array = array + .clone() + .execute::(&mut ctx)? + .unmasked_field_by_name(field)? + .clone(); for field in field_path { - #[expect(deprecated)] - let next = array.to_struct().unmasked_field_by_name(field)?.clone(); + let next = array + .clone() + .execute::(&mut ctx)? + .unmasked_field_by_name(field)? + .clone(); array = next; } - #[expect(deprecated)] - let result = array.to_primitive(); + let result = array.execute::(&mut ctx)?; Ok(result) } @@ -491,14 +495,19 @@ mod tests { ]) .unwrap() .into_array(); - #[expect(deprecated)] - let actual_array = test_array.apply(&expr).unwrap().to_struct(); + let mut ctx = array_session().create_execution_ctx(); + let actual_array = test_array + .apply(&expr) + .unwrap() + .execute::(&mut ctx) + .unwrap(); - #[expect(deprecated)] let inner_struct = actual_array .unmasked_field_by_name("a") .unwrap() - .to_struct(); + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!( inner_struct .names() @@ -511,6 +520,7 @@ mod tests { #[test] pub fn test_merge_order() { + let mut ctx = array_session().create_execution_ctx(); let expr = merge(vec![get_item("0", root()), get_item("1", root())]); let test_array = StructArray::from_fields(&[ @@ -535,8 +545,11 @@ mod tests { ]) .unwrap() .into_array(); - #[expect(deprecated)] - let actual_array = test_array.apply(&expr).unwrap().to_struct(); + let actual_array = test_array + .apply(&expr) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!(actual_array.names(), ["a", "c", "b", "d"]); } diff --git a/vortex-array/src/scalar_fn/fns/not/mod.rs b/vortex-array/src/scalar_fn/fns/not/mod.rs index 156a6568df7..d0f9e400bde 100644 --- a/vortex-array/src/scalar_fn/fns/not/mod.rs +++ b/vortex-array/src/scalar_fn/fns/not/mod.rs @@ -110,8 +110,8 @@ impl ScalarFnVTable for Not { #[cfg(test)] mod tests { use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::bool::BoolArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; @@ -124,10 +124,15 @@ mod tests { #[test] fn invert_booleans() { + let mut ctx = array_session().create_execution_ctx(); let not_expr = not(root()); let bools = BoolArray::from_iter([false, true, false, false, true, true]); - #[expect(deprecated)] - let result = bools.into_array().apply(¬_expr).unwrap().to_bool(); + let result = bools + .into_array() + .apply(¬_expr) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!( result.to_bit_buffer().iter().collect::>(), vec![true, false, true, true, false, false] diff --git a/vortex-array/src/scalar_fn/fns/pack.rs b/vortex-array/src/scalar_fn/fns/pack.rs index 81019e22d6f..45475623d32 100644 --- a/vortex-array/src/scalar_fn/fns/pack.rs +++ b/vortex-array/src/scalar_fn/fns/pack.rs @@ -170,8 +170,6 @@ mod tests { use super::PackOptions; use crate::ArrayRef; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::PrimitiveArray; @@ -194,26 +192,33 @@ mod tests { } fn primitive_field(array: &ArrayRef, field_path: &[&str]) -> VortexResult { + let mut ctx = array_session().create_execution_ctx(); let mut field_path = field_path.iter(); let Some(field) = field_path.next() else { vortex_bail!("empty field path"); }; - #[expect(deprecated)] - let mut array = array.to_struct().unmasked_field_by_name(field)?.clone(); + let mut array = array + .clone() + .execute::(&mut ctx)? + .unmasked_field_by_name(field)? + .clone(); for field in field_path { - #[expect(deprecated)] - let next = array.to_struct().unmasked_field_by_name(field)?.clone(); + let next = array + .clone() + .execute::(&mut ctx)? + .unmasked_field_by_name(field)? + .clone(); array = next; } - #[expect(deprecated)] - let result = array.to_primitive(); + let result = array.execute::(&mut ctx)?; Ok(result) } #[test] pub fn test_empty_pack() { + let mut ctx = array_session().create_execution_ctx(); let expr = Pack.new_expr( PackOptions { names: Default::default(), @@ -225,8 +230,11 @@ mod tests { let test_array = test_array(); let actual_array = test_array.clone().apply(&expr).unwrap(); assert_eq!(actual_array.len(), test_array.len()); - #[expect(deprecated)] - let nfields = actual_array.to_struct().struct_fields().nfields(); + let nfields = actual_array + .execute::(&mut ctx) + .unwrap() + .struct_fields() + .nfields(); assert_eq!(nfields, 0); } @@ -241,8 +249,11 @@ mod tests { [col("a"), col("b"), col("a")], ); - #[expect(deprecated)] - let actual_array = test_array().apply(&expr).unwrap().to_struct(); + let actual_array = test_array() + .apply(&expr) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!(actual_array.names(), ["one", "two", "three"]); assert!(matches!(actual_array.validity(), Ok(Validity::NonNullable))); @@ -285,8 +296,11 @@ mod tests { ], ); - #[expect(deprecated)] - let actual_array = test_array().apply(&expr).unwrap().to_struct(); + let actual_array = test_array() + .apply(&expr) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!(actual_array.names(), ["one", "two", "three"]); @@ -314,6 +328,7 @@ mod tests { #[test] pub fn test_pack_nullable() { + let mut ctx = array_session().create_execution_ctx(); let expr = Pack.new_expr( PackOptions { names: ["one", "two", "three"].into(), @@ -322,8 +337,11 @@ mod tests { [col("a"), col("b"), col("a")], ); - #[expect(deprecated)] - let actual_array = test_array().apply(&expr).unwrap().to_struct(); + let actual_array = test_array() + .apply(&expr) + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_eq!(actual_array.names(), ["one", "two", "three"]); assert!(matches!(actual_array.validity(), Ok(Validity::AllValid))); diff --git a/vortex-array/src/scalar_fn/fns/select.rs b/vortex-array/src/scalar_fn/fns/select.rs index 54b63a00f89..2fbe4593503 100644 --- a/vortex-array/src/scalar_fn/fns/select.rs +++ b/vortex-array/src/scalar_fn/fns/select.rs @@ -309,8 +309,8 @@ mod tests { use vortex_buffer::buffer; use crate::IntoArray; - #[expect(deprecated)] - use crate::ToCanonical as _; + use crate::VortexSessionExecute; + use crate::array_session; use crate::arrays::struct_::StructArrayExt; use crate::dtype::DType; use crate::dtype::FieldName; @@ -336,20 +336,30 @@ mod tests { #[test] pub fn include_columns() { + let mut ctx = array_session().create_execution_ctx(); let st = test_array(); let select = select(vec![FieldName::from("a")], root()); - #[expect(deprecated)] - let selected = st.into_array().apply(&select).unwrap().to_struct(); + let selected = st + .into_array() + .apply(&select) + .unwrap() + .execute::(&mut ctx) + .unwrap(); let selected_names = selected.names().clone(); assert_eq!(selected_names.as_ref(), &["a"]); } #[test] pub fn exclude_columns() { + let mut ctx = array_session().create_execution_ctx(); let st = test_array(); let select = select_exclude(vec![FieldName::from("a")], root()); - #[expect(deprecated)] - let selected = st.into_array().apply(&select).unwrap().to_struct(); + let selected = st + .into_array() + .apply(&select) + .unwrap() + .execute::(&mut ctx) + .unwrap(); let selected_names = selected.names().clone(); assert_eq!(selected_names.as_ref(), &["b"]); } diff --git a/vortex-array/src/scalar_fn/fns/variant_get/mod.rs b/vortex-array/src/scalar_fn/fns/variant_get/mod.rs index 1a4854a6a4a..2f7b563202c 100644 --- a/vortex-array/src/scalar_fn/fns/variant_get/mod.rs +++ b/vortex-array/src/scalar_fn/fns/variant_get/mod.rs @@ -138,7 +138,7 @@ impl ScalarFnVTable for VariantGet { builder.append_scalar(&output)?; } - return Ok(builder.finish_into_canonical().into_array()); + return Ok(builder.finish()); } // TODO(variant): replace this with a Variant builder once one exists. diff --git a/vortex-btrblocks/src/schemes/binary/zstd.rs b/vortex-btrblocks/src/schemes/binary/zstd.rs index 56be76cdcb8..b63049f36e7 100644 --- a/vortex-btrblocks/src/schemes/binary/zstd.rs +++ b/vortex-btrblocks/src/schemes/binary/zstd.rs @@ -45,7 +45,10 @@ impl Scheme for ZstdScheme { _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let compacted = data.array_as_varbinview().into_owned().compact_buffers()?; + let compacted = data + .array_as_varbinview() + .into_owned() + .compact_buffers(exec_ctx)?; Ok( vortex_zstd::Zstd::from_var_bin_view_without_dict(&compacted, 3, 8192, exec_ctx)? .into_array(), diff --git a/vortex-btrblocks/src/schemes/string/zstd.rs b/vortex-btrblocks/src/schemes/string/zstd.rs index 6e99cc27aa3..c799b64e359 100644 --- a/vortex-btrblocks/src/schemes/string/zstd.rs +++ b/vortex-btrblocks/src/schemes/string/zstd.rs @@ -45,7 +45,10 @@ impl Scheme for ZstdScheme { _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let compacted = data.array_as_varbinview().into_owned().compact_buffers()?; + let compacted = data + .array_as_varbinview() + .into_owned() + .compact_buffers(exec_ctx)?; Ok( vortex_zstd::Zstd::from_var_bin_view_without_dict(&compacted, 3, 8192, exec_ctx)? .into_array(), diff --git a/vortex-cxx/src/read.rs b/vortex-cxx/src/read.rs index 4ce229b7a2b..1fbbef6f44b 100644 --- a/vortex-cxx/src/read.rs +++ b/vortex-cxx/src/read.rs @@ -15,7 +15,6 @@ use arrow_schema::Schema; use arrow_schema::SchemaRef; use futures::stream::TryStreamExt; use vortex::array::ArrayRef; -use vortex::array::LEGACY_SESSION; use vortex::array::VortexSessionExecute; use vortex::array::arrow::ArrowSessionExt; use vortex::buffer::Buffer; @@ -167,7 +166,7 @@ pub(crate) fn scan_builder_into_threadsafe_cloneable_reader( .map(move |b| { SESSION .arrow() - .execute_arrow(b, Some(&target), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_arrow(b, Some(&target), &mut SESSION.create_execution_ctx()) .map(|struct_array| RecordBatch::from(struct_array.as_struct())) }) .into_stream()? diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index b196f68dc96..b128aa9f93b 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -237,12 +237,14 @@ async fn test_read_simple_with_spawn() { vec![vec![11, 12], vec![21, 22], vec![31, 32], vec![41, 42]], Arc::new(I32.into()), ) - .unwrap(), + .unwrap() + .into_array(), ListArray::from_iter_slow::( vec![vec![51, 52], vec![61, 62], vec![71, 72], vec![81, 82]], Arc::new(I32.into()), ) - .unwrap(), + .unwrap() + .into_array(), ]) .into_array(); diff --git a/vortex-layout/src/layouts/file_stats.rs b/vortex-layout/src/layouts/file_stats.rs index 983f9e59a58..c4cd032c686 100644 --- a/vortex-layout/src/layouts/file_stats.rs +++ b/vortex-layout/src/layouts/file_stats.rs @@ -11,7 +11,6 @@ use parking_lot::Mutex; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::sum::sum; use vortex_array::arrays::ConstantArray; @@ -115,7 +114,7 @@ impl StatsAccumulator { Ok(()) } - fn as_array(&mut self) -> VortexResult> { + fn as_array(&mut self, ctx: &mut ExecutionCtx) -> VortexResult> { let mut names = Vec::new(); let mut fields = Vec::new(); @@ -128,7 +127,7 @@ impl StatsAccumulator { let values = builder.finish(); // We drop any all-null stats columns. - if values.all_invalid()? { + if values.all_invalid(ctx)? { continue; } @@ -146,7 +145,7 @@ impl StatsAccumulator { /// Returns an aggregated stats set for the table. fn as_stats_set(&mut self, stats: &[Stat], ctx: &mut ExecutionCtx) -> VortexResult { let mut stats_set = StatsSet::default(); - let Some(array) = self.as_array()? else { + let Some(array) = self.as_array(ctx)? else { return Ok(stats_set); }; @@ -236,8 +235,8 @@ struct NamedArrays { } impl NamedArrays { - fn all_invalid(&self) -> VortexResult { - self.arrays[0].all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) + fn all_invalid(&self, ctx: &mut ExecutionCtx) -> VortexResult { + self.arrays[0].all_invalid(ctx) } } @@ -524,7 +523,10 @@ mod tests { .vortex_expect("push_chunk should succeed for test data"); acc.push_chunk(&builder2.finish(), &mut ctx) .vortex_expect("push_chunk should succeed for test data"); - let stats_table = acc.as_array().unwrap().expect("Must have stats table"); + let stats_table = acc + .as_array(&mut ctx) + .unwrap() + .expect("Must have stats table"); assert_eq!( stats_table.names().as_ref(), &[ @@ -561,7 +563,10 @@ mod tests { let mut acc = StatsAccumulator::new(array.dtype(), &[Stat::Max, Stat::Min, Stat::Sum], 12); acc.push_chunk(&array, &mut ctx) .vortex_expect("push_chunk should succeed for test array"); - let stats_table = acc.as_array().unwrap().expect("Must have stats table"); + let stats_table = acc + .as_array(&mut ctx) + .unwrap() + .expect("Must have stats table"); assert_eq!( stats_table.names().as_ref(), &[ diff --git a/vortex-layout/src/layouts/zoned/builder.rs b/vortex-layout/src/layouts/zoned/builder.rs index bbe359f7485..3c86f371fb6 100644 --- a/vortex-layout/src/layouts/zoned/builder.rs +++ b/vortex-layout/src/layouts/zoned/builder.rs @@ -8,8 +8,6 @@ use std::sync::Arc; use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::AggregateFnRef; use vortex_array::arrays::StructArray; use vortex_array::builders::ArrayBuilder; @@ -74,6 +72,7 @@ impl AggregateStatsAccumulator { pub(crate) fn as_array( &mut self, + ctx: &mut ExecutionCtx, ) -> VortexResult)>> { let mut names = Vec::new(); let mut fields = Vec::new(); @@ -86,7 +85,7 @@ impl AggregateStatsAccumulator { { let values = builder.finish(); - if values.all_invalid()? { + if values.all_invalid(ctx)? { continue; } @@ -153,8 +152,8 @@ struct NamedArrays { } impl NamedArrays { - fn all_invalid(&self) -> VortexResult { + fn all_invalid(&self, ctx: &mut ExecutionCtx) -> VortexResult { // By convention the first array is the logical validity signal for the stat column. - self.arrays[0].all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) + self.arrays[0].all_invalid(ctx) } } diff --git a/vortex-layout/src/layouts/zoned/writer.rs b/vortex-layout/src/layouts/zoned/writer.rs index 6ed7b541bcb..c1c12d7ddea 100644 --- a/vortex-layout/src/layouts/zoned/writer.rs +++ b/vortex-layout/src/layouts/zoned/writer.rs @@ -164,7 +164,10 @@ impl LayoutStrategy for ZonedStrategy { ) .await?; - let Some((stats_array, aggregate_fns)) = stats_accumulator.lock().as_array()? else { + let mut exec_ctx = session.create_execution_ctx(); + let Some((stats_array, aggregate_fns)) = + stats_accumulator.lock().as_array(&mut exec_ctx)? + else { // If we have no stats (e.g. the DType doesn't support them), then we just return the // child layout. return Ok(data_layout); diff --git a/vortex-python/src/iter/mod.rs b/vortex-python/src/iter/mod.rs index 89caab3e90f..fdbf189365f 100644 --- a/vortex-python/src/iter/mod.rs +++ b/vortex-python/src/iter/mod.rs @@ -20,7 +20,6 @@ use pyo3::prelude::*; use pyo3::types::PyIterator; use vortex::array::Canonical; use vortex::array::IntoArray; -use vortex::array::LEGACY_SESSION; use vortex::array::VortexSessionExecute; use vortex::array::arrow::ArrowSessionExt; use vortex::array::iter::ArrayIterator; @@ -132,7 +131,7 @@ impl PyArrayIterator { session().arrow().execute_arrow( chunk?, Some(&target), - &mut LEGACY_SESSION.create_execution_ctx(), + &mut session().create_execution_ctx(), ) }) .map(|chunk| chunk.map_err(|e| ArrowError::ExternalError(Box::new(e)))) diff --git a/vortex-web/crate/src/wasm.rs b/vortex-web/crate/src/wasm.rs index f1bb4df791e..c7318312134 100644 --- a/vortex-web/crate/src/wasm.rs +++ b/vortex-web/crate/src/wasm.rs @@ -322,7 +322,7 @@ impl VortexFileHandle { .map_err(|e| JsValue::from_str(&e.to_string()))?; for chunk in chunks { - let batch = array_to_record_batch(chunk, &dtype, &arrow_schema) + let batch = array_to_record_batch(chunk, &dtype, &arrow_schema, &self.session) .map_err(|e| JsValue::from_str(&e.to_string()))?; writer .write(&batch) From d7c69bdcf2d7b2aa8a11a91ab9a11159785473d7 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sat, 27 Jun 2026 18:56:30 +0100 Subject: [PATCH 2/3] fix Signed-off-by: Robert Kruszewski --- vortex-web/crate/src/wasm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vortex-web/crate/src/wasm.rs b/vortex-web/crate/src/wasm.rs index c7318312134..f1bb4df791e 100644 --- a/vortex-web/crate/src/wasm.rs +++ b/vortex-web/crate/src/wasm.rs @@ -322,7 +322,7 @@ impl VortexFileHandle { .map_err(|e| JsValue::from_str(&e.to_string()))?; for chunk in chunks { - let batch = array_to_record_batch(chunk, &dtype, &arrow_schema, &self.session) + let batch = array_to_record_batch(chunk, &dtype, &arrow_schema) .map_err(|e| JsValue::from_str(&e.to_string()))?; writer .write(&batch) From 947a966b926370d7131a3e7e3cf8fc28725c4b7b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 27 Jun 2026 20:12:28 +0000 Subject: [PATCH 3/3] Handle any list builder kind in append_to_builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `builder_with_capacity` produces a `ListViewBuilder` for `DType::List`, but `List::append_to_builder` downcast the builder to `ListBuilder` and bailed otherwise — so executing a `List`-encoded array into the canonical builder (the one `builder_with_capacity` returns) failed. `ListView::append_to_builder` had the mirror-image assumption, only accepting a `ListViewBuilder`. Add two macros that enumerate the possible list builder offset/size integer types and try downcasting to each, rather than assuming the types produced by `builder_with_capacity`: - `match_each_list_builder!` enumerates `ListBuilder`. - `match_each_listview_builder!` enumerates `ListViewBuilder`. `List::append_to_builder` tries `ListBuilder` (its non-canonical, directly-supplied builder) and otherwise canonicalizes and dispatches, routing the canonical `ListViewBuilder` case through `ListView::append_to_builder`, which tries `ListViewBuilder`. Either encoding works with any unsigned offset/size types. Other encodings that yield list dtypes (constant, dict, chunked, ...) already route here through the canonical-dispatch fallback. Signed-off-by: Robert Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Mkrgj5SuJpaKBXK9jydtSc --- vortex-array/src/arrays/list/vtable/mod.rs | 15 +++-- .../src/arrays/listview/vtable/mod.rs | 17 +++--- vortex-array/src/builders/list.rs | 52 ++++++++++++++++- vortex-array/src/builders/mod.rs | 57 +++++++++++++++++++ 4 files changed, 125 insertions(+), 16 deletions(-) diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index faa91c459bc..5b2b63f7a66 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -16,6 +16,7 @@ use vortex_session::registry::CachedId; use crate::ArrayEq; use crate::ArrayHash; use crate::ArrayRef; +use crate::Canonical; use crate::EqMode; use crate::ExecutionCtx; use crate::ExecutionResult; @@ -35,7 +36,7 @@ use crate::arrays::list::compute::rules::PARENT_RULES; use crate::arrays::listview::list_view_from_list; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; -use crate::builders::ListBuilder; +use crate::builders::match_each_list_builder; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; @@ -198,10 +199,14 @@ impl VTable for List { builder: &mut dyn ArrayBuilder, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - let Some(builder) = builder.as_any_mut().downcast_mut::>() else { - vortex_bail!("append_to_builder for List requires a ListBuilder"); - }; - builder.append_list_array(&array.into_owned().into_array(), ctx) + let array = array.into_owned().into_array(); + // A directly-supplied `ListBuilder` (not the canonical builder for `DType::List`). + match_each_list_builder!(&array, builder, ctx); + // The canonical builder is `ListViewBuilder`; canonicalize and dispatch to `ListView`. + array + .execute::(ctx)? + .into_array() + .append_to_builder(builder, ctx) } } diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index 95944bb99fe..31f4cc8f0fa 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -20,6 +20,7 @@ use crate::ArrayRef; use crate::EqMode; use crate::ExecutionCtx; use crate::ExecutionResult; +use crate::IntoArray; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -34,7 +35,7 @@ use crate::arrays::listview::array::SLOT_NAMES; use crate::arrays::listview::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; -use crate::builders::ListViewBuilder; +use crate::builders::match_each_listview_builder; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; @@ -222,14 +223,12 @@ impl VTable for ListView { builder: &mut dyn ArrayBuilder, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - // `builder_with_capacity` always produces a `ListViewBuilder` for `DType::List`. - let Some(builder) = builder - .as_any_mut() - .downcast_mut::>() - else { - vortex_bail!("append_to_builder for ListView requires a ListViewBuilder"); - }; - builder.append_listview_array(&array.into_owned(), ctx) + let array = array.into_owned().into_array(); + match_each_listview_builder!(&array, builder, ctx); + vortex_bail!( + "append_to_builder for ListView requires a ListViewBuilder, got a builder for {}", + builder.dtype() + ) } fn reduce_parent( diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index 71098796d00..02c67e05b7e 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -173,8 +173,9 @@ impl ListBuilder { /// [`ListViewArray`] and converting into the `ListArray` (`n + 1` offsets) layout. /// /// [`ListBuilder`] is not the canonical builder for [`DType::List`] (that is - /// [`ListViewBuilder`](crate::builders::ListViewBuilder)), so no encoding dispatches into it via - /// `append_to_builder`; this helper exists for direct callers such as tests. + /// [`ListViewBuilder`](crate::builders::ListViewBuilder)), so it is never produced by + /// [`builder_with_capacity`]. List encodings still dispatch into it via `match_each_list_builder` + /// when a caller supplies one directly. pub(crate) fn append_list_array( &mut self, array: &ArrayRef, @@ -334,6 +335,7 @@ mod tests { use Nullability::Nullable; use vortex_buffer::buffer; use vortex_error::VortexExpect; + use vortex_error::VortexResult; use crate::IntoArray; use crate::array_session; @@ -344,6 +346,8 @@ mod tests { use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::builders::ArrayBuilder; + use crate::builders::ListViewBuilder; + use crate::builders::builder_with_capacity; use crate::builders::list::ListArray; use crate::builders::list::ListBuilder; use crate::dtype::DType; @@ -514,6 +518,50 @@ mod tests { ); } + /// `append_to_builder` must handle any list builder kind without assuming the offset/size + /// integer types produced by `builder_with_capacity`. It appends a `List`-encoded array and a + /// `ListView`-encoded array into `ListViewBuilder`s and `ListBuilder`s with assorted (and + /// non-`u64`) offset/size types. + #[test] + fn test_append_to_builder_any_list_builder() -> VortexResult<()> { + let mut ctx = array_session().create_execution_ctx(); + + let list = ListArray::from_iter_opt_slow::( + [Some(vec![0, 1, 2]), None, Some(vec![4, 5])], + Arc::new(I32.into()), + )? + .into_array(); + let listview = list + .clone() + .execute::(&mut ctx)? + .into_array(); + let elem_dtype = || Arc::new(I32.into()); + + // `builder_with_capacity` produces a `ListViewBuilder` for `DType::List`; appending the + // `List`-encoded array must dispatch into it instead of bailing. + let mut listview_builder = builder_with_capacity(list.dtype(), list.len()); + list.append_to_builder(listview_builder.as_mut(), &mut ctx)?; + assert_arrays_eq!(listview_builder.finish(), list, &mut ctx); + + // Offset/size types are enumerated, not assumed: a `ListViewBuilder` with non-`u64` + // offset and size types must work for both source encodings. + let mut lv_u32_u8 = ListViewBuilder::::with_capacity(elem_dtype(), Nullable, 8, 4); + list.append_to_builder(&mut lv_u32_u8, &mut ctx)?; + assert_arrays_eq!(lv_u32_u8.finish(), list, &mut ctx); + + let mut lv_u16_u16 = + ListViewBuilder::::with_capacity(elem_dtype(), Nullable, 8, 4); + listview.append_to_builder(&mut lv_u16_u16, &mut ctx)?; + assert_arrays_eq!(lv_u16_u16.finish(), list, &mut ctx); + + // A `List`-encoded array appended into a `ListBuilder` with a non-`u64` offset type. + let mut list_builder = ListBuilder::::with_capacity(elem_dtype(), Nullable, 8, 4); + list.append_to_builder(&mut list_builder, &mut ctx)?; + assert_arrays_eq!(list_builder.finish(), list, &mut ctx); + + Ok(()) + } + #[test] fn test_extend_builder() { test_extend_builder_gen::(); diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs index e94cdd8c6dd..da91d74a5ed 100644 --- a/vortex-array/src/builders/mod.rs +++ b/vortex-array/src/builders/mod.rs @@ -284,3 +284,60 @@ pub fn builder_with_capacity_in( let _allocator = allocator; builder_with_capacity(dtype, capacity) } + +/// Append a list-typed `$array` into `$builder` if it is a [`ListBuilder`], enumerating the possible +/// offset integer types and trying each downcast. On a match it `return`s the append result from the +/// enclosing function; otherwise it falls through. +/// +/// [`ListBuilder`] is never produced by [`builder_with_capacity`] (the canonical list builder is +/// [`ListViewBuilder`]), so its offset type cannot be assumed — hence the enumeration. List offsets +/// are always unsigned integers. +macro_rules! match_each_list_builder { + ($array:expr, $builder:expr, $ctx:expr) => {{ + $crate::builders::match_each_list_builder!(@types $array, $builder, $ctx, [u8, u16, u32, u64]); + }}; + (@types $array:expr, $builder:expr, $ctx:expr, [$($O:ty),*]) => {{ + $( + if let Some(builder) = $builder + .as_any_mut() + .downcast_mut::<$crate::builders::ListBuilder<$O>>() + { + return builder.append_list_array($array, $ctx); + } + )* + }}; +} +pub(crate) use match_each_list_builder; + +/// Append a list-typed `$array` into `$builder` if it is a [`ListViewBuilder`], enumerating the +/// possible offset and size integer types and trying each downcast. On a match it `return`s the +/// append result from the enclosing function; otherwise it falls through. +/// +/// [`builder_with_capacity`] produces a `ListViewBuilder` for [`DType::List`], but a +/// caller may supply different offset/size types, so the concrete type cannot be assumed — hence the +/// enumeration. List offsets and sizes are always unsigned integers. +macro_rules! match_each_listview_builder { + ($array:expr, $builder:expr, $ctx:expr) => {{ + $crate::builders::match_each_listview_builder!( + @offsets $array, $builder, $ctx, [u8, u16, u32, u64], [u8, u16, u32, u64] + ); + }}; + // For each offset type `O`, expand over every size type `S` (cartesian product). + (@offsets $array:expr, $builder:expr, $ctx:expr, [$($O:ty),*], $sizes:tt) => {{ + $( + $crate::builders::match_each_listview_builder!(@sizes $array, $builder, $ctx, $O, $sizes); + )* + }}; + (@sizes $array:expr, $builder:expr, $ctx:expr, $O:ty, [$($S:ty),*]) => {{ + $( + if let Some(builder) = $builder + .as_any_mut() + .downcast_mut::<$crate::builders::ListViewBuilder<$O, $S>>() + { + let listview = $array.clone().execute::<$crate::arrays::ListViewArray>($ctx)?; + return builder.append_listview_array(&listview, $ctx); + } + )* + }}; +} +pub(crate) use match_each_listview_builder;