Skip to content

Expose array buffer replacement #8606

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gatesn wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Expose array buffer replacement #8606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions encodings/alp/src/alp/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,14 @@ impl VTable for ALP {
None
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)

@robert3005 robert3005 Jun 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

don't quite get this? It's just so you can add the method and assert that it panics?

@robert3005 robert3005 Jun 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok no there are some implementations, I thought everything is this

}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
8 changes: 8 additions & 0 deletions encodings/alp/src/alp_rd/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,14 @@ impl VTable for ALPRD {
None
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
17 changes: 17 additions & 0 deletions encodings/bytebool/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,23 @@ impl VTable for ByteBool {
}
}

fn with_buffers(

@robert3005 robert3005 Jun 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't this be truly with_buffers just like with slots? Try to mutate in place and only make a copy if it fails? Then we cannot take view but typed Array

&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_ensure!(
buffers.len() == 1,
"Expected 1 buffer, got {}",
buffers.len()
);
let data = ByteBoolData::new(buffers[0].clone());
Ok(
ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data)
.with_slots(array.slots().iter().cloned().collect()),
)
}

fn serialize(
_array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
8 changes: 8 additions & 0 deletions encodings/datetime-parts/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,14 @@ impl VTable for DateTimeParts {
vortex_panic!("DateTimePartsArray buffer_name index {idx} out of bounds")
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
8 changes: 8 additions & 0 deletions encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,14 @@ impl VTable for DecimalByteParts {
vortex_panic!("DecimalBytePartsArray buffer_name index {idx} out of bounds")
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
18 changes: 18 additions & 0 deletions encodings/experimental/onpair/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,24 @@ impl VTable for OnPair {
}
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_ensure!(
buffers.len() == 1,
"Expected 1 buffer, got {}",
buffers.len()
);
let mut data = array.data().clone();
data.dict_bytes = buffers[0].clone();
Ok(
ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data)
.with_slots(array.slots().iter().cloned().collect()),
)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
19 changes: 19 additions & 0 deletions encodings/fastlanes/src/bitpacking/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ use vortex_array::vtable::validity_to_child;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_error::vortex_ensure;
use vortex_error::vortex_err;
use vortex_error::vortex_panic;
use vortex_session::VortexSession;
Expand Down Expand Up @@ -141,6 +142,24 @@ impl VTable for BitPacked {
}
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_ensure!(
buffers.len() == 1,
"Expected 1 buffer, got {}",
buffers.len()
);
let mut data = array.data().clone();
data.packed = buffers[0].clone();
Ok(
ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data)
.with_slots(array.slots().iter().cloned().collect()),
)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
8 changes: 8 additions & 0 deletions encodings/fastlanes/src/delta/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,14 @@ impl VTable for Delta {
None
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn reduce_parent(
array: ArrayView<'_, Self>,
parent: &ArrayRef,
Expand Down
8 changes: 8 additions & 0 deletions encodings/fastlanes/src/for/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,14 @@ impl VTable for FoR {
None
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
SLOT_NAMES[idx].to_string()
}
Expand Down
8 changes: 8 additions & 0 deletions encodings/fastlanes/src/rle/vtable/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,14 @@ impl VTable for RLE {
None
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn reduce_parent(
array: ArrayView<'_, Self>,
parent: &ArrayRef,
Expand Down
19 changes: 19 additions & 0 deletions encodings/fsst/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,25 @@ impl VTable for FSST {
}
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_ensure!(
buffers.len() == 3,
"Expected 3 buffers, got {}",
buffers.len()
);
let symbols = Buffer::<Symbol>::from_byte_buffer(buffers[0].clone().try_to_host_sync()?);
let symbol_lengths = Buffer::<u8>::from_byte_buffer(buffers[1].clone().try_to_host_sync()?);
let data = FSSTData::try_new(symbols, symbol_lengths, buffers[2].clone(), array.len())?;
Ok(
ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data)
.with_slots(array.slots().iter().cloned().collect()),
)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
8 changes: 8 additions & 0 deletions encodings/parquet-variant/src/vtable.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,14 @@ impl VTable for ParquetVariant {
None
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
SLOT_NAMES[idx].to_string()
}
Expand Down
22 changes: 22 additions & 0 deletions encodings/pco/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,28 @@ impl VTable for Pco {
}
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
let mut data = array.data().clone();
let chunk_metas_len = data.metadata.chunks.len();
vortex_ensure!(buffers.len() >= chunk_metas_len);
data.chunk_metas = buffers[..chunk_metas_len]
.iter()
.map(|buffer| buffer.clone().try_to_host_sync())
.collect::<VortexResult<Vec<_>>>()?;
data.pages = buffers[chunk_metas_len..]
.iter()
.map(|buffer| buffer.clone().try_to_host_sync())
.collect::<VortexResult<Vec<_>>>()?;
Ok(
ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data)
.with_slots(array.slots().iter().cloned().collect()),
)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
8 changes: 8 additions & 0 deletions encodings/runend/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,14 @@ impl VTable for RunEnd {
vortex_panic!("RunEndArray buffer_name index {idx} out of bounds")
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
8 changes: 8 additions & 0 deletions encodings/sequence/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,14 @@ impl VTable for Sequence {
vortex_panic!("SequenceArray buffer_name index {idx} out of bounds")
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
8 changes: 8 additions & 0 deletions encodings/sparse/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,14 @@ impl VTable for Sparse {
}
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::unsupported_buffer_replacement(array, buffers)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
8 changes: 8 additions & 0 deletions encodings/zigzag/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,14 @@ impl VTable for ZigZag {
vortex_panic!("ZigZagArray buffer_name index {idx} out of bounds")
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
vortex_array::vtable::with_empty_buffers(self, array, buffers)
}

fn serialize(
_array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
27 changes: 27 additions & 0 deletions encodings/zstd/src/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,33 @@ impl VTable for Zstd {
}
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
let mut data = array.data().clone();
if data.dictionary.is_some() {
let Some((dictionary, frames)) = buffers.split_first() else {
vortex_bail!("Expected dictionary buffer");
};
data.dictionary = Some(dictionary.clone().try_to_host_sync()?);
data.frames = frames
.iter()
.map(|buffer| buffer.clone().try_to_host_sync())
.collect::<VortexResult<Vec<_>>>()?;
} else {
data.frames = buffers
.iter()
.map(|buffer| buffer.clone().try_to_host_sync())
.collect::<VortexResult<Vec<_>>>()?;
}
Ok(
ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data)
.with_slots(array.slots().iter().cloned().collect()),
)
}

fn serialize(
array: ArrayView<'_, Self>,
_session: &VortexSession,
Expand Down
13 changes: 13 additions & 0 deletions encodings/zstd/src/zstd_buffers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,19 @@ impl VTable for ZstdBuffers {
Some(format!("compressed_{idx}"))
}

fn with_buffers(
&self,
array: ArrayView<'_, Self>,
buffers: &[BufferHandle],
) -> VortexResult<ArrayParts<Self>> {
let mut data = array.data().clone();
data.compressed_buffers = buffers.to_vec();
Ok(
ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data)
.with_slots(array.slots().iter().cloned().collect()),
)
}

fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
format!("child_{idx}")
}
Expand Down
Loading
Loading