Skip to content

Chore: Remove nullability from Vectors #5029

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gatesn merged 2 commits into from
Oct 21, 2025
Merged

Chore: Remove nullability from Vectors #5029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6a41e8e
remove nullability from vectors
connortsui20 Oct 21, 2025
5e713bc
update docs for vectors always being null
connortsui20 Oct 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 16 additions & 29 deletions vortex-vector/src/bool/vector.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,42 +4,33 @@
//! Definition and implementation of [`BoolVector`].

use vortex_buffer::BitBuffer;
use vortex_dtype::Nullability;
use vortex_mask::Mask;

use super::BoolVectorMut;
use crate::VectorOps;

/// An immutable vector of boolean values.
///
/// Internally, the boolean values are stored as the bits of a [`BitBuffer`] plus an optional
/// [`Mask`] for null booleans (where `true` represents a _valid_ boolean and `false` represents a
/// `null` boolean).
///
/// The mutable equivalent of this type is [`BoolVectorMut`].
#[derive(Debug, Clone)]
pub struct BoolVector {
pub(super) bits: BitBuffer,
pub(super) validity: Option<Mask>,
pub(super) validity: Mask,
}

impl VectorOps for BoolVector {
type Mutable = BoolVectorMut;

fn nullability(&self) -> Nullability {
Nullability::from(self.validity.is_some())
}

fn len(&self) -> usize {
debug_assert!(
self.validity
.as_ref()
.is_none_or(|mask| mask.len() == self.bits.len())
);
debug_assert!(self.validity.len() == self.bits.len());

self.bits.len()
}

fn validity(&self) -> &Mask {
&self.validity
}

fn try_into_mut(self) -> Result<Self::Mutable, Self>
where
Self: Sized,
Expand All @@ -54,19 +45,15 @@ impl VectorOps for BoolVector {
}
};

let validity = match self.validity {
Some(v) => match v.try_into_mut() {
Ok(v) => Some(v),
Err(v) => {
return Err(BoolVector {
bits: bits.freeze(),
validity: Some(v),
});
}
},
None => None,
};

Ok(BoolVectorMut { bits, validity })
match self.validity.try_into_mut() {
Ok(validity_mut) => Ok(BoolVectorMut {
bits,
validity: validity_mut,
}),
Err(validity) => Err(BoolVector {
bits: bits.freeze(),
validity,
}),
}
}
}
79 changes: 11 additions & 68 deletions vortex-vector/src/bool/vector_mut.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,53 +4,35 @@
//! Definition and implementation of [`BoolVectorMut`].

use vortex_buffer::BitBufferMut;
use vortex_dtype::Nullability;
use vortex_error::vortex_panic;
use vortex_mask::MaskMut;

use super::BoolVector;
use crate::{VectorMutOps, VectorOps};

/// A mutable vector of boolean values.
///
/// Internally, the boolean values are stored as the bits of a [`BitBufferMut`] plus an optional
/// [`MaskMut`] for null booleans (where `true` represents a _valid_ boolean and `false` represents
/// a `null` boolean).
///
/// The immutable equivalent of this type is [`BoolVector`].
#[derive(Debug, Clone)]
pub struct BoolVectorMut {
pub(super) bits: BitBufferMut,
pub(super) validity: Option<MaskMut>,
pub(super) validity: MaskMut,
}

impl BoolVectorMut {
/// Creates a new mutable boolean vector with the given `capacity` and `nullability`.
pub fn with_capacity(capacity: usize, nullability: Nullability) -> Self {
let validity = nullability
.is_nullable()
.then(|| MaskMut::with_capacity(capacity));

/// Creates a new mutable boolean vector with the given `capacity`.
pub fn with_capacity(capacity: usize) -> Self {
Self {
bits: BitBufferMut::with_capacity(capacity),
validity,
validity: MaskMut::with_capacity(capacity),
}
}
}

impl VectorMutOps for BoolVectorMut {
type Immutable = BoolVector;

fn nullability(&self) -> Nullability {
Nullability::from(self.validity.is_some())
}

fn len(&self) -> usize {
debug_assert!(
self.validity
.as_ref()
.is_none_or(|mask| mask.len() == self.bits.len())
);
debug_assert!(self.validity.len() == self.bits.len());

self.bits.len()
}
Expand All @@ -61,74 +43,35 @@ impl VectorMutOps for BoolVectorMut {

fn reserve(&mut self, additional: usize) {
self.bits.reserve(additional);

if let Some(v) = self.validity.as_mut() {
v.reserve(additional);
}
self.validity.reserve(additional);
}

fn extend_from_vector(&mut self, other: &BoolVector) {
assert_eq!(
self.nullability(),
other.nullability(),
"tried to extend a vector with nullability {} with another vector with nullability {}",
self.nullability(),
other.nullability(),
);

self.bits.append_buffer(&other.bits);

match (&mut self.validity, &other.validity) {
(Some(self_v), Some(other_v)) => self_v.append_mask(other_v),
(Some(self_v), None) => self_v.append_n(true, other.bits.len()),
(None, Some(other_v)) => {
let mut new_validity = MaskMut::new_true(self.bits.len() - other.bits.len());
new_validity.append_mask(other_v);
self.validity = Some(new_validity);
}
(None, None) => {}
}
self.validity.append_mask(other.validity());
}

fn append_nulls(&mut self, n: usize) {
let Some(mask) = &mut self.validity else {
vortex_panic!("tried to append nulls to a non-nullable vector")
};

mask.append_n(false, n);

self.bits.append_n(false, n);
self.validity.append_n(false, n);
}

fn freeze(self) -> Self::Immutable {
BoolVector {
bits: self.bits.freeze(),
validity: self.validity.map(|v| v.freeze()),
validity: self.validity.freeze(),
}
}

fn split_off(&mut self, at: usize) -> Self {
BoolVectorMut {
bits: self.bits.split_off(at),
validity: self.validity.as_mut().map(|v| v.split_off(at)),
validity: self.validity.split_off(at),
}
}

fn unsplit(&mut self, other: Self) {
// TODO(connor): We must check `other`'s nullability in relation to `self`.

let other_len = other.bits.len();
self.bits.unsplit(other.bits);

match (&mut self.validity, other.validity) {
(Some(self_v), Some(other_v)) => self_v.unsplit(other_v),
(Some(self_v), None) => self_v.append_n(true, other_len),
(None, Some(other_v)) => {
let mut new_validity = MaskMut::new_true(self.bits.len() - other_len);
new_validity.unsplit(other_v);
self.validity = Some(new_validity);
}
(None, None) => {}
}
self.validity.unsplit(other.validity);
}
}
18 changes: 11 additions & 7 deletions vortex-vector/src/null/vector.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

//! Definition and implementation of [`NullVector`].

use vortex_dtype::Nullability;
use vortex_mask::Mask;

use crate::{NullVectorMut, VectorOps};

Expand All @@ -13,29 +13,33 @@ use crate::{NullVectorMut, VectorOps};
/// single `length` counter.
///
/// The mutable equivalent of this type is [`NullVectorMut`].
#[derive(Debug, Clone, Copy)]
#[derive(Debug, Clone)]
pub struct NullVector {
pub(super) len: usize,
pub(super) validity: Mask,
}

impl NullVector {
/// Creates a new immutable vector of nulls with the given length.
pub fn new(len: usize) -> Self {
Self { len }
Self {
len,
validity: Mask::AllFalse(len),
}
}
}

impl VectorOps for NullVector {
type Mutable = NullVectorMut;

fn nullability(&self) -> Nullability {
Nullability::Nullable
}

fn len(&self) -> usize {
self.len
}

fn validity(&self) -> &Mask {
&self.validity
}

fn try_into_mut(self) -> Result<Self::Mutable, Self>
where
Self: Sized,
Expand Down
6 changes: 0 additions & 6 deletions vortex-vector/src/null/vector_mut.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@

//! Definition and implementation of [`NullVectorMut`].

use vortex_dtype::Nullability;

use super::NullVector;
use crate::VectorMutOps;

Expand All @@ -29,10 +27,6 @@ impl NullVectorMut {
impl VectorMutOps for NullVectorMut {
type Immutable = NullVector;

fn nullability(&self) -> Nullability {
Nullability::Nullable
}

fn len(&self) -> usize {
self.len
}
Expand Down
43 changes: 11 additions & 32 deletions vortex-vector/src/ops.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
//! Definition and implementation of [`VectorOps`] and [`VectorMutOps`] for [`Vector`] and
//! [`VectorMut`], respectively.

use vortex_dtype::Nullability;
use vortex_mask::Mask;

use crate::{Vector, VectorMut, private};

Expand All @@ -13,16 +13,6 @@ pub trait VectorOps: private::Sealed + Into<Vector> {
/// The mutable equivalent of this immutable vector.
type Mutable: VectorMutOps<Immutable = Self>;

/// Returns the [`Nullability`] of the vector.
fn nullability(&self) -> Nullability;

/// Returns `true` if the nullability of this vector is [`Nullable`].
///
/// [`Nullable`]: Nullability::Nullable
fn is_nullable(&self) -> bool {
self.nullability().is_nullable()
}

/// Returns the number of elements in the vector, also referred to as its "length".
fn len(&self) -> usize;

Expand All @@ -31,6 +21,14 @@ pub trait VectorOps: private::Sealed + Into<Vector> {
self.len() == 0
}

/// Returns the validity mask of the vector, where `true` represents a _valid_ element and
/// `false` represents a `null` element.
///
/// Note that vectors are **always** considered nullable. "Non-nullable" data will simply have a
/// [`Mask`] of [`AllTrue(len)`](Mask::AllTrue). It is on the caller to ensure that they do not
/// add nullable data to a vector they want to keep as non-nullable.
fn validity(&self) -> &Mask;

/// Tries to convert `self` into a mutable vector (implementing [`VectorMutOps`]).
///
/// This method will only succeed if `self` is the only unique strong reference (it effectively
Expand All @@ -50,16 +48,6 @@ pub trait VectorMutOps: private::Sealed + Into<VectorMut> {
/// The immutable equivalent of this mutable vector.
type Immutable: VectorOps<Mutable = Self>;

/// Returns the [`Nullability`] of the vector.
fn nullability(&self) -> Nullability;

/// Returns `true` if the nullability of this vector is [`Nullable`].
///
/// [`Nullable`]: Nullability::Nullable
fn is_nullable(&self) -> bool {
self.nullability().is_nullable()
}

/// Returns the number of elements in the vector, also referred to as its "length".
fn len(&self) -> usize;

Expand All @@ -82,20 +70,12 @@ pub trait VectorMutOps: private::Sealed + Into<VectorMut> {
fn reserve(&mut self, additional: usize);

/// Extends the vector by appending elements from another vector.
///
/// # Panics
///
/// Panics if `other` does not have the same nullability as `self`.
fn extend_from_vector(&mut self, other: &Self::Immutable);

/// Appends `n` null elements to the vector (if it is nullable).
/// Appends `n` null elements to the vector.
///
/// Implementors should ensure that they correctly append "null" or garbage values to their
/// elements in addition to adding nulls to their validity mask.
///
/// # Panics
///
/// If `self` is a non-nullable vector, implementors should ensure that this function panics.
/// elements in addition to adding nulls to their validity mask.s.
fn append_nulls(&mut self, n: usize);

/// Converts `self` into an immutable vector.
Expand All @@ -115,7 +95,6 @@ pub trait VectorMutOps: private::Sealed + Into<VectorMut> {
/// `at > capacity`).
fn split_off(&mut self, at: usize) -> Self;

// TODO(connor): Should this panic if other has a different nullability?
/// Absorbs a mutable vector that was previously split off.
///
/// If the two vectors were previously contiguous and not mutated in a way that causes
Expand Down
Loading
Loading