Skip to content

layout reader ctx #8037

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
onursatici merged 4 commits into from
Jun 2, 2026
Merged

layout reader ctx #8037

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5a58206
layout reader ctx
onursatici May 20, 2026
7d37fcf
break
onursatici May 20, 2026
2b593e5
explicit id
onursatici May 28, 2026
34bf8aa
comments
onursatici Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions vortex-cuda/src/layout.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ impl VTable for CudaFlat {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
_ctx: &vortex::layout::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(CudaFlatReader {
layout: layout.clone(),
Expand Down
7 changes: 6 additions & 1 deletion vortex-file/src/file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,12 @@ impl VortexFile {
.footer
.layout()
// TODO(ngates): we may want to allow the user pass in a name here?
.new_reader("".into(), segment_source, &self.session)?;
.new_reader(
"".into(),
segment_source,
&self.session,
&Default::default(),
)?;

Ok(if let Some(stats) = self.file_stats().cloned() {
Arc::new(FileStatsLayoutReader::new(
Expand Down
8 changes: 4 additions & 4 deletions vortex-file/src/v2/file_stats_reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ mod tests {
)
.await?;

let child = layout.new_reader("".into(), segments, &SESSION)?;
let child = layout.new_reader("".into(), segments, &SESSION, &Default::default())?;

let reader =
FileStatsLayoutReader::new(child, test_file_stats(0, 100), SESSION.clone());
Expand Down Expand Up @@ -349,7 +349,7 @@ mod tests {
)
.await?;

let child = layout.new_reader("".into(), segments, &SESSION)?;
let child = layout.new_reader("".into(), segments, &SESSION, &Default::default())?;

let reader =
FileStatsLayoutReader::new(child, test_file_stats(0, 100), SESSION.clone());
Expand Down Expand Up @@ -400,7 +400,7 @@ mod tests {
)
.await?;

let child = layout.new_reader("".into(), segments, &SESSION)?;
let child = layout.new_reader("".into(), segments, &SESSION, &Default::default())?;

// File-level stats: 1 null in deleted_at.
let mut stats = StatsSet::default();
Expand Down Expand Up @@ -449,7 +449,7 @@ mod tests {
)
.await?;

let child = layout.new_reader("".into(), segments, &SESSION)?;
let child = layout.new_reader("".into(), segments, &SESSION, &Default::default())?;

let reader =
FileStatsLayoutReader::new(child, test_file_null_count_stats(5), SESSION.clone());
Expand Down
2,203 changes: 2,203 additions & 0 deletions vortex-layout/public-api.lock
View file
Open in desktop

Large diffs are not rendered by default.

19 changes: 18 additions & 1 deletion vortex-layout/src/layout.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use vortex_session::registry::Id;

use crate::LayoutEncodingId;
use crate::LayoutEncodingRef;
use crate::LayoutReaderContext;
use crate::LayoutReaderRef;
use crate::VTable;
use crate::display::DisplayLayoutTree;
Expand Down Expand Up @@ -63,11 +64,26 @@ pub trait Layout: 'static + Send + Sync + Debug + private::Sealed {
/// Get the segment IDs for this layout.
fn segment_ids(&self) -> Vec<SegmentId>;

/// Construct a new reader for this layout.
///
/// - `name` — human-readable label for this reader, propagated to child readers
/// (typically by appending a path component) and surfaced in tracing/debug output.
/// - `segment_source` — source of segment bytes for this and any descendant readers
/// constructed from the returned reader; recursive callers should pass the same
/// source through.
/// - `session` — the [`VortexSession`] hosting the encoding/scalar/layout registries
/// and execution context the reader needs at evaluation time.
/// - `ctx` — id-keyed dependency registry threaded through reader construction (see
/// [`LayoutReaderContext`]). Top-level callers (file open, tests) typically pass
/// `&LayoutReaderContext::new()`; recursive callers inside layout implementations
/// must propagate the `ctx` they were handed so ancestor-published values reach
/// descendants.
fn new_reader(
&self,
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
Comment thread
onursatici marked this conversation as resolved.
session: &VortexSession,
ctx: &LayoutReaderContext,
) -> VortexResult<LayoutReaderRef>;
}

Expand Down Expand Up @@ -311,8 +327,9 @@ impl<V: VTable> Layout for LayoutAdapter<V> {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: &LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
V::new_reader(&self.0, name, segment_source, session)
V::new_reader(&self.0, name, segment_source, session, ctx)
}
}

Expand Down
3 changes: 3 additions & 0 deletions vortex-layout/src/layouts/chunked/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use vortex_session::registry::ReadContext;
use crate::LayoutChildType;
use crate::LayoutEncodingRef;
use crate::LayoutId;
use crate::LayoutReaderContext;
use crate::LayoutReaderRef;
use crate::LayoutRef;
use crate::VTable;
Expand Down Expand Up @@ -74,12 +75,14 @@ impl VTable for Chunked {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: &LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(ChunkedReader::new(
layout.clone(),
name,
segment_source,
session,
ctx.clone(),
)))
}

Expand Down
12 changes: 10 additions & 2 deletions vortex-layout/src/layouts/chunked/reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ use vortex_error::vortex_panic;
use vortex_mask::Mask;
use vortex_session::VortexSession;

use crate::LayoutReaderContext;
use crate::LayoutReaderRef;
use crate::LazyReaderChildren;
use crate::layouts::chunked::ChunkedLayout;
Expand All @@ -51,6 +52,7 @@ impl ChunkedReader {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: LayoutReaderContext,
) -> Self {
let nchildren = layout.nchildren();

Expand Down Expand Up @@ -78,6 +80,7 @@ impl ChunkedReader {
names,
segment_source,
session.clone(),
ctx,
);

Self {
Expand Down Expand Up @@ -455,7 +458,12 @@ mod test {
) {
let layout = nested_chunked_layout();
let reader = layout
.new_reader("".into(), Arc::new(TestSegments::default()), &SESSION)
.new_reader(
"".into(),
Arc::new(TestSegments::default()),
&SESSION,
&Default::default(),
)
.unwrap();

let splits = SplitBy::Layout
Expand All @@ -471,7 +479,7 @@ mod test {
) {
block_on(|_h| async {
let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down
2 changes: 2 additions & 0 deletions vortex-layout/src/layouts/dict/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,14 @@ impl VTable for Dict {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: &crate::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(DictReader::try_new(
layout.clone(),
name,
segment_source,
session.clone(),
ctx.clone(),
)?))
}

Expand Down
18 changes: 11 additions & 7 deletions vortex-layout/src/layouts/dict/reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,17 +58,21 @@ impl DictReader {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: VortexSession,
ctx: crate::LayoutReaderContext,
) -> VortexResult<Self> {
let values_len = usize::try_from(layout.values.row_count())?;
let values = layout.values.new_reader(
format!("{name}.values").into(),
Arc::clone(&segment_source),
&session,
&ctx,
)?;
let codes = layout.codes.new_reader(
format!("{name}.codes").into(),
segment_source,
&session,
&ctx,
)?;
let codes =
layout
.codes
.new_reader(format!("{name}.codes").into(), segment_source, &session)?;

Ok(Self {
layout,
Expand Down Expand Up @@ -370,7 +374,7 @@ mod tests {
);
assert!(layout.encoding_id() == LayoutId::new("vortex.dict"));
let actual = layout
.new_reader("".into(), segments, &session)
.new_reader("".into(), segments, &session, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down Expand Up @@ -454,7 +458,7 @@ mod tests {
)),
);
let mask = layout
.new_reader("".into(), segments, &session)
.new_reader("".into(), segments, &session, &Default::default())
.unwrap()
.filter_evaluation(&(0..3), &filter, MaskFuture::new_true(3))
.unwrap()
Expand Down Expand Up @@ -515,7 +519,7 @@ mod tests {
let expression = is_not_null(root());
assert_eq!(layout.encoding_id(), LayoutId::new("vortex.dict"));
let actual = layout
.new_reader("".into(), segments, &session)
.new_reader("".into(), segments, &session, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down
1 change: 1 addition & 0 deletions vortex-layout/src/layouts/flat/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ impl VTable for Flat {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
_ctx: &crate::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(FlatReader::new(
layout.clone(),
Expand Down
6 changes: 3 additions & 3 deletions vortex-layout/src/layouts/flat/reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ mod test {
);

let result = layout
.new_reader("".into(), segments, &SESSION)?
.new_reader("".into(), segments, &SESSION, &Default::default())?
.projection_evaluation(
&(0..layout.row_count()),
&root(),
Expand Down Expand Up @@ -313,7 +313,7 @@ mod test {

let expr = gt(root(), lit(3i32));
let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down Expand Up @@ -350,7 +350,7 @@ mod test {
.unwrap();

let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(&(2..4), &root(), MaskFuture::new_true(2))
.unwrap()
Expand Down
6 changes: 3 additions & 3 deletions vortex-layout/src/layouts/flat/writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ mod tests {
.unwrap();

let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down Expand Up @@ -311,7 +311,7 @@ mod tests {
.unwrap();

let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down Expand Up @@ -384,7 +384,7 @@ mod tests {

// We should be able to read the array we just wrote.
let result: ArrayRef = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down
1 change: 1 addition & 0 deletions vortex-layout/src/layouts/foreign/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ impl Layout for ForeignLayout {
_name: Arc<str>,
_segment_source: Arc<dyn SegmentSource>,
_session: &VortexSession,
_ctx: &crate::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
vortex_bail!(
"Cannot read unknown layout encoding '{}'",
Expand Down
12 changes: 9 additions & 3 deletions vortex-layout/src/layouts/row_idx/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,9 @@ mod tests {
let expr = eq(root(), lit(3i32));
let result = RowIdxLayoutReader::new(
0,
layout.new_reader("".into(), segments, &SESSION).unwrap(),
layout
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap(),
SESSION.clone(),
)
.projection_evaluation(
Expand Down Expand Up @@ -411,7 +413,9 @@ mod tests {
let expr = gt(row_idx(), lit(3u64));
let result = RowIdxLayoutReader::new(
0,
layout.new_reader("".into(), segments, &SESSION).unwrap(),
layout
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap(),
SESSION.clone(),
)
.projection_evaluation(
Expand Down Expand Up @@ -456,7 +460,9 @@ mod tests {

let result = RowIdxLayoutReader::new(
0,
layout.new_reader("".into(), segments, &SESSION).unwrap(),
layout
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap(),
SESSION.clone(),
)
.projection_evaluation(
Expand Down
2 changes: 2 additions & 0 deletions vortex-layout/src/layouts/struct_/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,12 +114,14 @@ impl VTable for Struct {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: &crate::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(StructReader::try_new(
layout.clone(),
name,
segment_source,
session.session(),
ctx.clone(),
)?))
}

Expand Down
Loading
Loading