From 64370bb29b9dd44270f9959e1753c82723a5c327 Mon Sep 17 00:00:00 2001 From: "jianjian.xie" Date: Tue, 24 Mar 2026 12:59:17 -0700 Subject: [PATCH 1/2] fix(ci): install protobuf-compiler for lance-encoding build lance-encoding's build script requires protoc. Install it via apt-get on Linux and brew on macOS. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yml | 10 ++++++++++ src/async_dispatcher.rs | 2 +- src/dataset.rs | 4 ++-- src/error.rs | 2 +- src/helpers.rs | 2 +- src/lib.rs | 2 +- src/scanner.rs | 30 +++++++++++++++--------------- tests/c_api_test.rs | 2 +- 8 files changed, 32 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b42062c..a089e5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,8 @@ jobs: with: components: clippy - uses: Swatinem/rust-cache@v2 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - name: Clippy run: cargo clippy --all-targets -- -D warnings @@ -52,6 +54,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - name: Run tests run: cargo test - name: Run C/C++ compilation tests @@ -65,6 +69,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 + - name: Install dependencies + run: brew install protobuf - name: Run tests run: cargo test - name: Run C/C++ compilation tests @@ -78,6 +84,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - name: Check documentation run: RUSTDOCFLAGS="-D warnings" cargo doc --no-deps @@ -91,6 +99,8 @@ jobs: with: toolchain: "1.91.0" - uses: Swatinem/rust-cache@v2 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - name: Check MSRV run: cargo check --all-targets diff --git a/src/async_dispatcher.rs b/src/async_dispatcher.rs index e8fa432..ee4573d 100644 --- a/src/async_dispatcher.rs +++ b/src/async_dispatcher.rs @@ -8,7 +8,7 @@ //! sequentially, avoiding reentrancy and Tokio thread blocking. use std::ffi::c_void; -use std::sync::{mpsc, LazyLock}; +use std::sync::{LazyLock, mpsc}; /// C callback function pointer type for async operations. /// - `ctx`: opaque pointer passed back to the caller diff --git a/src/dataset.rs b/src/dataset.rs index cc2fc32..19ba370 100644 --- a/src/dataset.rs +++ b/src/dataset.rs @@ -9,11 +9,11 @@ use std::sync::Arc; use arrow::ffi::FFI_ArrowSchema; use arrow::ffi_stream::FFI_ArrowArrayStream; use arrow_schema::Schema as ArrowSchema; -use lance::dataset::builder::DatasetBuilder; use lance::Dataset; +use lance::dataset::builder::DatasetBuilder; use lance_core::Result; -use crate::error::{ffi_try, set_last_error, LanceErrorCode}; +use crate::error::{LanceErrorCode, ffi_try, set_last_error}; use crate::helpers; use crate::runtime::block_on; diff --git a/src/error.rs b/src/error.rs index 12a6bb8..3273464 100644 --- a/src/error.rs +++ b/src/error.rs @@ -7,7 +7,7 @@ //! the caller retrieves the error code and message from thread-local storage. use std::cell::RefCell; -use std::ffi::{c_char, CString}; +use std::ffi::{CString, c_char}; use std::ptr; /// Error codes returned by `lance_last_error_code()`. diff --git a/src/helpers.rs b/src/helpers.rs index ce34840..159ced5 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -4,7 +4,7 @@ //! C string parsing utilities. use std::collections::HashMap; -use std::ffi::{c_char, CStr}; +use std::ffi::{CStr, c_char}; use lance_core::{Error, Result}; use snafu::location; diff --git a/src/lib.rs b/src/lib.rs index 7f686d2..e8edec2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,6 +27,6 @@ mod scanner; pub use batch::*; pub use dataset::*; pub use error::{ - lance_free_string, lance_last_error_code, lance_last_error_message, LanceErrorCode, + LanceErrorCode, lance_free_string, lance_last_error_code, lance_last_error_message, }; pub use scanner::*; diff --git a/src/scanner.rs b/src/scanner.rs index 011a4af..1bd3c50 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -12,8 +12,8 @@ use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; use arrow::ffi_stream::FFI_ArrowArrayStream; use arrow_schema::SchemaRef; use futures::{Stream, StreamExt}; -use lance::dataset::scanner::DatasetRecordBatchStream; use lance::Dataset; +use lance::dataset::scanner::DatasetRecordBatchStream; use lance_core::Result; use lance_io::ffi::to_ffi_arrow_array_stream; use lance_io::stream::RecordBatchStream; @@ -21,9 +21,9 @@ use lance_io::stream::RecordBatchStream; use crate::async_dispatcher::{self, LanceCallback}; use crate::batch::LanceBatch; use crate::dataset::LanceDataset; -use crate::error::{clear_last_error, ffi_try, set_lance_error, set_last_error, LanceErrorCode}; +use crate::error::{LanceErrorCode, clear_last_error, ffi_try, set_lance_error, set_last_error}; use crate::helpers; -use crate::runtime::{block_on, RT}; +use crate::runtime::{RT, block_on}; /// Opaque scanner handle. Stores configuration until stream materialization. pub struct LanceScanner { @@ -294,12 +294,12 @@ pub unsafe extern "C" fn lance_scanner_next( let s = unsafe { &mut *scanner }; // Lazily materialize the stream on first call. - if s.stream.is_none() { - if let Err(err) = s.materialize_stream() { - set_lance_error(&err); - unsafe { *out = ptr::null_mut() }; - return -1; - } + if s.stream.is_none() + && let Err(err) = s.materialize_stream() + { + set_lance_error(&err); + unsafe { *out = ptr::null_mut() }; + return -1; } let stream = s.stream.as_mut().unwrap(); @@ -431,12 +431,12 @@ pub unsafe extern "C" fn lance_scanner_poll_next( let s = unsafe { &mut *scanner }; // Lazily materialize the stream. - if s.stream.is_none() { - if let Err(err) = s.materialize_stream() { - set_lance_error(&err); - unsafe { *out = ptr::null_mut() }; - return LancePollStatus::Error; - } + if s.stream.is_none() + && let Err(err) = s.materialize_stream() + { + set_lance_error(&err); + unsafe { *out = ptr::null_mut() }; + return LancePollStatus::Error; } let stream = s.stream.as_mut().unwrap(); diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs index cd0d2d8..f418733 100644 --- a/tests/c_api_test.rs +++ b/tests/c_api_test.rs @@ -10,8 +10,8 @@ use std::ffi::CString; use std::ptr; use std::sync::Arc; -use arrow::ffi::from_ffi; use arrow::ffi::FFI_ArrowSchema; +use arrow::ffi::from_ffi; use arrow::ffi_stream::ArrowArrayStreamReader; use arrow::ffi_stream::FFI_ArrowArrayStream; use arrow::record_batch::RecordBatchReader; From 65205c6dfd612e1a4b7f7a2c401f8868741473d7 Mon Sep 17 00:00:00 2001 From: "jianjian.xie" Date: Tue, 24 Mar 2026 13:12:24 -0700 Subject: [PATCH 2/2] docs: add AGENTS.md and CLAUDE.md for coding agent guidance Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 29 +++++++++++++++++++++++++++++ CLAUDE.md | 1 + 2 files changed, 30 insertions(+) create mode 100644 AGENTS.md create mode 120000 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..14232c3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS.md + +## Structure +Rust FFI source: `src/` +C header (stable ABI): `include/lance.h` +C++ RAII wrappers (header-only): `include/lance.hpp` +Tests (Rust): `tests/c_api_test.rs` +Tests (C/C++): `tests/cpp/` +Historical test data: `test_data/` + +## Commands +check: `cargo check --all-targets` +format: `cargo fmt` +lint: `cargo clippy --all-targets -- -D warnings` +test: `cargo test` +test C/C++ compilation: `cargo test --test compile_and_run_test -- --ignored` + +## Key Patterns +- Opaque handles with `lance_*_open`/`lance_*_close` lifecycle. +- Thread-local error handling via `ffi_try!` macro. +- Arrow C Data Interface for zero-copy data exchange. +- `panic = "abort"` in release to prevent unwinding across FFI. + +## Adding New APIs +1. Add `extern "C"` function in `src/`. +2. Add declaration to `include/lance.h`. +3. Add C++ wrapper to `include/lance.hpp`. +4. Add test in `tests/c_api_test.rs`. + diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file