From d0cfbb7df40ee39f79436db984a6dfb6353304eb Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 14 May 2026 10:44:49 +0100 Subject: [PATCH 01/13] Read encrypted manifest lists --- Cargo.lock | 723 ++++++++++++++++++-- crates/iceberg/src/catalog/utils.rs | 3 +- crates/iceberg/src/inspect/manifests.rs | 6 +- crates/iceberg/src/io/object_cache.rs | 37 +- crates/iceberg/src/spec/snapshot.rs | 136 +++- crates/iceberg/src/spec/table_properties.rs | 10 + crates/iceberg/src/table.rs | 267 +++++++- crates/iceberg/src/transaction/append.rs | 4 +- crates/iceberg/src/transaction/mod.rs | 4 +- crates/iceberg/src/transaction/snapshot.rs | 6 +- 10 files changed, 1136 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19ac987caa..370fbf8a31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", "cpufeatures 0.2.17", ] @@ -49,7 +49,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "const-random", "getrandom 0.3.4", "once_cell", @@ -206,6 +206,15 @@ dependencies = [ "zstd", ] +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -1038,7 +1047,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if", + "cfg-if 1.0.4", "constant_time_eq", "cpufeatures 0.2.17", ] @@ -1117,6 +1126,17 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -1198,6 +1218,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.4" @@ -1216,7 +1242,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.3.0", "rand_core 0.10.0", ] @@ -1416,12 +1442,37 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "const-str" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" + +[[package]] +name = "const_panic" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" +dependencies = [ + "typewit", +] + [[package]] name = "constant_time_eq" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1438,6 +1489,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "countio" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" +dependencies = [ + "futures-io", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1486,7 +1546,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -1660,7 +1720,7 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "crossbeam-utils", "hashbrown 0.14.5", "lock_api", @@ -2734,7 +2794,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "home", "windows-sys 0.48.0", ] @@ -2806,7 +2866,7 @@ version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "rustix", "windows-sys 0.59.0", ] @@ -3008,6 +3068,15 @@ dependencies = [ "slab", ] +[[package]] +name = "gearhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" +dependencies = [ + "cfg-if 0.1.10", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3024,10 +3093,10 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -3037,7 +3106,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", "r-efi 5.3.0", @@ -3051,12 +3120,14 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.0", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -3069,6 +3140,26 @@ dependencies = [ "polyval", ] +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "glob" version = "0.3.3" @@ -3113,7 +3204,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "bytemuck", - "cfg-if", + "cfg-if 1.0.4", "crunchy", "num-traits", "zerocopy", @@ -3168,6 +3259,12 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "heapify" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" + [[package]] name = "heck" version = "0.5.0" @@ -3180,6 +3277,28 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-xet" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.0", + "more-asserts", + "serde", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "uuid", + "xet-client", + "xet-core-structures", + "xet-data", + "xet-runtime", +] + [[package]] name = "hive_metastore" version = "0.2.0" @@ -3350,9 +3469,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.5.10", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -3656,7 +3777,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", - "cfg-if", + "cfg-if 1.0.4", "futures", "iceberg", "iceberg_test_utils", @@ -3958,7 +4079,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "combine", "jni-macros", "jni-sys", @@ -4038,6 +4159,23 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "konst" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" +dependencies = [ + "const_panic", + "konst_proc_macros", + "typewit", +] + +[[package]] +name = "konst_proc_macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" + [[package]] name = "lazy_static" version = "1.5.0" @@ -4267,13 +4405,22 @@ dependencies = [ "serde", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "md-5" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "digest", ] @@ -4356,7 +4503,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -4366,7 +4513,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "downcast", "fragile", "mockall_derive", @@ -4380,7 +4527,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "proc-macro2", "quote", "syn", @@ -4431,6 +4578,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + [[package]] name = "motore" version = "0.4.1" @@ -4508,7 +4661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.4", "cfg_aliases", "libc", "memoffset", @@ -4521,11 +4674,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.4", "cfg_aliases", "libc", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4629,6 +4791,34 @@ dependencies = [ "syn", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.37.3" @@ -4689,6 +4879,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -4710,6 +4906,7 @@ dependencies = [ "opendal-service-azdls", "opendal-service-fs", "opendal-service-gcs", + "opendal-service-hf", "opendal-service-oss", "opendal-service-s3", ] @@ -4849,6 +5046,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-service-hf" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2ab7a2a8a11dfe257ef4db5c0de798acbcd0d6429c37382dad2154bc06a388" +dependencies = [ + "bytes", + "hf-xet", + "http 1.4.0", + "log", + "opendal-core", + "percent-encoding", + "reqwest 0.13.3", + "serde", + "serde_json", +] + [[package]] name = "opendal-service-oss" version = "0.56.0" @@ -4929,6 +5143,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +dependencies = [ + "memchr", +] + [[package]] name = "outref" version = "0.5.2" @@ -4963,7 +5186,7 @@ version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", "redox_syscall 0.5.18", "smallvec", @@ -5225,7 +5448,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.2.17", "opaque-debug", "universal-hash", @@ -5635,6 +5858,15 @@ dependencies = [ "syn", ] +[[package]] +name = "redb" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" +dependencies = [ + "libc", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -5920,6 +6152,8 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", + "serde", + "serde_json", "sync_wrapper", "tokio", "tokio-rustls", @@ -5934,6 +6168,20 @@ dependencies = [ "web-sys", ] +[[package]] +name = "reqwest-middleware" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" +dependencies = [ + "anyhow", + "async-trait", + "http 1.4.0", + "reqwest 0.13.3", + "thiserror 2.0.18", + "tower-service", +] + [[package]] name = "ring" version = "0.17.14" @@ -5941,7 +6189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "getrandom 0.2.17", "libc", "untrusted 0.9.0", @@ -6025,7 +6273,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "glob", "proc-macro-crate", "proc-macro2", @@ -6043,7 +6291,7 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "ordered-multimap", ] @@ -6121,7 +6369,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni", "log", @@ -6167,7 +6415,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.4", "clipboard-win", "fd-lock", "home", @@ -6188,6 +6436,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe-transmute" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" + [[package]] name = "salsa20" version = "0.10.2" @@ -6287,7 +6541,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -6501,7 +6755,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.2.17", "digest", ] @@ -6512,9 +6766,19 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.2.17", "digest", + "sha2-asm", +] + +[[package]] +name = "sha2-asm" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" +dependencies = [ + "cc", ] [[package]] @@ -6526,6 +6790,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "bstr", + "dirs", + "os_str_bytes", +] + [[package]] name = "shlex" version = "1.3.0" @@ -6645,7 +6920,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5661364b38abad49cf1ade6631fcc35d2ccf882a7d68616b4228b7717feb5fba" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -6656,7 +6931,7 @@ checksum = "0275f9f2f07d47556fe60c2759da8bc4be6083b047b491b2d476aa0bfa558eb1" dependencies = [ "bumpalo", "bytes", - "cfg-if", + "cfg-if 1.0.4", "faststr", "itoa", "ref-cast", @@ -6674,7 +6949,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9f944718c33623919878cf74b4c9361eb3024f635733922b26722b14cd3f8cc" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -6866,7 +7141,7 @@ dependencies = [ "stringprep", "thiserror 2.0.18", "tracing", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -6903,7 +7178,7 @@ dependencies = [ "stringprep", "thiserror 2.0.18", "tracing", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -6943,12 +7218,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "libc", "psm", "windows-sys 0.59.0", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -7028,6 +7319,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "2.0.117" @@ -7059,6 +7356,41 @@ dependencies = [ "syn", ] +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -7130,7 +7462,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -7237,6 +7569,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-retry" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40f644c762e9d396831ae2f8935c954b0d758c4532e924bead0f666d0c1c8640" +dependencies = [ + "pin-project-lite", + "rand 0.10.1", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -7400,6 +7743,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.18", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.31" @@ -7432,18 +7788,35 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -7514,6 +7887,12 @@ dependencies = [ "syn", ] +[[package]] +name = "typewit" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc19094686c694eb41b3b99dcc2f2975d4b078512fa22ae6c63f7ca318bdcff7" + [[package]] name = "typify" version = "0.5.0" @@ -7793,6 +8172,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -7817,13 +8205,22 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "once_cell", "rustversion", "wasm-bindgen-macro", @@ -7836,7 +8233,7 @@ version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "futures-util", "js-sys", "once_cell", @@ -7990,9 +8387,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", - "wasite", + "wasite 0.1.0", +] + +[[package]] +name = "whoami" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite 1.0.2", + "web-sys", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -8002,6 +8428,33 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -8015,6 +8468,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -8043,6 +8507,27 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -8154,6 +8639,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -8414,6 +8908,153 @@ dependencies = [ "rustix", ] +[[package]] +name = "xet-client" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "bytes", + "clap", + "crc32fast", + "futures", + "http 1.4.0", + "hyper", + "lazy_static", + "more-asserts", + "rand 0.10.1", + "redb", + "reqwest 0.13.3", + "reqwest-middleware", + "serde", + "serde_json", + "serde_repr", + "statrs", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-retry", + "tracing", + "tracing-subscriber", + "url", + "urlencoding", + "web-time", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-core-structures" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" +dependencies = [ + "async-trait", + "base64", + "blake3", + "bytemuck", + "bytes", + "clap", + "countio", + "csv", + "futures", + "futures-util", + "getrandom 0.4.2", + "heapify", + "itertools 0.14.0", + "lazy_static", + "lz4_flex", + "more-asserts", + "rand 0.10.1", + "regex", + "safe-transmute", + "serde", + "static_assertions", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "uuid", + "web-time", + "xet-runtime", +] + +[[package]] +name = "xet-data" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "clap", + "gearhash", + "http 1.4.0", + "itertools 0.14.0", + "lazy_static", + "more-asserts", + "rand 0.10.1", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "url", + "uuid", + "walkdir", + "xet-client", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-runtime" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "colored", + "const-str", + "ctor", + "dirs", + "futures", + "git-version", + "humantime", + "konst", + "lazy_static", + "libc", + "more-asserts", + "oneshot", + "pin-project", + "rand 0.10.1", + "reqwest 0.13.3", + "serde", + "serde_json", + "shellexpand", + "sysinfo", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "tracing-appender", + "tracing-subscriber", + "whoami 2.1.1", + "winapi", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index d450f9df80..c62a745225 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -47,7 +47,8 @@ pub async fn drop_table_data( // Load all manifest lists concurrently let results: Vec<_> = futures::future::try_join_all(metadata.snapshots().map(|snapshot| async { - let manifest_list = snapshot.load_manifest_list(io, metadata).await?; + // TODO(encryption): thread encryption manager through + let manifest_list = snapshot.load_manifest_list(io, metadata, None).await?; Ok::<_, crate::Error>((snapshot.manifest_list().to_string(), manifest_list)) })) .await?; diff --git a/crates/iceberg/src/inspect/manifests.rs b/crates/iceberg/src/inspect/manifests.rs index 4c30ca2ec5..71af04ef7d 100644 --- a/crates/iceberg/src/inspect/manifests.rs +++ b/crates/iceberg/src/inspect/manifests.rs @@ -162,7 +162,11 @@ impl<'a> ManifestsTable<'a> { if let Some(snapshot) = self.table.metadata().current_snapshot() { let manifest_list = snapshot - .load_manifest_list(self.table.file_io(), &self.table.metadata_ref()) + .load_manifest_list( + self.table.file_io(), + &self.table.metadata_ref(), + self.table.encryption_manager(), + ) .await?; for manifest in manifest_list.entries() { content.append_value(manifest.content as i32); diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 5de45e2acc..87b324a471 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -18,6 +18,7 @@ use std::mem::size_of_val; use std::sync::Arc; +use crate::encryption::EncryptionManager; use crate::io::FileIO; use crate::spec::{ FormatVersion, Manifest, ManifestFile, ManifestList, SchemaId, SnapshotRef, TableMetadataRef, @@ -44,20 +45,25 @@ pub struct ObjectCache { cache: moka::future::Cache, file_io: FileIO, cache_disabled: bool, + encryption_manager: Option>, } impl ObjectCache { /// Creates a new [`ObjectCache`] /// with the default cache size - pub(crate) fn new(file_io: FileIO) -> Self { - Self::new_with_capacity(file_io, DEFAULT_CACHE_SIZE_BYTES) + pub(crate) fn new(file_io: FileIO, encryption_manager: Option>) -> Self { + Self::new_with_capacity(file_io, DEFAULT_CACHE_SIZE_BYTES, encryption_manager) } /// Creates a new [`ObjectCache`] /// with a specific cache size - pub(crate) fn new_with_capacity(file_io: FileIO, cache_size_bytes: u64) -> Self { + pub(crate) fn new_with_capacity( + file_io: FileIO, + cache_size_bytes: u64, + encryption_manager: Option>, + ) -> Self { if cache_size_bytes == 0 { - Self::with_disabled_cache(file_io) + Self::with_disabled_cache(file_io, encryption_manager) } else { Self { cache: moka::future::Cache::builder() @@ -69,17 +75,22 @@ impl ObjectCache { .build(), file_io, cache_disabled: false, + encryption_manager, } } } /// Creates a new [`ObjectCache`] /// with caching disabled - pub(crate) fn with_disabled_cache(file_io: FileIO) -> Self { + pub(crate) fn with_disabled_cache( + file_io: FileIO, + encryption_manager: Option>, + ) -> Self { Self { cache: moka::future::Cache::new(0), file_io, cache_disabled: true, + encryption_manager, } } @@ -127,7 +138,11 @@ impl ObjectCache { ) -> Result> { if self.cache_disabled { return snapshot - .load_manifest_list(&self.file_io, table_metadata) + .load_manifest_list( + &self.file_io, + table_metadata, + self.encryption_manager.as_deref(), + ) .await .map(Arc::new); } @@ -174,7 +189,11 @@ impl ObjectCache { table_metadata: &TableMetadataRef, ) -> Result { let manifest_list = snapshot - .load_manifest_list(&self.file_io, table_metadata) + .load_manifest_list( + &self.file_io, + table_metadata, + self.encryption_manager.as_deref(), + ) .await?; Ok(CachedItem::ManifestList(Arc::new(manifest_list))) @@ -319,7 +338,7 @@ mod tests { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - let object_cache = ObjectCache::with_disabled_cache(fixture.table.file_io().clone()); + let object_cache = ObjectCache::with_disabled_cache(fixture.table.file_io().clone(), None); let result_manifest_list = object_cache .get_manifest_list( @@ -352,7 +371,7 @@ mod tests { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - let object_cache = ObjectCache::new(fixture.table.file_io().clone()); + let object_cache = ObjectCache::new(fixture.table.file_io().clone(), None); // not in cache let result_manifest_list = object_cache diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 3b8a3c934e..eec551af26 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -26,6 +26,7 @@ use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; use super::table_metadata::SnapshotLog; +use crate::encryption::{EncryptedInputFile, EncryptionManager}; use crate::error::{Result, timestamp_ms_to_utc}; use crate::io::FileIO; use crate::spec::{ManifestList, SchemaId, SchemaRef, TableMetadata}; @@ -199,8 +200,22 @@ impl Snapshot { &self, file_io: &FileIO, table_metadata: &TableMetadata, + encryption_manager: Option<&EncryptionManager>, ) -> Result { - let manifest_list_content = file_io.new_input(&self.manifest_list)?.read().await?; + let manifest_list_content = match (&self.encryption_key_id, encryption_manager) { + (Some(_), None) => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snapshot has encryption_key_id but no EncryptionManager configured on Table", + )); + } + (Some(key_id), Some(em)) => { + let key_metadata = em.decrypt_manifest_list_key_metadata(key_id).await?; + let input = file_io.new_input(&self.manifest_list)?; + EncryptedInputFile::new(input, key_metadata).read().await? + } + (None, _) => file_io.new_input(&self.manifest_list)?.read().await?, + }; ManifestList::parse_with_version( &manifest_list_content, // TODO: You don't really need the version since you could just project any Avro in @@ -521,6 +536,7 @@ impl SnapshotRetention { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; use chrono::{TimeZone, Utc}; @@ -729,4 +745,122 @@ mod tests { assert_eq!(v2_snapshot.parent_snapshot_id(), None); assert_eq!(v2_snapshot.schema_id(), None); } + + use bytes::Bytes; + + use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; + use crate::encryption::{EncryptionManager, StandardKeyMetadata}; + use crate::io::FileIO; + use crate::spec::manifest_list::{ManifestList, ManifestListWriter}; + + const ENCRYPTION_TEST_V3_METADATA: &str = r#"{ + "format-version": 3, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "memory:///table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + "refs": {}, + "next-row-id": 0 + }"#; + + fn encryption_test_metadata() -> TableMetadata { + serde_json::from_str(ENCRYPTION_TEST_V3_METADATA).unwrap() + } + + fn encryption_test_kms() -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + Arc::new(kms) + } + + fn encryption_test_manager() -> EncryptionManager { + EncryptionManager::builder() + .kms_client(encryption_test_kms()) + .table_key_id("master-1") + .build() + } + + async fn write_v3_manifest_list_bytes(io: &FileIO, path: &str) -> Bytes { + let output = io.new_output(path).unwrap(); + let mut writer = ManifestListWriter::v3(output, 1, None, 0, Some(0)); + writer.add_manifests(std::iter::empty()).unwrap(); + writer.close().await.unwrap(); + io.new_input(path).unwrap().read().await.unwrap() + } + + fn snapshot_pointing_at(manifest_list_path: &str, key_id: Option) -> Snapshot { + Snapshot::builder() + .with_snapshot_id(1) + .with_sequence_number(0) + .with_timestamp_ms(0) + .with_manifest_list(manifest_list_path.to_string()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_encryption_key_id(key_id) + .build() + } + + #[tokio::test] + async fn load_manifest_list_errors_when_encrypted_but_no_manager_configured() { + let io = FileIO::new_with_memory(); + let snapshot = snapshot_pointing_at( + "memory:///table/metadata/manifest-list-enc.avro", + Some("k1".to_string()), + ); + let metadata = encryption_test_metadata(); + + let err = snapshot + .load_manifest_list(&io, &metadata, None) + .await + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + } + + #[tokio::test] + async fn load_manifest_list_decrypts_roundtrip() { + let io = FileIO::new_with_memory(); + let plain_path = "memory:///table/metadata/manifest-list-plain.avro"; + let encrypted_path = "memory:///table/metadata/manifest-list-enc.avro"; + + // Build raw manifest list bytes via the standard writer. + let raw_bytes = write_v3_manifest_list_bytes(&io, plain_path).await; + + // Encrypt those bytes to a second path and capture the file's key metadata. + let mgr = encryption_test_manager(); + let encrypted_output = mgr.encrypt(io.new_output(encrypted_path).unwrap()); + let std_key_metadata: StandardKeyMetadata = encrypted_output.key_metadata().clone(); + encrypted_output.write(raw_bytes).await.unwrap(); + + // Wrap the file's key metadata with a KEK and record the resulting wrapped + // entry's id on the snapshot. + let key_id = mgr + .encrypt_manifest_list_key_metadata(&std_key_metadata) + .await + .unwrap(); + + let snapshot = snapshot_pointing_at(encrypted_path, Some(key_id)); + let metadata = encryption_test_metadata(); + + let manifest_list: ManifestList = snapshot + .load_manifest_list(&io, &metadata, Some(&mgr)) + .await + .unwrap(); + assert_eq!(manifest_list.entries().len(), 0); + } } diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index dc21da565c..fecfebf9d4 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -128,6 +128,9 @@ pub struct TableProperties { pub cdc_max_chunk_size: usize, /// Content-defined chunking normalization level (gearhash bit adjustment). pub cdc_norm_level: i32, + /// The master key id used to encrypt this table's manifest list and data + /// files. `None` if `encryption.key-id` is not set. + pub encryption_key_id: Option, } impl TableProperties { @@ -253,6 +256,10 @@ impl TableProperties { "write.parquet.content-defined-chunking.norm-level"; /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`. pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; + + /// Property key for the master key id used to encrypt the table's manifest + /// list and data files. Matches Java's `TableProperties.ENCRYPTION_TABLE_KEY`. + pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; } impl TryFrom<&HashMap> for TableProperties { @@ -322,6 +329,9 @@ impl TryFrom<&HashMap> for TableProperties { TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL, TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, )?, + encryption_key_id: props + .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) + .cloned(), }) } } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index d2ba93f854..a750aa3558 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -20,12 +20,14 @@ use std::sync::Arc; use crate::arrow::ArrowReaderBuilder; +use crate::encryption::EncryptionManager; +use crate::encryption::kms::KeyManagementClient; use crate::inspect::MetadataTable; use crate::io::FileIO; use crate::io::object_cache::ObjectCache; use crate::runtime::Runtime; use crate::scan::TableScanBuilder; -use crate::spec::{SchemaRef, TableMetadata, TableMetadataRef}; +use crate::spec::{FormatVersion, SchemaRef, TableMetadata, TableMetadataRef}; use crate::{Error, ErrorKind, Result, TableIdent}; /// Builder to create table scan. @@ -34,6 +36,7 @@ pub struct TableBuilder { metadata_location: Option, metadata: Option, identifier: Option, + kms_client: Option>, readonly: bool, disable_cache: bool, cache_size_bytes: Option, @@ -47,6 +50,7 @@ impl TableBuilder { metadata_location: None, metadata: None, identifier: None, + kms_client: None, readonly: false, disable_cache: false, cache_size_bytes: None, @@ -104,6 +108,16 @@ impl TableBuilder { self } + /// optional - sets the KMS client used to unwrap keys for table encryption. + /// + /// If the table metadata has the `encryption.key-id` property set, a + /// [`KeyManagementClient`] must be provided here so the table can build + /// an [`EncryptionManager`]; otherwise [`Self::build`] will return an error. + pub fn kms_client(mut self, kms_client: Arc) -> Self { + self.kms_client = Some(kms_client); + self + } + /// build the Table pub fn build(self) -> Result { let Self { @@ -111,6 +125,7 @@ impl TableBuilder { metadata_location, metadata, identifier, + kms_client, readonly, disable_cache, cache_size_bytes, @@ -145,15 +160,24 @@ impl TableBuilder { )); }; + let encryption_manager = maybe_configure_encryption(kms_client.as_ref(), &metadata)?; + let object_cache = if disable_cache { - Arc::new(ObjectCache::with_disabled_cache(file_io.clone())) + Arc::new(ObjectCache::with_disabled_cache( + file_io.clone(), + encryption_manager.clone(), + )) } else if let Some(cache_size_bytes) = cache_size_bytes { Arc::new(ObjectCache::new_with_capacity( file_io.clone(), cache_size_bytes, + encryption_manager.clone(), )) } else { - Arc::new(ObjectCache::new(file_io.clone())) + Arc::new(ObjectCache::new( + file_io.clone(), + encryption_manager.clone(), + )) }; Ok(Table { @@ -164,6 +188,7 @@ impl TableBuilder { readonly, object_cache, runtime, + encryption_manager, }) } } @@ -178,6 +203,7 @@ pub struct Table { readonly: bool, object_cache: Arc, runtime: Runtime, + encryption_manager: Option>, } impl Table { @@ -238,6 +264,16 @@ impl Table { self.object_cache.clone() } + /// Returns the [`EncryptionManager`] for this table, if encryption is + /// configured. + /// + /// A manager is present iff the table metadata has the + /// `encryption.key-id` property set and a [`KeyManagementClient`] was + /// supplied to the [`TableBuilder`]. + pub fn encryption_manager(&self) -> Option<&EncryptionManager> { + self.encryption_manager.as_deref() + } + /// Creates a table scan. pub fn scan(&self) -> TableScanBuilder<'_> { TableScanBuilder::new(self) @@ -356,9 +392,54 @@ impl StaticTable { } } +/// If the table metadata sets the `encryption.key-id` property, build an +/// [`EncryptionManager`] for the table. +/// +/// Returns `Ok(None)` if the property is not set. Returns an error if the +/// property is set but no [`KeyManagementClient`] was provided. +fn maybe_configure_encryption( + kms_client: Option<&Arc>, + metadata: &TableMetadataRef, +) -> Result>> { + let Some(table_key_id) = metadata.table_properties()?.encryption_key_id else { + return Ok(None); + }; + + // Encryption is a v3 feature: `encryption-keys` table metadata and the + // snapshot `key-id` field are introduced in format version 3. + if metadata.format_version() < FormatVersion::V3 { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Table encryption requires format version 3, found {}", + metadata.format_version() + ), + )); + } + + let kms_client = kms_client.ok_or_else(|| { + Error::new( + ErrorKind::FeatureUnsupported, + "Table has encryption.key-id set but no KeyManagementClient was provided to TableBuilder", + ) + })?; + + let em = EncryptionManager::builder() + .kms_client(Arc::clone(kms_client)) + .table_key_id(table_key_id) + .encryption_keys(metadata.encryption_keys.clone()) + .build(); + Ok(Some(Arc::new(em))) +} + #[cfg(test)] mod tests { + use std::collections::HashMap; + use super::*; + use crate::encryption::StandardKeyMetadata; + use crate::encryption::kms::MemoryKeyManagementClient; + use crate::spec::{ManifestListWriter, Operation, Snapshot, Summary, TableProperties}; #[tokio::test] async fn test_static_table_from_file() { @@ -432,4 +513,184 @@ mod tests { assert!(!table.readonly()); assert_eq!(table.identifier.name(), "table"); } + + const V3_METADATA: &str = r#"{ + "format-version": 3, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "memory:///table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + "refs": {}, + "next-row-id": 0 + }"#; + + const V2_METADATA: &str = r#"{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "memory:///table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + "refs": {} + }"#; + + fn make_kms() -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + Arc::new(kms) + } + + async fn write_empty_manifest_list_bytes(io: &FileIO, path: &str) -> bytes::Bytes { + let output = io.new_output(path).unwrap(); + let mut writer = ManifestListWriter::v3(output, 1, None, 0, Some(0)); + writer.add_manifests(std::iter::empty()).unwrap(); + writer.close().await.unwrap(); + io.new_input(path).unwrap().read().await.unwrap() + } + + #[tokio::test] + async fn table_decrypts_manifest_list_via_object_cache() { + let io = FileIO::new_with_memory(); + let plain_path = "memory:///table/metadata/manifest-list-plain.avro"; + let encrypted_path = "memory:///table/metadata/manifest-list-enc.avro"; + + // Encrypt a real manifest list onto the encrypted path. + let raw = write_empty_manifest_list_bytes(&io, plain_path).await; + let kms = make_kms(); + let mgr = EncryptionManager::builder() + .kms_client(Arc::clone(&kms)) + .table_key_id("master-1") + .build(); + let encrypted_output = mgr.encrypt(io.new_output(encrypted_path).unwrap()); + let std_km: StandardKeyMetadata = encrypted_output.key_metadata().clone(); + encrypted_output.write(raw).await.unwrap(); + let key_id = mgr + .encrypt_manifest_list_key_metadata(&std_km) + .await + .unwrap(); + + // Snapshot the wrapped keys (manifest-list entry + KEK) the manager produced. + let encryption_keys = mgr.with_encryption_keys(|keys| keys.clone()); + + // Build a TableMetadata with those keys, the encryption.key-id property, + // and a snapshot whose encryption_key_id points at the wrapped entry. + let mut metadata: TableMetadata = serde_json::from_str(V3_METADATA).unwrap(); + metadata.properties.insert( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "master-1".to_string(), + ); + metadata.encryption_keys = encryption_keys; + + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_sequence_number(0) + .with_timestamp_ms(0) + .with_manifest_list(encrypted_path.to_string()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_schema_id(0) + .with_encryption_key_id(Some(key_id)) + .build(); + let snapshot_ref = Arc::new(snapshot); + metadata + .snapshots + .insert(snapshot_ref.snapshot_id(), snapshot_ref.clone()); + metadata.current_snapshot_id = Some(snapshot_ref.snapshot_id()); + + // Build the table with the KMS client, then read via the object cache. + let table = Table::builder() + .file_io(io) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .kms_client(kms) + .build() + .unwrap(); + assert!(table.encryption_manager().is_some()); + + let manifest_list = table + .object_cache() + .get_manifest_list(&snapshot_ref, &table.metadata_ref()) + .await + .unwrap(); + assert_eq!(manifest_list.entries().len(), 0); + } + + #[tokio::test] + async fn table_builder_errors_when_encryption_key_id_set_but_no_kms() { + let mut metadata: TableMetadata = serde_json::from_str(V3_METADATA).unwrap(); + metadata.properties.insert( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "master-1".to_string(), + ); + + let err = Table::builder() + .file_io(FileIO::new_with_memory()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .build() + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + } + + #[tokio::test] + async fn table_builder_errors_when_encryption_set_on_pre_v3_table() { + // Encryption is a v3 spec feature; setting encryption.key-id on a v2 table + // must be rejected even when a KMS client is available. + let mut metadata: TableMetadata = serde_json::from_str(V2_METADATA).unwrap(); + metadata.properties.insert( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "master-1".to_string(), + ); + + let err = Table::builder() + .file_io(FileIO::new_with_memory()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .kms_client(make_kms()) + .build() + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + } + + #[tokio::test] + async fn table_builder_skips_encryption_when_property_absent() { + let metadata: TableMetadata = serde_json::from_str(V2_METADATA).unwrap(); + let table = Table::builder() + .file_io(FileIO::new_with_memory()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "plain"]).unwrap()) + .kms_client(make_kms()) + .build() + .unwrap(); + assert!(table.encryption_manager().is_none()); + } } diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 08d4032409..dc99d9a159 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -129,9 +129,11 @@ impl SnapshotProduceOperation for FastAppendOperation { }; let manifest_list = snapshot + // TODO(encryption): thread encryption manager through .load_manifest_list( snapshot_produce.table.file_io(), &snapshot_produce.table.metadata_ref(), + snapshot_produce.table.encryption_manager(), ) .await?; @@ -305,7 +307,7 @@ mod tests { unreachable!() }; let manifest_list = new_snapshot - .load_manifest_list(table.file_io(), table.metadata()) + .load_manifest_list(table.file_io(), table.metadata(), None) .await .unwrap(); assert_eq!(1, manifest_list.entries().len()); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 04ee1997d0..16f0aad277 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -609,7 +609,7 @@ mod test_row_lineage { .metadata() .current_snapshot() .unwrap() - .load_manifest_list(table.file_io(), table.metadata()) + .load_manifest_list(table.file_io(), table.metadata(), None) .await .unwrap(); @@ -637,7 +637,7 @@ mod test_row_lineage { .metadata() .current_snapshot() .unwrap() - .load_manifest_list(table.file_io(), table.metadata()) + .load_manifest_list(table.file_io(), table.metadata(), None) .await .unwrap(); assert_eq!(manifest_list.entries().len(), 2); diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 8f643a7d1e..b9b8912525 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -173,7 +173,11 @@ impl<'a> SnapshotProducer<'a> { let mut referenced_files = Vec::new(); if let Some(current_snapshot) = self.table.metadata().current_snapshot() { let manifest_list = current_snapshot - .load_manifest_list(self.table.file_io(), &self.table.metadata_ref()) + .load_manifest_list( + self.table.file_io(), + &self.table.metadata_ref(), + self.table.encryption_manager(), + ) .await?; for manifest_list_entry in manifest_list.entries() { let manifest = manifest_list_entry From f98f69c1be4debcce9ffc84e5b00902000cb5e85 Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 14 May 2026 15:07:14 +0100 Subject: [PATCH 02/13] wire in purge_table --- crates/catalog/glue/src/catalog.rs | 1 + crates/catalog/hms/src/catalog.rs | 1 + crates/catalog/sql/src/catalog.rs | 1 + crates/iceberg/src/catalog/memory/catalog.rs | 1 + crates/iceberg/src/catalog/utils.rs | 7 +++++-- 5 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/catalog/glue/src/catalog.rs b/crates/catalog/glue/src/catalog.rs index c51f6a6a89..11a9ffd0bc 100644 --- a/crates/catalog/glue/src/catalog.rs +++ b/crates/catalog/glue/src/catalog.rs @@ -681,6 +681,7 @@ impl Catalog for GlueCatalog { table_info.file_io(), table_info.metadata(), table_info.metadata_location(), + table_info.encryption_manager(), ) .await } diff --git a/crates/catalog/hms/src/catalog.rs b/crates/catalog/hms/src/catalog.rs index d778a3d5fc..c76dab8ed3 100644 --- a/crates/catalog/hms/src/catalog.rs +++ b/crates/catalog/hms/src/catalog.rs @@ -628,6 +628,7 @@ impl Catalog for HmsCatalog { table_info.file_io(), table_info.metadata(), table_info.metadata_location(), + table_info.encryption_manager(), ) .await } diff --git a/crates/catalog/sql/src/catalog.rs b/crates/catalog/sql/src/catalog.rs index c7bf9d0cfd..4e6413fb46 100644 --- a/crates/catalog/sql/src/catalog.rs +++ b/crates/catalog/sql/src/catalog.rs @@ -778,6 +778,7 @@ impl Catalog for SqlCatalog { table_info.file_io(), table_info.metadata(), table_info.metadata_location(), + table_info.encryption_manager(), ) .await } diff --git a/crates/iceberg/src/catalog/memory/catalog.rs b/crates/iceberg/src/catalog/memory/catalog.rs index 3ae01a23df..2f2e2a215e 100644 --- a/crates/iceberg/src/catalog/memory/catalog.rs +++ b/crates/iceberg/src/catalog/memory/catalog.rs @@ -347,6 +347,7 @@ impl Catalog for MemoryCatalog { table_info.file_io(), table_info.metadata(), table_info.metadata_location(), + table_info.encryption_manager(), ) .await } diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index c62a745225..3e4e1f6e45 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -22,6 +22,7 @@ use std::collections::HashSet; use futures::{TryStreamExt, stream}; use crate::Result; +use crate::encryption::EncryptionManager; use crate::io::FileIO; use crate::spec::TableMetadata; @@ -40,6 +41,7 @@ pub async fn drop_table_data( io: &FileIO, metadata: &TableMetadata, metadata_location: Option<&str>, + encryption_manager: Option<&EncryptionManager>, ) -> Result<()> { let mut manifest_lists_to_delete: HashSet = HashSet::new(); let mut manifests_to_delete: HashSet = HashSet::new(); @@ -47,8 +49,9 @@ pub async fn drop_table_data( // Load all manifest lists concurrently let results: Vec<_> = futures::future::try_join_all(metadata.snapshots().map(|snapshot| async { - // TODO(encryption): thread encryption manager through - let manifest_list = snapshot.load_manifest_list(io, metadata, None).await?; + let manifest_list = snapshot + .load_manifest_list(io, metadata, encryption_manager) + .await?; Ok::<_, crate::Error>((snapshot.manifest_list().to_string(), manifest_list)) })) .await?; From f8122d97acd9e3ae259f9feca240ee331cd976a5 Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 14 May 2026 15:23:51 +0100 Subject: [PATCH 03/13] fmt --- crates/iceberg/src/spec/table_properties.rs | 2 +- crates/iceberg/src/transaction/append.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index fecfebf9d4..5647e8113f 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -258,7 +258,7 @@ impl TableProperties { pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; /// Property key for the master key id used to encrypt the table's manifest - /// list and data files. Matches Java's `TableProperties.ENCRYPTION_TABLE_KEY`. + /// list and data files as defined in https://iceberg.apache.org/docs/nightly/encryption/. pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; } diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index dc99d9a159..e4cff058a7 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -129,7 +129,6 @@ impl SnapshotProduceOperation for FastAppendOperation { }; let manifest_list = snapshot - // TODO(encryption): thread encryption manager through .load_manifest_list( snapshot_produce.table.file_io(), &snapshot_produce.table.metadata_ref(), From 1aee0908e7a33bd664898ab310cebf77df959269 Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 14 May 2026 15:52:54 +0100 Subject: [PATCH 04/13] fix --- crates/integrations/datafusion/src/physical_plan/commit.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs index 3b3ff3d6b3..f5b75a788d 100644 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ b/crates/integrations/datafusion/src/physical_plan/commit.rs @@ -505,7 +505,11 @@ mod tests { // Load the manifest list to verify the data files were added let manifest_list = current_snapshot - .load_manifest_list(updated_table.file_io(), updated_table.metadata()) + .load_manifest_list( + updated_table.file_io(), + updated_table.metadata(), + updated_table.encryption_manager(), + ) .await?; // There should be at least one manifest From 2feaeeaaaacf29bd04c1a641f6ec47fa42ca8b0c Mon Sep 17 00:00:00 2001 From: Xander Date: Fri, 15 May 2026 11:02:26 +0100 Subject: [PATCH 05/13] key_size --- crates/iceberg/src/spec/table_properties.rs | 12 ++++++++++++ crates/iceberg/src/table.rs | 10 ++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 5647e8113f..883f05e5a5 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -131,6 +131,8 @@ pub struct TableProperties { /// The master key id used to encrypt this table's manifest list and data /// files. `None` if `encryption.key-id` is not set. pub encryption_key_id: Option, + /// The encryption data encryption key length in bytes. + pub encryption_data_key_length: usize, } impl TableProperties { @@ -260,6 +262,11 @@ impl TableProperties { /// Property key for the master key id used to encrypt the table's manifest /// list and data files as defined in https://iceberg.apache.org/docs/nightly/encryption/. pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; + + /// Property key for the encryption data encryption key (DEK) length in bytes. + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; + /// Default value for the encryption DEK length (16 bytes = AES-128). + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; } impl TryFrom<&HashMap> for TableProperties { @@ -332,6 +339,11 @@ impl TryFrom<&HashMap> for TableProperties { encryption_key_id: props .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) .cloned(), + encryption_data_key_length: parse_property( + props, + TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, + TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, + )?, }) } } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index a750aa3558..0b34c1f474 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -19,9 +19,11 @@ use std::sync::Arc; +use backon::BackoffBuilder; + use crate::arrow::ArrowReaderBuilder; -use crate::encryption::EncryptionManager; use crate::encryption::kms::KeyManagementClient; +use crate::encryption::{AesKeySize, EncryptionManager}; use crate::inspect::MetadataTable; use crate::io::FileIO; use crate::io::object_cache::ObjectCache; @@ -401,7 +403,8 @@ fn maybe_configure_encryption( kms_client: Option<&Arc>, metadata: &TableMetadataRef, ) -> Result>> { - let Some(table_key_id) = metadata.table_properties()?.encryption_key_id else { + let table_properties = metadata.table_properties()?; + let Some(table_key_id) = table_properties.encryption_key_id else { return Ok(None); }; @@ -428,6 +431,9 @@ fn maybe_configure_encryption( .kms_client(Arc::clone(kms_client)) .table_key_id(table_key_id) .encryption_keys(metadata.encryption_keys.clone()) + .key_size(AesKeySize::from_key_length( + table_properties.encryption_data_key_length, + )?) .build(); Ok(Some(Arc::new(em))) } From 925280546575b41bb65ef4da59ab0a29181447c0 Mon Sep 17 00:00:00 2001 From: Xander Date: Fri, 15 May 2026 13:24:37 +0100 Subject: [PATCH 06/13] Whoops --- crates/iceberg/src/table.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 0b34c1f474..6abb0cf609 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -19,8 +19,6 @@ use std::sync::Arc; -use backon::BackoffBuilder; - use crate::arrow::ArrowReaderBuilder; use crate::encryption::kms::KeyManagementClient; use crate::encryption::{AesKeySize, EncryptionManager}; From 73b0aa2a48145882325a886c89c209f4bd5958a9 Mon Sep 17 00:00:00 2001 From: Xander Date: Tue, 19 May 2026 15:27:44 +0100 Subject: [PATCH 07/13] fix tests --- crates/iceberg/src/table.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 6abb0cf609..1eb2e6bdc9 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -636,6 +636,7 @@ mod tests { .metadata(metadata) .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) .kms_client(kms) + .runtime(Runtime::try_current().unwrap()) .build() .unwrap(); assert!(table.encryption_manager().is_some()); @@ -660,6 +661,7 @@ mod tests { .file_io(FileIO::new_with_memory()) .metadata(metadata) .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); @@ -680,6 +682,7 @@ mod tests { .metadata(metadata) .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) .kms_client(make_kms()) + .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); @@ -693,6 +696,7 @@ mod tests { .metadata(metadata) .identifier(TableIdent::from_strs(["ns", "plain"]).unwrap()) .kms_client(make_kms()) + .runtime(Runtime::try_current().unwrap()) .build() .unwrap(); assert!(table.encryption_manager().is_none()); From 44f58f60b5a374dd8298d467d6c36d57da38569b Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 20 May 2026 18:58:24 +0100 Subject: [PATCH 08/13] Use precondition error --- crates/iceberg/src/spec/snapshot.rs | 2 +- crates/iceberg/src/table.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index eec551af26..30bcf4e493 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -205,7 +205,7 @@ impl Snapshot { let manifest_list_content = match (&self.encryption_key_id, encryption_manager) { (Some(_), None) => { return Err(Error::new( - ErrorKind::FeatureUnsupported, + ErrorKind::PreconditionFailed, "Snapshot has encryption_key_id but no EncryptionManager configured on Table", )); } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 1eb2e6bdc9..0dcc65524a 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -410,7 +410,7 @@ fn maybe_configure_encryption( // snapshot `key-id` field are introduced in format version 3. if metadata.format_version() < FormatVersion::V3 { return Err(Error::new( - ErrorKind::FeatureUnsupported, + ErrorKind::PreconditionFailed, format!( "Table encryption requires format version 3, found {}", metadata.format_version() @@ -420,7 +420,7 @@ fn maybe_configure_encryption( let kms_client = kms_client.ok_or_else(|| { Error::new( - ErrorKind::FeatureUnsupported, + ErrorKind::PreconditionFailed, "Table has encryption.key-id set but no KeyManagementClient was provided to TableBuilder", ) })?; From f1f6496f099bb890fb009508d02f97f049efe1a9 Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 20 May 2026 19:33:55 +0100 Subject: [PATCH 09/13] hoist test functions and imports --- crates/iceberg/src/spec/snapshot.rs | 140 ++++++++++++++-------------- 1 file changed, 69 insertions(+), 71 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 30bcf4e493..e21543858b 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -538,12 +538,80 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; + use bytes::Bytes; use chrono::{TimeZone, Utc}; + use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; + use crate::encryption::{EncryptionManager, StandardKeyMetadata}; + use crate::io::FileIO; use crate::spec::TableMetadata; + use crate::spec::manifest_list::{ManifestList, ManifestListWriter}; use crate::spec::snapshot::_serde::SnapshotV1; use crate::spec::snapshot::{Operation, Snapshot, Summary}; + const ENCRYPTION_TEST_V3_METADATA: &str = r#"{ + "format-version": 3, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "memory:///table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + "refs": {}, + "next-row-id": 0 + }"#; + + fn encryption_test_metadata() -> TableMetadata { + serde_json::from_str(ENCRYPTION_TEST_V3_METADATA).unwrap() + } + + fn encryption_test_kms() -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + Arc::new(kms) + } + + fn encryption_test_manager() -> EncryptionManager { + EncryptionManager::builder() + .kms_client(encryption_test_kms()) + .table_key_id("master-1") + .build() + } + + async fn write_v3_manifest_list_bytes(io: &FileIO, path: &str) -> Bytes { + let output = io.new_output(path).unwrap(); + let mut writer = ManifestListWriter::v3(output, 1, None, 0, Some(0)); + writer.add_manifests(std::iter::empty()).unwrap(); + writer.close().await.unwrap(); + io.new_input(path).unwrap().read().await.unwrap() + } + + fn snapshot_pointing_at(manifest_list_path: &str, key_id: Option) -> Snapshot { + Snapshot::builder() + .with_snapshot_id(1) + .with_sequence_number(0) + .with_timestamp_ms(0) + .with_manifest_list(manifest_list_path.to_string()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_encryption_key_id(key_id) + .build() + } + #[test] fn schema() { let record = r#" @@ -746,76 +814,6 @@ mod tests { assert_eq!(v2_snapshot.schema_id(), None); } - use bytes::Bytes; - - use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; - use crate::encryption::{EncryptionManager, StandardKeyMetadata}; - use crate::io::FileIO; - use crate::spec::manifest_list::{ManifestList, ManifestListWriter}; - - const ENCRYPTION_TEST_V3_METADATA: &str = r#"{ - "format-version": 3, - "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", - "location": "memory:///table", - "last-sequence-number": 0, - "last-updated-ms": 1602638573590, - "last-column-id": 1, - "current-schema-id": 0, - "schemas": [{"type": "struct", "schema-id": 0, "fields": [ - {"id": 1, "name": "x", "required": true, "type": "long"} - ]}], - "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": []}], - "last-partition-id": 1000, - "default-sort-order-id": 0, - "sort-orders": [{"order-id": 0, "fields": []}], - "properties": {}, - "snapshots": [], - "snapshot-log": [], - "metadata-log": [], - "refs": {}, - "next-row-id": 0 - }"#; - - fn encryption_test_metadata() -> TableMetadata { - serde_json::from_str(ENCRYPTION_TEST_V3_METADATA).unwrap() - } - - fn encryption_test_kms() -> Arc { - let kms = MemoryKeyManagementClient::new(); - kms.add_master_key("master-1").unwrap(); - Arc::new(kms) - } - - fn encryption_test_manager() -> EncryptionManager { - EncryptionManager::builder() - .kms_client(encryption_test_kms()) - .table_key_id("master-1") - .build() - } - - async fn write_v3_manifest_list_bytes(io: &FileIO, path: &str) -> Bytes { - let output = io.new_output(path).unwrap(); - let mut writer = ManifestListWriter::v3(output, 1, None, 0, Some(0)); - writer.add_manifests(std::iter::empty()).unwrap(); - writer.close().await.unwrap(); - io.new_input(path).unwrap().read().await.unwrap() - } - - fn snapshot_pointing_at(manifest_list_path: &str, key_id: Option) -> Snapshot { - Snapshot::builder() - .with_snapshot_id(1) - .with_sequence_number(0) - .with_timestamp_ms(0) - .with_manifest_list(manifest_list_path.to_string()) - .with_summary(Summary { - operation: Operation::Append, - additional_properties: HashMap::new(), - }) - .with_encryption_key_id(key_id) - .build() - } - #[tokio::test] async fn load_manifest_list_errors_when_encrypted_but_no_manager_configured() { let io = FileIO::new_with_memory(); @@ -829,7 +827,7 @@ mod tests { .load_manifest_list(&io, &metadata, None) .await .unwrap_err(); - assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + assert_eq!(err.kind(), crate::ErrorKind::PreconditionFailed); } #[tokio::test] From 6062a1a367f933a752f63ca3bcc7150b901eb193 Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 20 May 2026 22:33:15 +0100 Subject: [PATCH 10/13] fix tests --- crates/iceberg/src/table.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 0dcc65524a..cf36c3653f 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -664,7 +664,7 @@ mod tests { .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); - assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); } #[tokio::test] @@ -685,7 +685,7 @@ mod tests { .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); - assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); } #[tokio::test] From 43bccf5bbb11dd4e441b6b684b6c2128bf03e1db Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 20 May 2026 22:48:11 +0100 Subject: [PATCH 11/13] fix different test --- crates/iceberg/src/table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index cf36c3653f..f09f529414 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -664,7 +664,7 @@ mod tests { .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); - assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); } #[tokio::test] From 06c643ee9ff7e88803fc156ffd56a4abe22c915a Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 21 May 2026 10:09:47 +0100 Subject: [PATCH 12/13] Add manifest list loader --- crates/iceberg/src/inspect/manifests.rs | 8 +---- crates/iceberg/src/spec/snapshot.rs | 2 +- crates/iceberg/src/table.rs | 31 ++++++++++++++++++- crates/iceberg/src/transaction/append.rs | 10 +++--- crates/iceberg/src/transaction/mod.rs | 17 ++-------- crates/iceberg/src/transaction/snapshot.rs | 10 +++--- .../datafusion/src/physical_plan/commit.rs | 9 ++---- 7 files changed, 46 insertions(+), 41 deletions(-) diff --git a/crates/iceberg/src/inspect/manifests.rs b/crates/iceberg/src/inspect/manifests.rs index 71af04ef7d..e8d84ea5ce 100644 --- a/crates/iceberg/src/inspect/manifests.rs +++ b/crates/iceberg/src/inspect/manifests.rs @@ -161,13 +161,7 @@ impl<'a> ManifestsTable<'a> { let mut partition_summaries = self.partition_summary_builder()?; if let Some(snapshot) = self.table.metadata().current_snapshot() { - let manifest_list = snapshot - .load_manifest_list( - self.table.file_io(), - &self.table.metadata_ref(), - self.table.encryption_manager(), - ) - .await?; + let manifest_list = self.table.manifest_list_loader(snapshot).load().await?; for manifest in manifest_list.entries() { content.append_value(manifest.content as i32); path.append_value(manifest.manifest_path.clone()); diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index e21543858b..d7f82097da 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -196,7 +196,7 @@ impl Snapshot { } /// Load manifest list. - pub async fn load_manifest_list( + pub(crate) async fn load_manifest_list( &self, file_io: &FileIO, table_metadata: &TableMetadata, diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index f09f529414..59c0e0fd61 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -27,7 +27,9 @@ use crate::io::FileIO; use crate::io::object_cache::ObjectCache; use crate::runtime::Runtime; use crate::scan::TableScanBuilder; -use crate::spec::{FormatVersion, SchemaRef, TableMetadata, TableMetadataRef}; +use crate::spec::{ + FormatVersion, ManifestList, SchemaRef, Snapshot, TableMetadata, TableMetadataRef, +}; use crate::{Error, ErrorKind, Result, TableIdent}; /// Builder to create table scan. @@ -300,12 +302,39 @@ impl Table { self.metadata.current_schema().clone() } + /// Creates a [`ManifestListLoader`] for the given snapshot. + pub fn manifest_list_loader<'a>(&'a self, snapshot: &'a Snapshot) -> ManifestListLoader<'a> { + ManifestListLoader { + snapshot, + file_io: &self.file_io, + table_metadata: &self.metadata, + encryption_manager: self.encryption_manager.as_deref(), + } + } + /// Create a reader for the table. pub fn reader_builder(&self) -> ArrowReaderBuilder { ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone()) } } +/// Loads a [`ManifestList`] for a snapshot. +pub struct ManifestListLoader<'a> { + snapshot: &'a Snapshot, + file_io: &'a FileIO, + table_metadata: &'a TableMetadataRef, + encryption_manager: Option<&'a EncryptionManager>, +} + +impl ManifestListLoader<'_> { + /// Loads and returns the [`ManifestList`] for this snapshot. + pub async fn load(&self) -> Result { + self.snapshot + .load_manifest_list(self.file_io, self.table_metadata, self.encryption_manager) + .await + } +} + /// `StaticTable` is a read-only table struct that can be created from a metadata file or from `TableMetaData` without a catalog. /// It can only be used to read metadata and for table scan. /// # Examples diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index e4cff058a7..1d263a1719 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -128,12 +128,10 @@ impl SnapshotProduceOperation for FastAppendOperation { return Ok(vec![]); }; - let manifest_list = snapshot - .load_manifest_list( - snapshot_produce.table.file_io(), - &snapshot_produce.table.metadata_ref(), - snapshot_produce.table.encryption_manager(), - ) + let manifest_list = snapshot_produce + .table + .manifest_list_loader(snapshot) + .load() .await?; Ok(manifest_list diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 16f0aad277..f1445a6415 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -605,13 +605,8 @@ mod test_row_lineage { assert_eq!(table.metadata().next_row_id(), 30); // Check written manifest for first_row_id - let manifest_list = table - .metadata() - .current_snapshot() - .unwrap() - .load_manifest_list(table.file_io(), table.metadata(), None) - .await - .unwrap(); + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_loader(snapshot).load().await.unwrap(); assert_eq!(manifest_list.entries().len(), 1); let manifest_file = &manifest_list.entries()[0]; @@ -633,13 +628,7 @@ mod test_row_lineage { assert_eq!(table.metadata().next_row_id(), 30 + 17 + 11); // Check written manifest for first_row_id - let manifest_list = table - .metadata() - .current_snapshot() - .unwrap() - .load_manifest_list(table.file_io(), table.metadata(), None) - .await - .unwrap(); + let manifest_list = table.manifest_list_loader(snapshot).load().await.unwrap(); assert_eq!(manifest_list.entries().len(), 2); let manifest_file = &manifest_list.entries()[1]; assert_eq!(manifest_file.first_row_id, Some(30)); diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index b9b8912525..4b6d05a189 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -172,12 +172,10 @@ impl<'a> SnapshotProducer<'a> { let mut referenced_files = Vec::new(); if let Some(current_snapshot) = self.table.metadata().current_snapshot() { - let manifest_list = current_snapshot - .load_manifest_list( - self.table.file_io(), - &self.table.metadata_ref(), - self.table.encryption_manager(), - ) + let manifest_list = self + .table + .manifest_list_loader(current_snapshot) + .load() .await?; for manifest_list_entry in manifest_list.entries() { let manifest = manifest_list_entry diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs index f5b75a788d..bb59e854f5 100644 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ b/crates/integrations/datafusion/src/physical_plan/commit.rs @@ -504,12 +504,9 @@ mod tests { let current_snapshot = updated_table.metadata().current_snapshot().unwrap(); // Load the manifest list to verify the data files were added - let manifest_list = current_snapshot - .load_manifest_list( - updated_table.file_io(), - updated_table.metadata(), - updated_table.encryption_manager(), - ) + let manifest_list = updated_table + .manifest_list_loader(current_snapshot) + .load() .await?; // There should be at least one manifest From f3d4a9b4522d7822cc2693163299f3ec810eced2 Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 21 May 2026 10:28:35 +0100 Subject: [PATCH 13/13] fix lock --- Cargo.lock | 723 +++-------------------------------------------------- 1 file changed, 41 insertions(+), 682 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 370fbf8a31..19ac987caa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cipher", "cpufeatures 0.2.17", ] @@ -49,7 +49,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "const-random", "getrandom 0.3.4", "once_cell", @@ -206,15 +206,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "approx" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" -dependencies = [ - "num-traits", -] - [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -1047,7 +1038,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if 1.0.4", + "cfg-if", "constant_time_eq", "cpufeatures 0.2.17", ] @@ -1126,17 +1117,6 @@ dependencies = [ "alloc-stdlib", ] -[[package]] -name = "bstr" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" -dependencies = [ - "memchr", - "regex-automata", - "serde", -] - [[package]] name = "bumpalo" version = "3.20.2" @@ -1218,12 +1198,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.4" @@ -1242,7 +1216,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures 0.3.0", "rand_core 0.10.0", ] @@ -1442,37 +1416,12 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "const-str" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" - -[[package]] -name = "const_panic" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" -dependencies = [ - "typewit", -] - [[package]] name = "constant_time_eq" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -1489,15 +1438,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "countio" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" -dependencies = [ - "futures-io", -] - [[package]] name = "cpufeatures" version = "0.2.17" @@ -1546,7 +1486,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -1720,7 +1660,7 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "crossbeam-utils", "hashbrown 0.14.5", "lock_api", @@ -2794,7 +2734,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "home", "windows-sys 0.48.0", ] @@ -2866,7 +2806,7 @@ version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "rustix", "windows-sys 0.59.0", ] @@ -3068,15 +3008,6 @@ dependencies = [ "slab", ] -[[package]] -name = "gearhash" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" -dependencies = [ - "cfg-if 0.1.10", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -3093,10 +3024,10 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -3106,7 +3037,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "js-sys", "libc", "r-efi 5.3.0", @@ -3120,14 +3051,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "cfg-if 1.0.4", - "js-sys", + "cfg-if", "libc", "r-efi 6.0.0", "rand_core 0.10.0", "wasip2", "wasip3", - "wasm-bindgen", ] [[package]] @@ -3140,26 +3069,6 @@ dependencies = [ "polyval", ] -[[package]] -name = "git-version" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" -dependencies = [ - "git-version-macro", -] - -[[package]] -name = "git-version-macro" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "glob" version = "0.3.3" @@ -3204,7 +3113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "bytemuck", - "cfg-if 1.0.4", + "cfg-if", "crunchy", "num-traits", "zerocopy", @@ -3259,12 +3168,6 @@ dependencies = [ "hashbrown 0.15.5", ] -[[package]] -name = "heapify" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" - [[package]] name = "heck" version = "0.5.0" @@ -3277,28 +3180,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hf-xet" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" -dependencies = [ - "async-trait", - "bytes", - "http 1.4.0", - "more-asserts", - "serde", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "uuid", - "xet-client", - "xet-core-structures", - "xet-data", - "xet-runtime", -] - [[package]] name = "hive_metastore" version = "0.2.0" @@ -3469,11 +3350,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.5.10", - "system-configuration", "tokio", "tower-service", "tracing", - "windows-registry", ] [[package]] @@ -3777,7 +3656,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", - "cfg-if 1.0.4", + "cfg-if", "futures", "iceberg", "iceberg_test_utils", @@ -4079,7 +3958,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "combine", "jni-macros", "jni-sys", @@ -4159,23 +4038,6 @@ dependencies = [ "simple_asn1", ] -[[package]] -name = "konst" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" -dependencies = [ - "const_panic", - "konst_proc_macros", - "typewit", -] - -[[package]] -name = "konst_proc_macros" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" - [[package]] name = "lazy_static" version = "1.5.0" @@ -4405,22 +4267,13 @@ dependencies = [ "serde", ] -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - [[package]] name = "md-5" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "digest", ] @@ -4503,7 +4356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "windows-sys 0.61.2", ] @@ -4513,7 +4366,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "downcast", "fragile", "mockall_derive", @@ -4527,7 +4380,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "proc-macro2", "quote", "syn", @@ -4578,12 +4431,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "more-asserts" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" - [[package]] name = "motore" version = "0.4.1" @@ -4661,7 +4508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags", - "cfg-if 1.0.4", + "cfg-if", "cfg_aliases", "libc", "memoffset", @@ -4674,20 +4521,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ "bitflags", - "cfg-if 1.0.4", + "cfg-if", "cfg_aliases", "libc", ] -[[package]] -name = "ntapi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" -dependencies = [ - "winapi", -] - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4791,34 +4629,6 @@ dependencies = [ "syn", ] -[[package]] -name = "objc2-core-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" -dependencies = [ - "bitflags", -] - -[[package]] -name = "objc2-io-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" -dependencies = [ - "libc", - "objc2-core-foundation", -] - -[[package]] -name = "objc2-system-configuration" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" -dependencies = [ - "objc2-core-foundation", -] - [[package]] name = "object" version = "0.37.3" @@ -4879,12 +4689,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "oneshot" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" - [[package]] name = "opaque-debug" version = "0.3.1" @@ -4906,7 +4710,6 @@ dependencies = [ "opendal-service-azdls", "opendal-service-fs", "opendal-service-gcs", - "opendal-service-hf", "opendal-service-oss", "opendal-service-s3", ] @@ -5046,23 +4849,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "opendal-service-hf" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2ab7a2a8a11dfe257ef4db5c0de798acbcd0d6429c37382dad2154bc06a388" -dependencies = [ - "bytes", - "hf-xet", - "http 1.4.0", - "log", - "opendal-core", - "percent-encoding", - "reqwest 0.13.3", - "serde", - "serde_json", -] - [[package]] name = "opendal-service-oss" version = "0.56.0" @@ -5143,15 +4929,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "os_str_bytes" -version = "6.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" -dependencies = [ - "memchr", -] - [[package]] name = "outref" version = "0.5.2" @@ -5186,7 +4963,7 @@ version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "libc", "redox_syscall 0.5.18", "smallvec", @@ -5448,7 +5225,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures 0.2.17", "opaque-debug", "universal-hash", @@ -5858,15 +5635,6 @@ dependencies = [ "syn", ] -[[package]] -name = "redb" -version = "3.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" -dependencies = [ - "libc", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -6152,8 +5920,6 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", - "serde", - "serde_json", "sync_wrapper", "tokio", "tokio-rustls", @@ -6168,20 +5934,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "reqwest-middleware" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" -dependencies = [ - "anyhow", - "async-trait", - "http 1.4.0", - "reqwest 0.13.3", - "thiserror 2.0.18", - "tower-service", -] - [[package]] name = "ring" version = "0.17.14" @@ -6189,7 +5941,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if 1.0.4", + "cfg-if", "getrandom 0.2.17", "libc", "untrusted 0.9.0", @@ -6273,7 +6025,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "glob", "proc-macro-crate", "proc-macro2", @@ -6291,7 +6043,7 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "ordered-multimap", ] @@ -6369,7 +6121,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation 0.10.1", + "core-foundation", "core-foundation-sys", "jni", "log", @@ -6415,7 +6167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ "bitflags", - "cfg-if 1.0.4", + "cfg-if", "clipboard-win", "fd-lock", "home", @@ -6436,12 +6188,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "safe-transmute" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" - [[package]] name = "salsa20" version = "0.10.2" @@ -6541,7 +6287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation 0.10.1", + "core-foundation", "core-foundation-sys", "libc", "security-framework-sys", @@ -6755,7 +6501,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures 0.2.17", "digest", ] @@ -6766,19 +6512,9 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures 0.2.17", "digest", - "sha2-asm", -] - -[[package]] -name = "sha2-asm" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" -dependencies = [ - "cc", ] [[package]] @@ -6790,17 +6526,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shellexpand" -version = "3.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" -dependencies = [ - "bstr", - "dirs", - "os_str_bytes", -] - [[package]] name = "shlex" version = "1.3.0" @@ -6920,7 +6645,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5661364b38abad49cf1ade6631fcc35d2ccf882a7d68616b4228b7717feb5fba" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -6931,7 +6656,7 @@ checksum = "0275f9f2f07d47556fe60c2759da8bc4be6083b047b491b2d476aa0bfa558eb1" dependencies = [ "bumpalo", "bytes", - "cfg-if 1.0.4", + "cfg-if", "faststr", "itoa", "ref-cast", @@ -6949,7 +6674,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9f944718c33623919878cf74b4c9361eb3024f635733922b26722b14cd3f8cc" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -7141,7 +6866,7 @@ dependencies = [ "stringprep", "thiserror 2.0.18", "tracing", - "whoami 1.6.1", + "whoami", ] [[package]] @@ -7178,7 +6903,7 @@ dependencies = [ "stringprep", "thiserror 2.0.18", "tracing", - "whoami 1.6.1", + "whoami", ] [[package]] @@ -7218,28 +6943,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" dependencies = [ "cc", - "cfg-if 1.0.4", + "cfg-if", "libc", "psm", "windows-sys 0.59.0", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "statrs" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" -dependencies = [ - "approx", - "num-traits", -] - [[package]] name = "stringprep" version = "0.1.5" @@ -7319,12 +7028,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "symlink" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" - [[package]] name = "syn" version = "2.0.117" @@ -7356,41 +7059,6 @@ dependencies = [ "syn", ] -[[package]] -name = "sysinfo" -version = "0.38.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" -dependencies = [ - "libc", - "memchr", - "ntapi", - "objc2-core-foundation", - "objc2-io-kit", - "windows", -] - -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tagptr" version = "0.2.0" @@ -7462,7 +7130,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -7569,17 +7237,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-retry" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f644c762e9d396831ae2f8935c954b0d758c4532e924bead0f666d0c1c8640" -dependencies = [ - "pin-project-lite", - "rand 0.10.1", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" @@ -7743,19 +7400,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-appender" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" -dependencies = [ - "crossbeam-channel", - "symlink", - "thiserror 2.0.18", - "time", - "tracing-subscriber", -] - [[package]] name = "tracing-attributes" version = "0.1.31" @@ -7788,35 +7432,18 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-serde" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" -dependencies = [ - "serde", - "tracing-core", -] - [[package]] name = "tracing-subscriber" version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ - "matchers", "nu-ansi-term", - "once_cell", - "regex-automata", - "serde", - "serde_json", "sharded-slab", "smallvec", "thread_local", - "tracing", "tracing-core", "tracing-log", - "tracing-serde", ] [[package]] @@ -7887,12 +7514,6 @@ dependencies = [ "syn", ] -[[package]] -name = "typewit" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc19094686c694eb41b3b99dcc2f2975d4b078512fa22ae6c63f7ca318bdcff7" - [[package]] name = "typify" version = "0.5.0" @@ -8172,15 +7793,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasi" -version = "0.14.7+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" -dependencies = [ - "wasip2", -] - [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -8205,22 +7817,13 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" -[[package]] -name = "wasite" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" -dependencies = [ - "wasi 0.14.7+wasi-0.2.4", -] - [[package]] name = "wasm-bindgen" version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", @@ -8233,7 +7836,7 @@ version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "futures-util", "js-sys", "once_cell", @@ -8387,38 +7990,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", - "wasite 0.1.0", -] - -[[package]] -name = "whoami" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" -dependencies = [ - "libc", - "libredox", - "objc2-system-configuration", - "wasite 1.0.2", - "web-sys", + "wasite", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.11" @@ -8428,33 +8002,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" -dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" -dependencies = [ - "windows-core", -] - [[package]] name = "windows-core" version = "0.62.2" @@ -8468,17 +8015,6 @@ dependencies = [ "windows-strings", ] -[[package]] -name = "windows-future" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" -dependencies = [ - "windows-core", - "windows-link", - "windows-threading", -] - [[package]] name = "windows-implement" version = "0.60.2" @@ -8507,27 +8043,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-numerics" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" -dependencies = [ - "windows-core", - "windows-link", -] - -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -8639,15 +8154,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] -[[package]] -name = "windows-threading" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" -dependencies = [ - "windows-link", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -8908,153 +8414,6 @@ dependencies = [ "rustix", ] -[[package]] -name = "xet-client" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" -dependencies = [ - "anyhow", - "async-trait", - "base64", - "bytes", - "clap", - "crc32fast", - "futures", - "http 1.4.0", - "hyper", - "lazy_static", - "more-asserts", - "rand 0.10.1", - "redb", - "reqwest 0.13.3", - "reqwest-middleware", - "serde", - "serde_json", - "serde_repr", - "statrs", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-retry", - "tracing", - "tracing-subscriber", - "url", - "urlencoding", - "web-time", - "xet-core-structures", - "xet-runtime", -] - -[[package]] -name = "xet-core-structures" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" -dependencies = [ - "async-trait", - "base64", - "blake3", - "bytemuck", - "bytes", - "clap", - "countio", - "csv", - "futures", - "futures-util", - "getrandom 0.4.2", - "heapify", - "itertools 0.14.0", - "lazy_static", - "lz4_flex", - "more-asserts", - "rand 0.10.1", - "regex", - "safe-transmute", - "serde", - "static_assertions", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "uuid", - "web-time", - "xet-runtime", -] - -[[package]] -name = "xet-data" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" -dependencies = [ - "anyhow", - "async-trait", - "bytes", - "chrono", - "clap", - "gearhash", - "http 1.4.0", - "itertools 0.14.0", - "lazy_static", - "more-asserts", - "rand 0.10.1", - "serde", - "serde_json", - "sha2", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "url", - "uuid", - "walkdir", - "xet-client", - "xet-core-structures", - "xet-runtime", -] - -[[package]] -name = "xet-runtime" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" -dependencies = [ - "anyhow", - "async-trait", - "bytes", - "chrono", - "colored", - "const-str", - "ctor", - "dirs", - "futures", - "git-version", - "humantime", - "konst", - "lazy_static", - "libc", - "more-asserts", - "oneshot", - "pin-project", - "rand 0.10.1", - "reqwest 0.13.3", - "serde", - "serde_json", - "shellexpand", - "sysinfo", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "tracing-appender", - "tracing-subscriber", - "whoami 2.1.1", - "winapi", -] - [[package]] name = "xmlparser" version = "0.13.6"