diff --git a/Cargo.lock b/Cargo.lock index 783149532..dea1cabd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,13 +47,13 @@ dependencies = [ "actix-utils", "base64", "bitflags 2.10.0", - "brotli 8.0.2", + "brotli", "bytes", "bytestring", "derive_more 2.1.1", "encoding_rs", "flate2", - "foldhash", + "foldhash 0.1.5", "futures-core", "h2 0.3.27", "http 0.2.12", @@ -187,7 +187,7 @@ dependencies = [ "cookie 0.16.2", "derive_more 2.1.1", "encoding_rs", - "foldhash", + "foldhash 0.1.5", "futures-core", "futures-util", "impl-more 0.1.9", @@ -492,9 +492,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" +checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" dependencies = [ "arrow-arith", "arrow-array", @@ -513,9 +513,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" +checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" dependencies = [ "arrow-array", "arrow-buffer", @@ -527,9 +527,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" +checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" dependencies = [ "ahash", "arrow-buffer", @@ -546,9 +546,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626" +checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" dependencies = [ "bytes", "half", @@ -558,9 +558,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271" +checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" dependencies = [ "arrow-array", "arrow-buffer", @@ -580,9 +580,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b" +checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" dependencies = [ "arrow-array", "arrow-cast", @@ -595,9 +595,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" +checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" dependencies = [ "arrow-buffer", "arrow-schema", @@ -608,9 +608,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5f57c3d39d1b1b7c1376a772ea86a131e7da310aed54ebea9363124bb885e3" +checksum = "302b2e036335f3f04d65dad3f74ff1f2aae6dc671d6aa04dc6b61193761e16fb" dependencies = [ "arrow-array", "arrow-buffer", @@ -628,9 +628,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3" +checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" dependencies = [ "arrow-array", "arrow-buffer", @@ -644,9 +644,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0" +checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" dependencies = [ "arrow-array", "arrow-buffer", @@ -668,9 +668,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" +checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" dependencies = [ "arrow-array", "arrow-buffer", @@ -681,9 +681,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" +checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" dependencies = [ "arrow-array", "arrow-buffer", @@ -694,9 +694,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0" +checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" dependencies = [ "serde", "serde_core", @@ -705,9 +705,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" +checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" dependencies = [ "ahash", "arrow-array", @@ -719,9 +719,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" +checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" dependencies = [ "arrow-array", "arrow-buffer", @@ -736,20 +736,14 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.19" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ - "brotli 7.0.0", - "bzip2 0.5.2", - "flate2", - "futures-core", - "memchr", + "compression-codecs", + "compression-core", "pin-project-lite", "tokio", - "xz2", - "zstd", - "zstd-safe", ] [[package]] @@ -957,17 +951,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "brotli" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 4.0.3", -] - [[package]] name = "brotli" version = "8.0.2" @@ -976,17 +959,7 @@ checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor 5.0.0", -] - -[[package]] -name = "brotli-decompressor" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "brotli-decompressor", ] [[package]] @@ -1026,15 +999,6 @@ dependencies = [ "bytes", ] -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - [[package]] name = "bzip2" version = "0.6.1" @@ -1044,16 +1008,6 @@ dependencies = [ "libbz2-rs-sys", ] -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "camino" version = "1.2.2" @@ -1143,9 +1097,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -1247,6 +1201,28 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "const-hex" version = "1.17.0" @@ -1532,15 +1508,15 @@ checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "datafusion" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" +checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" dependencies = [ "arrow", "arrow-schema", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1570,27 +1546,26 @@ dependencies = [ "flate2", "futures", "itertools 0.14.0", + "liblzma", "log", "object_store", "parking_lot", "parquet", "rand 0.9.4", "regex", - "rstest 0.26.1", "sqlparser", "tempfile", "tokio", "url", "uuid", - "xz2", "zstd", ] [[package]] name = "datafusion-catalog" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" +checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" dependencies = [ "arrow", "async-trait", @@ -1613,9 +1588,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" +checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ "arrow", "async-trait", @@ -1632,22 +1607,22 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "tokio", ] [[package]] name = "datafusion-common" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" +checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" dependencies = [ "ahash", "arrow", "arrow-ipc", "chrono", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap", + "itertools 0.14.0", "libc", "log", "object_store", @@ -1661,9 +1636,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95" +checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" dependencies = [ "futures", "log", @@ -1672,15 +1647,15 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" +checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -1695,21 +1670,21 @@ dependencies = [ "futures", "glob", "itertools 0.14.0", + "liblzma", "log", "object_store", "rand 0.9.4", "tokio", "tokio-util", "url", - "xz2", "zstd", ] [[package]] name = "datafusion-datasource-arrow" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" +checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" dependencies = [ "arrow", "arrow-ipc", @@ -1731,9 +1706,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" +checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" dependencies = [ "arrow", "async-trait", @@ -1754,9 +1729,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" +checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" dependencies = [ "arrow", "async-trait", @@ -1771,14 +1746,16 @@ dependencies = [ "datafusion-session", "futures", "object_store", + "serde_json", "tokio", + "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9" +checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" dependencies = [ "arrow", "async-trait", @@ -1806,21 +1783,24 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" +checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" [[package]] name = "datafusion-execution" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" +checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" dependencies = [ "arrow", + "arrow-buffer", "async-trait", + "chrono", "dashmap", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", "object_store", @@ -1832,9 +1812,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" +checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" dependencies = [ "arrow", "async-trait", @@ -1855,9 +1835,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" +checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ "arrow", "datafusion-common", @@ -1868,9 +1848,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" +checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" dependencies = [ "arrow", "arrow-buffer", @@ -1878,6 +1858,7 @@ dependencies = [ "blake2", "blake3", "chrono", + "chrono-tz", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -1888,6 +1869,7 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", + "memchr", "num-traits", "rand 0.9.4", "regex", @@ -1898,9 +1880,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" +checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash", "arrow", @@ -1914,14 +1896,15 @@ dependencies = [ "datafusion-physical-expr-common", "half", "log", + "num-traits", "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" +checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash", "arrow", @@ -1932,9 +1915,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" +checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" dependencies = [ "arrow", "arrow-ord", @@ -1948,16 +1931,18 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", + "hashbrown 0.16.1", "itertools 0.14.0", + "itoa", "log", "paste", ] [[package]] name = "datafusion-functions-table" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" +checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ "arrow", "async-trait", @@ -1971,9 +1956,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" +checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ "arrow", "datafusion-common", @@ -1989,9 +1974,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" +checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1999,9 +1984,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" +checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ "datafusion-doc", "quote", @@ -2010,9 +1995,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" +checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ "arrow", "chrono", @@ -2030,9 +2015,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" +checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash", "arrow", @@ -2042,19 +2027,21 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap", "itertools 0.14.0", "parking_lot", "paste", "petgraph", + "recursive", + "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" +checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ "arrow", "datafusion-common", @@ -2067,23 +2054,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" +checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" dependencies = [ "ahash", "arrow", + "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.14.5", + "hashbrown 0.16.1", + "indexmap", "itertools 0.14.0", + "parking_lot", ] [[package]] name = "datafusion-physical-optimizer" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" +checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ "arrow", "datafusion-common", @@ -2100,30 +2090,31 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" +checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" dependencies = [ "ahash", "arrow", "arrow-ord", "arrow-schema", "async-trait", - "chrono", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap", "itertools 0.14.0", "log", + "num-traits", "parking_lot", "pin-project-lite", "tokio", @@ -2131,9 +2122,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" +checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ "arrow", "datafusion-common", @@ -2148,9 +2139,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" +checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" dependencies = [ "async-trait", "datafusion-common", @@ -2162,15 +2153,16 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "51.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" +checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" dependencies = [ "arrow", "bigdecimal", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-functions-nested", "indexmap", "log", "recursive", @@ -2417,13 +2409,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "libz-rs-sys", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -2438,6 +2430,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -2681,10 +2679,6 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] [[package]] name = "hashbrown" @@ -2692,7 +2686,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -2700,6 +2694,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -3236,25 +3235,36 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.178" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] -name = "libm" -version = "0.2.15" +name = "liblzma" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" +dependencies = [ + "liblzma-sys", +] [[package]] -name = "libz-rs-sys" -version = "0.5.5" +name = "liblzma-sys" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" +checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6" dependencies = [ - "zlib-rs", + "cc", + "libc", + "pkg-config", ] +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "libz-sys" version = "1.1.23" @@ -3325,24 +3335,13 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" dependencies = [ "twox-hash", ] -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "mach" version = "0.3.2" @@ -3379,9 +3378,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "mime" @@ -3559,16 +3558,18 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.4" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1be0c6c22ec0817cdc77d3842f721a17fd30ab6965001415b5402a74e6b740" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", "base64", "bytes", "chrono", "form_urlencoded", - "futures", + "futures-channel", + "futures-core", + "futures-util", "http 1.4.0", "http-body-util", "httparse", @@ -3579,10 +3580,10 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.9.4", + "rand 0.10.1", "reqwest", "ring", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -3829,20 +3830,19 @@ dependencies = [ [[package]] name = "parquet" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89" +checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" dependencies = [ "ahash", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", "arrow-select", "base64", - "brotli 8.0.2", + "brotli", "bytes", "chrono", "flate2", @@ -3932,7 +3932,7 @@ dependencies = [ "regex", "relative-path", "reqwest", - "rstest 0.23.0", + "rstest", "rustls", "rustls-pemfile", "sasl2-sys", @@ -4315,9 +4315,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" dependencies = [ "memchr", "serde", @@ -4380,9 +4380,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.42" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -4607,9 +4607,9 @@ checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "relative-path" @@ -4686,21 +4686,10 @@ checksum = "0a2c585be59b6b5dd66a9d2084aa1d8bd52fbdb806eafdeffb52791147862035" dependencies = [ "futures", "futures-timer", - "rstest_macros 0.23.0", + "rstest_macros", "rustc_version", ] -[[package]] -name = "rstest" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" -dependencies = [ - "futures-timer", - "futures-util", - "rstest_macros 0.26.1", -] - [[package]] name = "rstest_macros" version = "0.23.0" @@ -4719,24 +4708,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "rstest_macros" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" -dependencies = [ - "cfg-if", - "glob", - "proc-macro-crate", - "proc-macro2", - "quote", - "regex", - "relative-path", - "rustc_version", - "syn", - "unicode-ident", -] - [[package]] name = "rustc-demangle" version = "0.1.26" @@ -5184,9 +5155,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.59.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" dependencies = [ "log", "recursive", @@ -5195,9 +5166,9 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", @@ -5259,9 +5230,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.111" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -5508,6 +5479,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -5926,11 +5898,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.19.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.2", "js-sys", "wasm-bindgen", ] @@ -6646,15 +6618,6 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yoke" version = "0.8.1" @@ -6777,9 +6740,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index 756a6889b..bc55a6011 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,24 +9,24 @@ build = "build.rs" [dependencies] # Arrow and DataFusion ecosystem -arrow = "57.1.0" -arrow-array = "57.1.0" -arrow-flight = { version = "57.1.0", features = [ +arrow = "58.0.0" +arrow-array = "58.0.0" +arrow-flight = { version = "58.0.0", features = [ "tls-aws-lc", "tls-native-roots", ] } -arrow-ipc = { version = "57.1.0", features = ["zstd"] } -arrow-json = "57.1.0" -arrow-schema = { version = "57.1.0", features = ["serde"] } -arrow-select = "57.1.0" -datafusion = "51.0.0" -object_store = { version = "0.12.4", features = [ +arrow-ipc = { version = "58.0.0", features = ["zstd"] } +arrow-json = "58.0.0" +arrow-schema = { version = "58.0.0", features = ["serde"] } +arrow-select = "58.0.0" +datafusion = "53.0.0" +object_store = { version = "0.13.1", features = [ "cloud", "aws", "azure", "gcp", ] } -parquet = "57.1.0" +parquet = "58.0.0" # Web server and HTTP-related actix-cors = "0.7.0" @@ -200,7 +200,7 @@ anyhow = "1.0" [dev-dependencies] rstest = "0.23.0" -arrow = "57.1.0" +arrow = "58.0.0" temp-dir = "0.1.14" [package.metadata.parseable_ui] diff --git a/src/hottier.rs b/src/hottier.rs index 88971cd87..c472f91c5 100644 --- a/src/hottier.rs +++ b/src/hottier.rs @@ -35,7 +35,7 @@ use chrono::NaiveDate; use clokwerk::{AsyncScheduler, Interval, Job}; use futures::{StreamExt, TryStreamExt, stream::FuturesUnordered}; use futures_util::TryFutureExt; -use object_store::{ObjectStore, local::LocalFileSystem}; +use object_store::{ObjectStoreExt, local::LocalFileSystem}; use once_cell::sync::OnceCell; use parquet::errors::ParquetError; use relative_path::RelativePathBuf; diff --git a/src/parseable/streams.rs b/src/parseable/streams.rs index 97659e497..b7aec7ef2 100644 --- a/src/parseable/streams.rs +++ b/src/parseable/streams.rs @@ -582,7 +582,7 @@ impl Stream { let time_partition_idx = merged_schema.index_of(time_partition_field).unwrap_or(0); let mut props = WriterProperties::builder() - .set_max_row_group_size(self.options.row_group_size) + .set_max_row_group_row_count(Some(self.options.row_group_size)) .set_compression(self.options.parquet_compression.into()) .set_column_encoding( ColumnPath::new(vec![time_partition_field.to_string()]), diff --git a/src/query/stream_schema_provider.rs b/src/query/stream_schema_provider.rs index 28d3aed94..6d4005972 100644 --- a/src/query/stream_schema_provider.rs +++ b/src/query/stream_schema_provider.rs @@ -150,22 +150,21 @@ impl StandardTableProvider { // parquet file source, default table parquet options let file_source = if let Some(phyiscal_expr) = filters { - ParquetSource::default().with_predicate(phyiscal_expr) + ParquetSource::new(self.schema.clone()).with_predicate(phyiscal_expr) } else { - ParquetSource::default() + ParquetSource::new(self.schema.clone()) }; - let mut conf_builder = - FileScanConfigBuilder::new(object_store_url, self.schema.clone(), file_source.into()) - .with_statistics(statistics) - .with_batch_size(Some(20000)) - .with_constraints(Constraints::default()) - .with_file_groups(file_groups) - .with_output_ordering(vec![LexOrdering::new([sort_expr]).unwrap()]); + let mut conf_builder = FileScanConfigBuilder::new(object_store_url, file_source.into()) + .with_statistics(statistics) + .with_batch_size(Some(20000)) + .with_constraints(Constraints::default()) + .with_file_groups(file_groups) + .with_output_ordering(vec![LexOrdering::new([sort_expr]).unwrap()]); // Set projection if provided if let Some(proj_indices) = projection { - conf_builder = conf_builder.with_projection_indices(Some(proj_indices.clone())); + conf_builder = conf_builder.with_projection_indices(Some(proj_indices.clone()))?; } // Set limit if provided @@ -423,6 +422,7 @@ impl StandardTableProvider { min_value: Precision::Exact(min), distinct_count: Precision::Absent, sum_value: Precision::Absent, + byte_size: Precision::Absent, }) .unwrap_or_default() }) diff --git a/src/storage/azure_blob.rs b/src/storage/azure_blob.rs index 79415b073..62abad0db 100644 --- a/src/storage/azure_blob.rs +++ b/src/storage/azure_blob.rs @@ -30,7 +30,8 @@ use datafusion::{ }; use futures::{StreamExt, TryStreamExt, stream::FuturesUnordered}; use object_store::{ - BackoffConfig, ClientOptions, ListResult, ObjectMeta, ObjectStore, PutPayload, RetryConfig, + BackoffConfig, ClientOptions, ListResult, ObjectMeta, ObjectStore, ObjectStoreExt, PutPayload, + RetryConfig, azure::{MicrosoftAzure, MicrosoftAzureBuilder}, buffered::BufReader, limit::LimitStore, diff --git a/src/storage/gcs.rs b/src/storage/gcs.rs index 96dabc1c5..019802f24 100644 --- a/src/storage/gcs.rs +++ b/src/storage/gcs.rs @@ -38,7 +38,8 @@ use datafusion::{ }; use futures::{StreamExt, TryStreamExt, stream::FuturesUnordered}; use object_store::{ - BackoffConfig, ClientOptions, ListResult, ObjectMeta, ObjectStore, PutPayload, RetryConfig, + BackoffConfig, ClientOptions, ListResult, ObjectMeta, ObjectStore, ObjectStoreExt, PutPayload, + RetryConfig, buffered::BufReader, gcp::{GoogleCloudStorage, GoogleCloudStorageBuilder}, limit::LimitStore, diff --git a/src/storage/metrics_layer.rs b/src/storage/metrics_layer.rs index 45aabcc5f..ac82e43d6 100644 --- a/src/storage/metrics_layer.rs +++ b/src/storage/metrics_layer.rs @@ -26,9 +26,9 @@ use async_trait::async_trait; use bytes::Bytes; use futures_util::{Stream, StreamExt, stream::BoxStream}; use object_store::{ - GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, - PutMultipartOptions, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult, - path::Path, + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, + Result as ObjectStoreResult, path::Path, }; use crate::metrics::STORAGE_REQUEST_RESPONSE_TIME; @@ -85,31 +85,10 @@ impl std::fmt::Display for MetricLayer { #[async_trait] impl ObjectStore for MetricLayer { - /// PutPayload.from_bytes(bytes) - async fn put( - &self, - location: &Path, - bytes: PutPayload, /* PutPayload */ - ) -> ObjectStoreResult { - let time = time::Instant::now(); - let put_result = self.inner.put(location, bytes).await; - let elapsed = time.elapsed().as_secs_f64(); - - let status = match &put_result { - Ok(_) => "200", - Err(err) => error_to_status_code(err), - }; - - STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "PUT", status]) - .observe(elapsed); - put_result - } - async fn put_opts( &self, location: &Path, - payload: PutPayload, /* PutPayload */ + payload: PutPayload, opts: PutOptions, ) -> ObjectStoreResult { let time = time::Instant::now(); @@ -122,27 +101,11 @@ impl ObjectStore for MetricLayer { }; STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "PUT_OPTS", status]) + .with_label_values(&[&self.provider, "PUT", status]) .observe(elapsed); put_result } - // // ! removed in object_store 0.10.0 - // async fn abort_multipart( - // &self, - // location: &Path, - // multipart_id: &MultipartId, - // ) -> object_store::Result<()> { - // let time = time::Instant::now(); - // let elapsed = time.elapsed().as_secs_f64(); - // self.inner.abort_multipart(location, multipart_id).await?; - // STORAGE_REQUEST_RESPONSE_TIME - // .with_label_values(&["PUT_MULTIPART_ABORT", "200"]) - // .observe(elapsed); - // Ok(()) - // } - - /* Keep for easier migration to object_store 0.10.0 */ async fn put_multipart_opts( &self, location: &Path, @@ -157,46 +120,12 @@ impl ObjectStore for MetricLayer { Err(err) => error_to_status_code(err), }; - STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "PUT_MULTIPART_OPTS", status]) - .observe(elapsed); - result - } - - // todo completly tracking multipart upload - async fn put_multipart(&self, location: &Path) -> ObjectStoreResult> /* ObjectStoreResult<(MultipartId, Box)> */ - { - let time = time::Instant::now(); - let result = self.inner.put_multipart(location).await; - let elapsed = time.elapsed().as_secs_f64(); - - let status = match &result { - Ok(_) => "200", - Err(err) => error_to_status_code(err), - }; - STORAGE_REQUEST_RESPONSE_TIME .with_label_values(&[&self.provider, "PUT_MULTIPART", status]) .observe(elapsed); result } - async fn get(&self, location: &Path) -> ObjectStoreResult { - let time = time::Instant::now(); - let get_result = self.inner.get(location).await; - let elapsed = time.elapsed().as_secs_f64(); - - let status = match &get_result { - Ok(_) => "200", - Err(err) => error_to_status_code(err), - }; - - STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "GET", status]) - .observe(elapsed); - get_result - } - async fn get_opts(&self, location: &Path, options: GetOptions) -> ObjectStoreResult { let time = time::Instant::now(); let result = self.inner.get_opts(location, options).await; @@ -208,23 +137,7 @@ impl ObjectStore for MetricLayer { }; STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "GET_OPTS", status]) - .observe(elapsed); - result - } - - async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { - let time = time::Instant::now(); - let result = self.inner.get_range(location, range).await; - let elapsed = time.elapsed().as_secs_f64(); - - let status = match &result { - Ok(_) => "200", - Err(err) => error_to_status_code(err), - }; - - STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "GET_RANGE", status]) + .with_label_values(&[&self.provider, "GET", status]) .observe(elapsed); result } @@ -249,42 +162,10 @@ impl ObjectStore for MetricLayer { result } - async fn head(&self, location: &Path) -> ObjectStoreResult { - let time = time::Instant::now(); - let result = self.inner.head(location).await; - let elapsed = time.elapsed().as_secs_f64(); - - let status = match &result { - Ok(_) => "200", - Err(err) => error_to_status_code(err), - }; - - STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "HEAD", status]) - .observe(elapsed); - result - } - - async fn delete(&self, location: &Path) -> ObjectStoreResult<()> { - let time = time::Instant::now(); - let result = self.inner.delete(location).await; - let elapsed = time.elapsed().as_secs_f64(); - - let status = match &result { - Ok(_) => "200", - Err(err) => error_to_status_code(err), - }; - - STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "DELETE", status]) - .observe(elapsed); - result - } - - fn delete_stream<'a>( - &'a self, - locations: BoxStream<'a, ObjectStoreResult>, - ) -> BoxStream<'a, ObjectStoreResult> { + fn delete_stream( + &self, + locations: BoxStream<'static, ObjectStoreResult>, + ) -> BoxStream<'static, ObjectStoreResult> { self.inner.delete_stream(locations) } @@ -335,9 +216,14 @@ impl ObjectStore for MetricLayer { result } - async fn copy(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: CopyOptions, + ) -> ObjectStoreResult<()> { let time = time::Instant::now(); - let result = self.inner.copy(from, to).await; + let result = self.inner.copy_opts(from, to, options).await; let elapsed = time.elapsed().as_secs_f64(); let status = match &result { @@ -351,9 +237,14 @@ impl ObjectStore for MetricLayer { result } - async fn rename(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { + async fn rename_opts( + &self, + from: &Path, + to: &Path, + options: RenameOptions, + ) -> ObjectStoreResult<()> { let time = time::Instant::now(); - let result = self.inner.rename(from, to).await; + let result = self.inner.rename_opts(from, to, options).await; let elapsed = time.elapsed().as_secs_f64(); let status = match &result { @@ -366,38 +257,6 @@ impl ObjectStore for MetricLayer { .observe(elapsed); result } - - async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { - let time = time::Instant::now(); - let result = self.inner.copy_if_not_exists(from, to).await; - let elapsed = time.elapsed().as_secs_f64(); - - let status = match &result { - Ok(_) => "200", - Err(err) => error_to_status_code(err), - }; - - STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "COPY_IF", status]) - .observe(elapsed); - result - } - - async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { - let time = time::Instant::now(); - let result = self.inner.rename_if_not_exists(from, to).await; - let elapsed = time.elapsed().as_secs_f64(); - - let status = match &result { - Ok(_) => "200", - Err(err) => error_to_status_code(err), - }; - - STORAGE_REQUEST_RESPONSE_TIME - .with_label_values(&[&self.provider, "RENAME_IF", status]) - .observe(elapsed); - result - } } struct StreamMetricWrapper<'a, T> { diff --git a/src/storage/s3.rs b/src/storage/s3.rs index ac9964bad..169d3c508 100644 --- a/src/storage/s3.rs +++ b/src/storage/s3.rs @@ -32,7 +32,8 @@ use datafusion::{ }; use futures::{StreamExt, TryStreamExt, stream::FuturesUnordered}; use object_store::{ - BackoffConfig, ClientOptions, ListResult, ObjectMeta, ObjectStore, PutPayload, RetryConfig, + BackoffConfig, ClientOptions, ListResult, ObjectMeta, ObjectStore, ObjectStoreExt, PutPayload, + RetryConfig, aws::{AmazonS3, AmazonS3Builder, AmazonS3ConfigKey, Checksum}, buffered::BufReader, limit::LimitStore,