diff --git a/CHANGELOG.md b/CHANGELOG.md index 4868a383a1..51948ec6cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## 0.7.0 (unreleased) -- Updated Polars to 0.33.2 +- Updated Polars to 0.34.1 - Added `!` for `Expr` - Added `Config` module - Added `none?` method to `Series` diff --git a/Cargo.lock b/Cargo.lock index a1762b3045..79308bb025 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -16,14 +25,15 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" dependencies = [ "cfg-if", "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -96,42 +106,6 @@ dependencies = [ "serde", ] -[[package]] -name = "arrow2" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963fef509b757bcbbf9e5ffa23bcb345614d99f4f6f531f97417b27b8604d389" -dependencies = [ - "ahash", - "arrow-format", - "avro-schema", - "base64", - "bytemuck", - "chrono", - "chrono-tz", - "dyn-clone", - "either", - "ethnum", - "fallible-streaming-iterator", - "foreign_vec", - "futures", - "getrandom", - "hash_hasher", - "hashbrown 0.14.0", - "lexical-core", - "lz4", - "multiversion", - "num-traits", - "parquet2", - "regex", - "regex-syntax", - "rustc_version", - "simdutf8", - "streaming-iterator", - "strength_reduce", - "zstd", -] - [[package]] name = "async-stream" version = "0.3.5" @@ -194,6 +168,21 @@ dependencies = [ "snap", ] +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.21.4" @@ -656,6 +645,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + [[package]] name = "glob" version = "0.3.1" @@ -672,12 +667,6 @@ dependencies = [ "serde", ] -[[package]] -name = "hash_hasher" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" - [[package]] name = "hashbrown" version = "0.13.2" @@ -1145,6 +1134,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -1199,7 +1197,8 @@ dependencies = [ "seq-macro", "snap", "streaming-decompression", - "zstd", + "xxhash-rust", + "zstd 0.12.4", ] [[package]] @@ -1217,6 +1216,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + [[package]] name = "phf" version = "0.11.2" @@ -1292,7 +1297,7 @@ dependencies = [ "jemallocator", "magnus", "mimalloc", - "polars 0.33.2", + "polars 0.34.1", "polars-core", "serde_json", "smartstring", @@ -1300,9 +1305,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3030de163b9ff2c9dac9a12dcb9be25cc0f2bc7c8e7cd2e4b2592ebed458ce6a" +checksum = "ed842e5aad34e81b6904f68fa57196341bb7eeb84e04f5f71a6d4f50609a1fc1" dependencies = [ "getrandom", "polars-core", @@ -1316,33 +1321,49 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35cd38a64fb389fd990e4efd433a36331c995c981d353bfef83b5de4d87f1828" +checksum = "fc02e90fbdc3dccc0fb61becb6c63f57a2660f715bbd220e6f689ed5691915b6" dependencies = [ - "arrow2", + "ahash", + "arrow-format", "atoi", + "avro-schema", + "base64", + "bytemuck", "chrono", "chrono-tz", + "dyn-clone", + "either", "ethnum", + "fallible-streaming-iterator", + "foreign_vec", + "futures", + "getrandom", "hashbrown 0.14.0", + "lexical-core", + "lz4", "multiversion", "num-traits", + "parquet2", "polars-error", + "rustc_version", "serde", - "thiserror", - "version_check", + "simdutf8", + "streaming-iterator", + "strength_reduce", + "zstd 0.13.0", ] [[package]] name = "polars-core" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08367c014c07fa8f141680e024f926cab3a1fe839605a8fcf2223647eb45ca71" +checksum = "e3fc406dbfef1e83f651b34e5a5525b63303f062b74cc8816a36d6432e487c3d" dependencies = [ "ahash", - "arrow2", "bitflags 2.4.0", + "bytemuck", "chrono", "chrono-tz", "comfy-table", @@ -1370,34 +1391,40 @@ dependencies = [ [[package]] name = "polars-error" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b20a09651a299979354945819dc2ce017964b80b916954e9d2ce39002a5f949" +checksum = "cc9233939f0cfaf8bf91c18044d04f88cdc152448b142e7fd55493e84adfe6b1" dependencies = [ - "arrow2", + "arrow-format", + "avro-schema", + "parquet2", "regex", + "simdutf8", "thiserror", ] [[package]] name = "polars-io" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88cf4a89c18a90ac20dfbcdfd19ab50ad4ac5a76fc7bb775d3c28bb738cf1f34" +checksum = "6ae894c0ce199b2a7a02cfb59bdd70e0a21baac2923ca4fe106b4afdfe2d65b5" dependencies = [ "ahash", - "arrow2", + "async-trait", "bytes", "chrono", "chrono-tz", "fast-float", + "futures", "home", + "itoa", "lexical", "lexical-core", "memchr", "memmap2", "num-traits", "once_cell", + "percent-encoding", "polars-arrow", "polars-core", "polars-error", @@ -1406,35 +1433,42 @@ dependencies = [ "polars-utils", "rayon", "regex", + "ryu", "serde", "serde_json", "simd-json", "simdutf8", + "smartstring", + "tokio", + "tokio-util", ] [[package]] name = "polars-json" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6d5666176d681706aef5a06a57597c83391948b3d958f9fbe9b4cf016527eb8" +checksum = "45f27c57f7a7c0c2156a116388710f756804d6b523afde239b394786d6bb7e4e" dependencies = [ "ahash", - "arrow2", + "chrono", "fallible-streaming-iterator", "hashbrown 0.14.0", "indexmap", + "itoa", "num-traits", "polars-arrow", "polars-error", "polars-utils", + "ryu", "simd-json", + "streaming-iterator", ] [[package]] name = "polars-lazy" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5110eab438848c981cc5f541fbc5b21bb263fd707000b4715233074fb2630fcf" +checksum = "45f1bbb176e3fcd1130a68307727def479b48b1b6082cdc67fb605e45218a065" dependencies = [ "ahash", "bitflags 2.4.0", @@ -1456,24 +1490,31 @@ dependencies = [ [[package]] name = "polars-ops" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7740d7bc4c2ca08044f9ef599638e116fdd7d687e80d1974b698e390c6ce4252" +checksum = "2f9af534f705acaf83515a64ba88f208f6f800b5c47939fec1b8b26c1d401b15" dependencies = [ + "ahash", "argminmax", - "arrow2", "base64", + "bytemuck", "chrono", "chrono-tz", "either", + "hashbrown 0.14.0", "hex", "indexmap", "jsonpath_lib", "memchr", + "num-traits", "polars-arrow", "polars-core", + "polars-error", "polars-json", "polars-utils", + "rand", + "rand_distr", + "rayon", "regex", "serde", "serde_json", @@ -1483,9 +1524,9 @@ dependencies = [ [[package]] name = "polars-pipe" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f30c5e77c5594ddc958a46fe2e021da2feba9c94e767e1d798bd82ac5a33c3b" +checksum = "3c361a437ad24882ee47210e74a1ecdf83c211a3e6a7a1ae3fc1a9aa8a4c57cd" dependencies = [ "crossbeam-channel", "crossbeam-queue", @@ -1506,15 +1547,16 @@ dependencies = [ [[package]] name = "polars-plan" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678cbeb730e29e50f0f8d844102d15454fc6113a74c667eab046c0e4a4322a9e" +checksum = "c886b06d95ba37a3342a891f2ded8c779b8b9375f6648648624e3d04e753e22b" dependencies = [ "ahash", - "arrow2", + "bytemuck", "chrono", "chrono-tz", "once_cell", + "percent-encoding", "polars-arrow", "polars-core", "polars-io", @@ -1531,25 +1573,27 @@ dependencies = [ [[package]] name = "polars-row" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c52ef8885b9d13f848839594fbab21ad79fc63f7e11c19cdc2cfe9bb03c313ac" +checksum = "58b22c6535db871c2a8919a0b5ade6668bb068aaedc99c8e2523f40163ef9903" dependencies = [ - "arrow2", + "polars-arrow", "polars-error", "polars-utils", ] [[package]] name = "polars-sql" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d716855267e3516f722287f68cf10e650e33f7197df83a79e680602471456fc" +checksum = "9a0e7d72e0a918b12f3eeb8ff250d927c31dcf2beaabd96f5a5b955d69577912" dependencies = [ "polars-arrow", "polars-core", + "polars-error", "polars-lazy", "polars-plan", + "rand", "serde", "serde_json", "sqlparser", @@ -1557,11 +1601,10 @@ dependencies = [ [[package]] name = "polars-time" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb75a24f11b55a400b52dc19a2a3e949aaaa46a911f99496de4485b1127063" +checksum = "441618b303021a5f9367658d88d9ab2734cf643ac957126dab56c4447c80bbb0" dependencies = [ - "arrow2", "atoi", "chrono", "chrono-tz", @@ -1569,6 +1612,7 @@ dependencies = [ "once_cell", "polars-arrow", "polars-core", + "polars-error", "polars-ops", "polars-utils", "regex", @@ -1578,9 +1622,9 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.33.2" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4a5e743509096322cad39104d56e329fe2748483a3354a0f0c354724f3cef6" +checksum = "e40876670842ec450eface74aa9391ae2fff5e8934fbfc7a7b10051c7f79023d" dependencies = [ "ahash", "bytemuck", @@ -1660,9 +1704,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -1670,14 +1714,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -1754,6 +1796,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1875,9 +1923,9 @@ dependencies = [ [[package]] name = "simd-json" -version = "0.10.7" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ea1dfc2c400965867fc4ddd6f502572be2de2074b39f90984ed15fbdbdd8eb" +checksum = "f0f07a84c7456b901b8dd2c1d44caca8b0fd2c2616206ee5acc9d9da61e8d9ec" dependencies = [ "ahash", "getrandom", @@ -1935,11 +1983,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" +[[package]] +name = "socket2" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "sqlparser" -version = "0.36.1" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eaa1e88e78d2c2460d78b7dc3f0c08dbb606ab4222f9aff36f420d36e307d87" +checksum = "0272b7bb0a225320170c99901b4b5fb3a4384e255a7f2cc228f61e2ba3893e75" dependencies = [ "log", ] @@ -2065,6 +2123,35 @@ dependencies = [ "syn 2.0.36", ] +[[package]] +name = "tokio" +version = "1.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "unicode-ident" version = "1.0.12" @@ -2258,13 +2345,42 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" +[[package]] +name = "zerocopy" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8db0ac2df3d060f81ec0380ccc5b71c2a7c092cfced671feeee1320e95559c87" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b6093bc6d5265ff40b479c834cdd25d8e20784781a2a29a8106327393d0a9ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.36", +] + [[package]] name = "zstd" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ - "zstd-safe", + "zstd-safe 6.0.6", +] + +[[package]] +name = "zstd" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +dependencies = [ + "zstd-safe 7.0.0", ] [[package]] @@ -2277,6 +2393,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.8+zstd.1.5.5" diff --git a/ext/polars/Cargo.toml b/ext/polars/Cargo.toml index af9df8cb7a..292c2a2c8f 100644 --- a/ext/polars/Cargo.toml +++ b/ext/polars/Cargo.toml @@ -15,12 +15,12 @@ ahash = "0.8" chrono = "0.4" either = "1.8" magnus = "0.6" -polars-core = "0.33.2" +polars-core = "0.34.1" serde_json = "1" smartstring = "1" [dependencies.polars] -version = "0.33.2" +version = "0.34.1" features = [ "abs", "approx_unique", @@ -65,6 +65,7 @@ features = [ "parquet", "partition_by", "pct_change", + "peaks", "performant", "pivot", "product", @@ -83,7 +84,7 @@ features = [ "sign", "string_encoding", "string_from_radix", - "string_justify", + "string_pad", "strings", "timezones", "to_dummies", diff --git a/ext/polars/src/batched_csv.rs b/ext/polars/src/batched_csv.rs index 4fcb963ecc..442399dc16 100644 --- a/ext/polars/src/batched_csv.rs +++ b/ext/polars/src/batched_csv.rs @@ -31,7 +31,7 @@ impl RbBatchedCsv { let n_rows = Option::::try_convert(arguments[4])?; let skip_rows = usize::try_convert(arguments[5])?; let projection = Option::>::try_convert(arguments[6])?; - let sep = String::try_convert(arguments[7])?; + let separator = String::try_convert(arguments[7])?; let rechunk = bool::try_convert(arguments[8])?; let columns = Option::>::try_convert(arguments[9])?; let encoding = Wrap::::try_convert(arguments[10])?; @@ -90,7 +90,7 @@ impl RbBatchedCsv { .infer_schema(infer_schema_length) .has_header(has_header) .with_n_rows(n_rows) - .with_delimiter(sep.as_bytes()[0]) + .with_separator(separator.as_bytes()[0]) .with_skip_rows(skip_rows) .with_ignore_errors(ignore_errors) .with_projection(projection) diff --git a/ext/polars/src/conversion.rs b/ext/polars/src/conversion.rs index 250b7d9c81..b935ef3779 100644 --- a/ext/polars/src/conversion.rs +++ b/ext/polars/src/conversion.rs @@ -14,6 +14,7 @@ use polars::frame::NullStrategy; use polars::io::avro::AvroCompression; use polars::prelude::*; use polars::series::ops::NullBehavior; +use polars_core::utils::arrow::util::total_ord::TotalEq; use smartstring::alias::String as SmartString; use crate::object::OBJECT_NAME; @@ -655,6 +656,12 @@ impl PartialEq for ObjectValue { } } +impl TotalEq for ObjectValue { + fn tot_eq(&self, other: &Self) -> bool { + self == other + } +} + impl Display for ObjectValue { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.to_object()) @@ -900,6 +907,22 @@ impl TryConvert for Wrap { } } +impl TryConvert for Wrap