From 11c1ab6cdc00666952e354065788adb243779906 Mon Sep 17 00:00:00 2001 From: Yang Xiufeng Date: Tue, 2 Jul 2024 16:50:20 +0800 Subject: [PATCH] ci: add tests for nested column. --- tests/basic/data/nested_array_float.orc | Bin 0 -> 358 bytes tests/basic/data/nested_array_struct.orc | Bin 0 -> 653 bytes tests/basic/data/nested_map_struct.orc | Bin 0 -> 742 bytes tests/basic/data/write.py | 30 +++++++++++ tests/basic/main.rs | 63 +++++++++++++++++++++++ 5 files changed, 93 insertions(+) create mode 100644 tests/basic/data/nested_array_float.orc create mode 100644 tests/basic/data/nested_array_struct.orc create mode 100644 tests/basic/data/nested_map_struct.orc diff --git a/tests/basic/data/nested_array_float.orc b/tests/basic/data/nested_array_float.orc new file mode 100644 index 0000000000000000000000000000000000000000..341f00b136441edde37fc351119ceecb5d756c9c GIT binary patch literal 358 zcmaivF;2rk5JmsY#?~-cXtNU1VjAPh2q|)eQs5@5xCj-AA~*n%=xCusLY#n}h9hwl z3VK-E2+B;c@9i}I&uY_Z>PEtA=z%ier1LF%=&6nn$DN4DkM}wSKIh{>BpeOm!ZBO=Rk!;DENLcn9Ns(D>0E@0i|3=Tv&2_8iI#l~FwY!WM z`$oA&w0>MHD=wX6jwnwaZ|@)O3RI|3?u$C&-;YQ#Sj|lz*!%@Hy92}P)g^6{>EvQE Vm8W=iQ8Kb>R;Mpt#;i8&;SXmYBbopJ literal 0 HcmV?d00001 diff --git a/tests/basic/data/nested_array_struct.orc b/tests/basic/data/nested_array_struct.orc new file mode 100644 index 0000000000000000000000000000000000000000..be822746be14e1832e749f9de00bab7bba8175f3 GIT binary patch literal 653 zcma)(F;2rk5JhKa*Jjulw5&u}3`GbOK}eB9D%@ZZ2~m+Kf&$TyAUay85nO;Hz-egV z1~~!+eHiaXfhYw?mf!#1znSlDzoTQT2z#KHH3C>F^$;ECik85EBH0sqS1UGPw@9WN zi=_nkJX{HaiuE8|!PQkAd1n)nHYPN&OE^w-97@)ZtZR)~%VO3pza5E}@%wlL@NmeL zf}dhMI>wzV&bXHCXKfu*Y`~GHaK;nelBaZgp31FxMsAadMmWg?gAP5Vr!dkb-m8gM zPI(`Bk0wws%Wc4vBcOi={X5}2mFH(2eWAmQKLdv4{;}BXP;>K;G;f%4$vg?SU3(P` ztXoQ~5NUKjxV^inK?5?VS4dld6I11alqyQaAQi(@Tu*YMXsYB?Ed-Te?vKc)N3iGL rpZ|SCK66AqcSP}gdLo8Vt9j6D(TCuFS*s%?jms>4{F1WU?^M45)rCEi literal 0 HcmV?d00001 diff --git a/tests/basic/data/nested_map_struct.orc b/tests/basic/data/nested_map_struct.orc new file mode 100644 index 0000000000000000000000000000000000000000..4c95e3c7e9501d207f550b8ca3a081c15371dd51 GIT binary patch literal 742 zcma)4yH3L}6!oUCYy+ zGc;eElZ3k(nB7JhZLRwA?5)3E1}1Nn!A?$dR&!4-Z9$FBQtdbL&)E^ofWDUQrw3_| zv4=h}<+SL1$d5&Q_1)D9m0=D&f@^x@uH{iwb?FkH2X4co@_lB&amqQN@W_1*B)#AR zBc1UWkQK$8s`|1T$~&KLHw@}nM!^Rt1z(WTML{??5xZT*L%Auf7ZBSK+bc@z>n zgWGGrKq6v`p#_M7`^oj)Wt$}|WpRjWmUB^Mh$~gppjjaU6~QEeX#^MZiC`YpPNgFA zDzhb(S?%X&iY`y~w@M;fP+2F+pH5F;8gx5{oi4ruU4(YbS(x0W>hVK};ds#a0*H4) ATmS$7 literal 0 HcmV?d00001 diff --git a/tests/basic/data/write.py b/tests/basic/data/write.py index 7c43c045..1606c99c 100644 --- a/tests/basic/data/write.py +++ b/tests/basic/data/write.py @@ -98,6 +98,7 @@ def _write( _write("struct>", nested_struct, "nested_struct.orc") + nested_array = { "value": [ [1, None, 3, 43, 5], @@ -110,6 +111,25 @@ def _write( _write("struct>", nested_array, "nested_array.orc") + +nested_array_float = { + "value": [ + [1.0, 3.0], + [None, 2.0], + ], +} + +_write("struct>", nested_array_float, "nested_array_float.orc") + +nested_array_struct = { + "value": [ + [(1.0, 1, "01"), (2.0, 2, "02")], + [None, (3.0, 3, "03")], + ], +} + +_write("struct>>", nested_array_struct, "nested_array_struct.orc") + nested_map = { "map": [ {"zero": 0, "one": 1}, @@ -121,6 +141,16 @@ def _write( _write("struct>", nested_map, "nested_map.orc") +nested_map_struct = { + "map": [ + {"01": (1.0, 1, "01"), "02": (2.0, 1, "02")}, + None, + {"03": (3.0, 3, "03"), "04": (4.0, 4, "04")}, + ], +} + +_write("struct>>", nested_map_struct, "nested_map_struct.orc") + _write( infer_schema(data), diff --git a/tests/basic/main.rs b/tests/basic/main.rs index 9a541f7b..e6537400 100644 --- a/tests/basic/main.rs +++ b/tests/basic/main.rs @@ -252,6 +252,58 @@ pub fn basic_test_nested_array() { assert_batches_eq(&batch, &expected); } +#[test] +pub fn basic_test_nested_array_float() { + let path = basic_path("nested_array_float.orc"); + let reader = new_arrow_reader_root(&path); + let batch = reader.collect::, _>>().unwrap(); + + let expected = [ + "+------------+", + "| value |", + "+------------+", + "| [1.0, 3.0] |", + "| [, 2.0] |", + "+------------+", + ]; + assert_batches_eq(&batch, &expected); +} + +#[test] +pub fn basic_test_nested_array_struct() { + let path = basic_path("nested_array_struct.orc"); + let reader = new_arrow_reader_root(&path); + let batch = reader.collect::, _>>().unwrap(); + + let expected = [ + "+------------------------------------------------+", + "| value |", + "+------------------------------------------------+", + "| [{a: 1.0, b: 1, c: 01}, {a: 2.0, b: 2, c: 02}] |", + "| [, {a: 3.0, b: 3, c: 03}] |", + "+------------------------------------------------+", + ]; + assert_batches_eq(&batch, &expected); +} + +#[test] +pub fn basic_test_nested_map_struct() { + let path = basic_path("nested_map_struct.orc"); + let reader = new_arrow_reader_root(&path); + let batch = reader.collect::, _>>().unwrap(); + + let expected = [ + "+--------------------------------------------------------+", + "| value |", + "+--------------------------------------------------------+", + "| {01: {a: 1.0, b: 1, c: 01}, 02: {a: 2.0, b: 1, c: 02}} |", + "| |", + "| {03: {a: 3.0, b: 3, c: 03}, 04: {a: 4.0, b: 4, c: 04}} |", + "+--------------------------------------------------------+", + ]; + assert_batches_eq(&batch, &expected); +} + #[test] pub fn basic_test_nested_map() { let path = basic_path("nested_map.orc"); @@ -342,6 +394,17 @@ pub async fn v0_file_test_async() { assert_eq!(1_920_800, total_rows); } +#[cfg(feature = "async")] +#[tokio::test] +pub async fn vx_file_test_async() { + let path = basic_path("0104.orc"); + let reader = new_arrow_stream_reader_root(&path).await; + let batches = reader.try_collect::>().await.unwrap(); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(149235, total_rows); +} + + #[test] pub fn alltypes_test() { let compressions = ["none", "snappy", "zlib", "lzo", "zstd", "lz4"];