Skip to content

Commit

Permalink
ci: add tests for nested column.
Browse files Browse the repository at this point in the history
  • Loading branch information
youngsofun committed Jul 2, 2024
1 parent 81f800f commit 11c1ab6
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 0 deletions.
Binary file added tests/basic/data/nested_array_float.orc
Binary file not shown.
Binary file added tests/basic/data/nested_array_struct.orc
Binary file not shown.
Binary file added tests/basic/data/nested_map_struct.orc
Binary file not shown.
30 changes: 30 additions & 0 deletions tests/basic/data/write.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ def _write(

_write("struct<nest:struct<a:float,b:boolean>>", nested_struct, "nested_struct.orc")


nested_array = {
"value": [
[1, None, 3, 43, 5],
Expand All @@ -110,6 +111,25 @@ def _write(

_write("struct<value:array<int>>", nested_array, "nested_array.orc")


nested_array_float = {
"value": [
[1.0, 3.0],
[None, 2.0],
],
}

_write("struct<value:array<float>>", nested_array_float, "nested_array_float.orc")

nested_array_struct = {
"value": [
[(1.0, 1, "01"), (2.0, 2, "02")],
[None, (3.0, 3, "03")],
],
}

_write("struct<value:array<struct<a:float,b:int,c:string>>>", nested_array_struct, "nested_array_struct.orc")

nested_map = {
"map": [
{"zero": 0, "one": 1},
Expand All @@ -121,6 +141,16 @@ def _write(

_write("struct<map:map<string,int>>", nested_map, "nested_map.orc")

nested_map_struct = {
"map": [
{"01": (1.0, 1, "01"), "02": (2.0, 1, "02")},
None,
{"03": (3.0, 3, "03"), "04": (4.0, 4, "04")},
],
}

_write("struct<value:map<string,struct<a:float,b:int,c:string>>>", nested_map_struct, "nested_map_struct.orc")


_write(
infer_schema(data),
Expand Down
63 changes: 63 additions & 0 deletions tests/basic/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,58 @@ pub fn basic_test_nested_array() {
assert_batches_eq(&batch, &expected);
}

#[test]
pub fn basic_test_nested_array_float() {
let path = basic_path("nested_array_float.orc");
let reader = new_arrow_reader_root(&path);
let batch = reader.collect::<Result<Vec<_>, _>>().unwrap();

let expected = [
"+------------+",
"| value |",
"+------------+",
"| [1.0, 3.0] |",
"| [, 2.0] |",
"+------------+",
];
assert_batches_eq(&batch, &expected);
}

#[test]
pub fn basic_test_nested_array_struct() {
let path = basic_path("nested_array_struct.orc");
let reader = new_arrow_reader_root(&path);
let batch = reader.collect::<Result<Vec<_>, _>>().unwrap();

let expected = [
"+------------------------------------------------+",
"| value |",
"+------------------------------------------------+",
"| [{a: 1.0, b: 1, c: 01}, {a: 2.0, b: 2, c: 02}] |",
"| [, {a: 3.0, b: 3, c: 03}] |",
"+------------------------------------------------+",
];
assert_batches_eq(&batch, &expected);
}

#[test]
pub fn basic_test_nested_map_struct() {
let path = basic_path("nested_map_struct.orc");
let reader = new_arrow_reader_root(&path);
let batch = reader.collect::<Result<Vec<_>, _>>().unwrap();

let expected = [
"+--------------------------------------------------------+",
"| value |",
"+--------------------------------------------------------+",
"| {01: {a: 1.0, b: 1, c: 01}, 02: {a: 2.0, b: 1, c: 02}} |",
"| |",
"| {03: {a: 3.0, b: 3, c: 03}, 04: {a: 4.0, b: 4, c: 04}} |",
"+--------------------------------------------------------+",
];
assert_batches_eq(&batch, &expected);
}

#[test]
pub fn basic_test_nested_map() {
let path = basic_path("nested_map.orc");
Expand Down Expand Up @@ -342,6 +394,17 @@ pub async fn v0_file_test_async() {
assert_eq!(1_920_800, total_rows);
}

#[cfg(feature = "async")]
#[tokio::test]
pub async fn vx_file_test_async() {
let path = basic_path("0104.orc");
let reader = new_arrow_stream_reader_root(&path).await;
let batches = reader.try_collect::<Vec<_>>().await.unwrap();
let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
assert_eq!(149235, total_rows);
}


#[test]
pub fn alltypes_test() {
let compressions = ["none", "snappy", "zlib", "lzo", "zstd", "lz4"];
Expand Down

0 comments on commit 11c1ab6

Please sign in to comment.