Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Add deserialization of Bytes -> Decimal
Browse files Browse the repository at this point in the history
  • Loading branch information
Jay Chia committed Sep 5, 2023
1 parent 87ab844 commit 8e6d836
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 1 deletion.
32 changes: 31 additions & 1 deletion src/io/parquet/read/deserialize/simple.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use parquet2::{

use crate::types::i256;
use crate::{
array::{Array, DictionaryKey, MutablePrimitiveArray, PrimitiveArray},
array::{Array, DictionaryKey, MutablePrimitiveArray, PrimitiveArray, BinaryArray},
datatypes::{DataType, IntervalUnit, TimeUnit},
error::{Error, Result},
types::{days_ms, NativeType},
Expand Down Expand Up @@ -230,6 +230,36 @@ pub fn page_iter_to_arrays<'a, I: Pages + 'a>(

Box::new(arrays) as _
}
(PhysicalType::ByteArray, Decimal(_, _)) => {
let pages = binary::Iter::<i32, _>::new(
pages,
DataType::Binary,
chunk_size,
num_rows,
);

let pages = pages.map(move |maybe_array| {
let array = maybe_array?;
let array_len = array.len();
let array = array.as_any().downcast_ref::<BinaryArray<i32>>().unwrap();
let values = (0..array_len).map(|i| {
let value = array.value(i);
let n = value.len();
if n > 16 {
return Err(Error::Overflow);
}
Ok(super::super::convert_i128(value, n))
})
.collect::<Result<Vec<_>>>();
let validity = array.validity().cloned();

PrimitiveArray::<i128>::try_new(data_type.clone(), values?.into(), validity)
});

let arrays = pages.map(|x| x.map(|x| x.boxed()));

Box::new(arrays) as _

Check warning on line 261 in src/io/parquet/read/deserialize/simple.rs

View check run for this annotation

Codecov / codecov/patch

src/io/parquet/read/deserialize/simple.rs#L234-L261

Added lines #L234 - L261 were not covered by tests
}
(PhysicalType::Int32, Decimal256(_, _)) => dyn_iter(iden(primitive::IntegerIter::new(
pages,
data_type,
Expand Down
2 changes: 2 additions & 0 deletions src/io/parquet/read/schema/convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ fn from_byte_array(
(_, Some(PrimitiveConvertedType::Bson)) => DataType::Binary,
(_, Some(PrimitiveConvertedType::Enum)) => DataType::Binary,
(_, Some(PrimitiveConvertedType::Utf8)) => DataType::Utf8,
(Some(PrimitiveLogicalType::Decimal(precision, scale)), _) => DataType::Decimal(*precision, *scale),
(_, Some(PrimitiveConvertedType::Decimal(precision, scale))) => DataType::Decimal(*precision, *scale),

Check warning on line 146 in src/io/parquet/read/schema/convert.rs

View check run for this annotation

Codecov / codecov/patch

src/io/parquet/read/schema/convert.rs#L145-L146

Added lines #L145 - L146 were not covered by tests
(_, _) => DataType::Binary,
}
}
Expand Down

0 comments on commit 8e6d836

Please sign in to comment.