Skip to content

Commit

Permalink
Upgrade Magnus to 0.6.0 (#28)
Browse files Browse the repository at this point in the history
  • Loading branch information
@matsadler
matsadler authored Jul 29, 2023
1 parent c85fcd1 commit dbed1be
Show file tree
Hide file tree
Showing 21 changed files with 310 additions and 286 deletions.
11 changes: 6 additions & 5 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion ext/polars/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ crate-type = ["cdylib"]
[dependencies]
ahash = "0.8"
chrono = "=0.4.24"
magnus = "0.5"
magnus = "0.6"
polars-core = "0.31.1"
serde_json = "1"
smartstring = "1"
Expand Down
61 changes: 28 additions & 33 deletions ext/polars/src/apply/dataframe.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use magnus::{class, IntoValue, RArray, TryConvert, Value};
use magnus::{class, prelude::*, typed_data::Obj, IntoValue, RArray, TryConvert, Value};
use polars::prelude::*;
use polars_core::frame::row::{rows_to_schema_first_non_null, Row};
use polars_core::series::SeriesIter;
Expand Down Expand Up @@ -34,48 +34,48 @@ pub fn apply_lambda_unknown<'a>(
null_count += 1;
continue;
} else if out.is_kind_of(class::true_class()) || out.is_kind_of(class::false_class()) {
let first_value = out.try_convert::<bool>().ok();
let first_value = bool::try_convert(out).ok();
return Ok((
RbSeries::new(
Obj::wrap(RbSeries::new(
apply_lambda_with_bool_out_type(df, lambda, null_count, first_value)
.into_series(),
)
.into(),
))
.as_value(),
false,
));
} else if out.is_kind_of(class::float()) {
let first_value = out.try_convert::<f64>().ok();
let first_value = f64::try_convert(out).ok();

return Ok((
RbSeries::new(
Obj::wrap(RbSeries::new(
apply_lambda_with_primitive_out_type::<Float64Type>(
df,
lambda,
null_count,
first_value,
)
.into_series(),
)
.into(),
))
.as_value(),
false,
));
} else if out.is_kind_of(class::integer()) {
let first_value = out.try_convert::<i64>().ok();
let first_value = i64::try_convert(out).ok();
return Ok((
RbSeries::new(
Obj::wrap(RbSeries::new(
apply_lambda_with_primitive_out_type::<Int64Type>(
df,
lambda,
null_count,
first_value,
)
.into_series(),
)
.into(),
))
.as_value(),
false,
));
// } else if out.is_kind_of(class::string()) {
// let first_value = out.try_convert::<String>().ok();
// let first_value = String::try_convert(out).ok();
// return Ok((
// RbSeries::new(
// apply_lambda_with_utf8_out_type(df, lambda, null_count, first_value)
Expand All @@ -85,25 +85,21 @@ pub fn apply_lambda_unknown<'a>(
// false,
// ));
} else if out.respond_to("_s", true)? {
let rb_rbseries: Value = out.funcall("_s", ()).unwrap();
let series = rb_rbseries
.try_convert::<&RbSeries>()
.unwrap()
.series
.borrow();
let rb_rbseries: Obj<RbSeries> = out.funcall("_s", ()).unwrap();
let series = rb_rbseries.series.borrow();
let dt = series.dtype();
return Ok((
RbSeries::new(
Obj::wrap(RbSeries::new(
apply_lambda_with_list_out_type(df, lambda, null_count, Some(&series), dt)?
.into_series(),
)
.into(),
))
.as_value(),
false,
));
} else if out.try_convert::<Wrap<Row<'a>>>().is_ok() {
let first_value = out.try_convert::<Wrap<Row<'a>>>().unwrap().0;
} else if Wrap::<Row<'a>>::try_convert(out).is_ok() {
let first_value = Wrap::<Row<'a>>::try_convert(out).unwrap().0;
return Ok((
RbDataFrame::from(
Obj::wrap(RbDataFrame::from(
apply_lambda_with_rows_output(
df,
lambda,
Expand All @@ -112,8 +108,8 @@ pub fn apply_lambda_unknown<'a>(
inference_size,
)
.map_err(RbPolarsErr::from)?,
)
.into(),
))
.as_value(),
true,
));
} else if out.is_kind_of(class::array()) {
Expand Down Expand Up @@ -143,7 +139,7 @@ where
let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap()));
let tpl = (RArray::from_iter(iter),);
match lambda.funcall::<_, _, Value>("call", tpl) {
Ok(val) => val.try_convert::<T>().ok(),
Ok(val) => T::try_convert(val).ok(),
Err(e) => panic!("ruby function failed {}", e),
}
})
Expand Down Expand Up @@ -219,8 +215,7 @@ pub fn apply_lambda_with_list_out_type(
let tpl = (RArray::from_iter(iter),);
match lambda.funcall::<_, _, Value>("call", tpl) {
Ok(val) => match val.funcall::<_, _, Value>("_s", ()) {
Ok(val) => val
.try_convert::<&RbSeries>()
Ok(val) => Obj::<RbSeries>::try_convert(val)
.ok()
.map(|ps| ps.series.borrow().clone()),
Err(_) => {
Expand Down Expand Up @@ -257,11 +252,11 @@ pub fn apply_lambda_with_rows_output<'a>(
let tpl = (RArray::from_iter(iter),);
match lambda.funcall::<_, _, Value>("call", tpl) {
Ok(val) => {
match val.try_convert::<RArray>().ok() {
match RArray::try_convert(val).ok() {
Some(tuple) => {
row_buf.0.clear();
for v in tuple.each() {
let v = v.unwrap().try_convert::<Wrap<AnyValue>>().unwrap().0;
let v = Wrap::<AnyValue>::try_convert(v.unwrap()).unwrap().0;
row_buf.0.push(v);
}
let ptr = &row_buf as *const Row;
Expand Down
6 changes: 3 additions & 3 deletions ext/polars/src/apply/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ pub mod dataframe;
pub mod lazy;
pub mod series;

use magnus::{RHash, Value};
use magnus::{prelude::*, RHash, Value};
use polars::chunked_array::builder::get_list_builder;
use polars::prelude::*;
use polars_core::export::rayon::prelude::*;
Expand Down Expand Up @@ -68,7 +68,7 @@ fn iterator_to_struct(
}
}
Some(dict) => {
let dict = dict.try_convert::<RHash>()?;
let dict = RHash::try_convert(dict)?;
if dict.len() != struct_width {
return Err(crate::error::ComputeError::new_err(
format!("Cannot create struct type.\n> The struct dtype expects {} fields, but it got a dict with {} fields.", struct_width, dict.len())
Expand All @@ -78,7 +78,7 @@ fn iterator_to_struct(
// the first item determines the output name
todo!()
// for ((_, val), field_items) in dict.iter().zip(&mut items) {
// let item = val.try_convert::<Wrap<AnyValue>>()?;
// let item = Wrap::<AnyValue>::try_convert(val)?;
// field_items.push(item.0)
// }
}
Expand Down
28 changes: 12 additions & 16 deletions ext/polars/src/apply/series.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use magnus::{class, IntoValue, RHash, TryConvert, Value};
use magnus::{class, prelude::*, typed_data::Obj, IntoValue, RHash, TryConvert, Value};
use polars::prelude::*;

use super::*;
Expand All @@ -14,12 +14,12 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>(
null_count: usize,
) -> RbResult<RbSeries> {
if out.is_kind_of(class::true_class()) || out.is_kind_of(class::false_class()) {
let first_value = out.try_convert::<bool>().unwrap();
let first_value = bool::try_convert(out).unwrap();
applyer
.apply_lambda_with_bool_out_type(lambda, null_count, Some(first_value))
.map(|ca| ca.into_series().into())
} else if out.is_kind_of(class::float()) {
let first_value = out.try_convert::<f64>().unwrap();
let first_value = f64::try_convert(out).unwrap();
applyer
.apply_lambda_with_primitive_out_type::<Float64Type>(
lambda,
Expand All @@ -28,7 +28,7 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>(
)
.map(|ca| ca.into_series().into())
} else if out.is_kind_of(class::string()) {
let first_value = out.try_convert::<String>().unwrap();
let first_value = String::try_convert(out).unwrap();
applyer
.apply_lambda_with_utf8_out_type(lambda, null_count, Some(first_value.as_str()))
.map(|ca| ca.into_series().into())
Expand All @@ -37,21 +37,21 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>(
} else if out.is_kind_of(class::array()) {
todo!()
} else if out.is_kind_of(class::hash()) {
let first = out.try_convert::<Wrap<AnyValue<'_>>>()?;
let first = Wrap::<AnyValue<'_>>::try_convert(out)?;
applyer.apply_to_struct(lambda, null_count, first.0)
}
// this succeeds for numpy ints as well, where checking if it is pyint fails
// we do this later in the chain so that we don't extract integers from string chars.
else if out.try_convert::<i64>().is_ok() {
let first_value = out.try_convert::<i64>().unwrap();
else if i64::try_convert(out).is_ok() {
let first_value = i64::try_convert(out).unwrap();
applyer
.apply_lambda_with_primitive_out_type::<Int64Type>(
lambda,
null_count,
Some(first_value),
)
.map(|ca| ca.into_series().into())
} else if let Ok(av) = out.try_convert::<Wrap<AnyValue>>() {
} else if let Ok(av) = Wrap::<AnyValue>::try_convert(out) {
applyer
.apply_extract_any_values(lambda, null_count, av.0)
.map(|s| s.into())
Expand Down Expand Up @@ -141,7 +141,7 @@ where
S: TryConvert,
{
match call_lambda(lambda, in_val) {
Ok(out) => out.try_convert::<S>(),
Ok(out) => S::try_convert(out),
Err(e) => panic!("ruby function failed {}", e),
}
}
Expand All @@ -151,13 +151,9 @@ where
T: IntoValue,
{
let out: Value = lambda.funcall("call", (in_val,))?;
let py_series: Value = out.funcall("_s", ())?;
Ok(py_series
.try_convert::<&RbSeries>()
.unwrap()
.series
.borrow()
.clone())
let py_series: Obj<RbSeries> = out.funcall("_s", ())?;
let tmp = py_series.series.borrow();
Ok(tmp.clone())
}

impl<'a> ApplyLambda<'a> for BooleanChunked {
Expand Down
50 changes: 25 additions & 25 deletions ext/polars/src/batched_csv.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use magnus::{RArray, Value};
use magnus::{prelude::*, RArray, Value};
use polars::io::mmap::MmapBytesReader;
use polars::io::RowCount;
use polars::prelude::read_impl::OwnedBatchedCsvReader;
Expand All @@ -24,31 +24,31 @@ impl RbBatchedCsv {
pub fn new(arguments: &[Value]) -> RbResult<Self> {
// start arguments
// this pattern is needed for more than 16
let infer_schema_length: Option<usize> = arguments[0].try_convert()?;
let chunk_size: usize = arguments[1].try_convert()?;
let has_header: bool = arguments[2].try_convert()?;
let ignore_errors: bool = arguments[3].try_convert()?;
let n_rows: Option<usize> = arguments[4].try_convert()?;
let skip_rows: usize = arguments[5].try_convert()?;
let projection: Option<Vec<usize>> = arguments[6].try_convert()?;
let sep: String = arguments[7].try_convert()?;
let rechunk: bool = arguments[8].try_convert()?;
let columns: Option<Vec<String>> = arguments[9].try_convert()?;
let encoding: Wrap<CsvEncoding> = arguments[10].try_convert()?;
let n_threads: Option<usize> = arguments[11].try_convert()?;
let path: PathBuf = arguments[12].try_convert()?;
let overwrite_dtype: Option<Vec<(String, Wrap<DataType>)>> = arguments[13].try_convert()?;
let infer_schema_length = Option::<usize>::try_convert(arguments[0])?;
let chunk_size = usize::try_convert(arguments[1])?;
let has_header = bool::try_convert(arguments[2])?;
let ignore_errors = bool::try_convert(arguments[3])?;
let n_rows = Option::<usize>::try_convert(arguments[4])?;
let skip_rows = usize::try_convert(arguments[5])?;
let projection = Option::<Vec<usize>>::try_convert(arguments[6])?;
let sep = String::try_convert(arguments[7])?;
let rechunk = bool::try_convert(arguments[8])?;
let columns = Option::<Vec<String>>::try_convert(arguments[9])?;
let encoding = Wrap::<CsvEncoding>::try_convert(arguments[10])?;
let n_threads = Option::<usize>::try_convert(arguments[11])?;
let path = PathBuf::try_convert(arguments[12])?;
let overwrite_dtype = Option::<Vec<(String, Wrap<DataType>)>>::try_convert(arguments[13])?;
// TODO fix
let overwrite_dtype_slice: Option<Vec<Wrap<DataType>>> = None; // arguments[14].try_convert()?;
let low_memory: bool = arguments[15].try_convert()?;
let comment_char: Option<String> = arguments[16].try_convert()?;
let quote_char: Option<String> = arguments[17].try_convert()?;
let null_values: Option<Wrap<NullValues>> = arguments[18].try_convert()?;
let try_parse_dates: bool = arguments[19].try_convert()?;
let skip_rows_after_header: usize = arguments[20].try_convert()?;
let row_count: Option<(String, IdxSize)> = arguments[21].try_convert()?;
let sample_size: usize = arguments[22].try_convert()?;
let eol_char: String = arguments[23].try_convert()?;
let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::None; // Option::<Vec<Wrap<DataType>>>::try_convert(arguments[14])?;
let low_memory = bool::try_convert(arguments[15])?;
let comment_char = Option::<String>::try_convert(arguments[16])?;
let quote_char = Option::<String>::try_convert(arguments[17])?;
let null_values = Option::<Wrap<NullValues>>::try_convert(arguments[18])?;
let try_parse_dates = bool::try_convert(arguments[19])?;
let skip_rows_after_header = usize::try_convert(arguments[20])?;
let row_count = Option::<(String, IdxSize)>::try_convert(arguments[21])?;
let sample_size = usize::try_convert(arguments[22])?;
let eol_char = String::try_convert(arguments[23])?;
// end arguments

let null_values = null_values.map(|w| w.0);
Expand Down
Loading

0 comments on commit dbed1be

Please sign in to comment.