Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Upgrade Magnus to 0.6.0 #28

Merged
merged 3 commits into from
Jul 29, 2023
Merged

Upgrade Magnus to 0.6.0 #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
33881df
upgrade magnus and get compiling
matsadler Jul 16, 2023
f94f9d6
fix warnings
matsadler Jul 16, 2023
9cf491b
work around Ruby types no longer being Send + Sync
matsadler Jul 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion ext/polars/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ crate-type = ["cdylib"]
[dependencies]
ahash = "0.8"
chrono = "=0.4.24"
magnus = "0.5"
magnus = "0.6"
polars-core = "0.31.1"
serde_json = "1"
smartstring = "1"
Expand Down
61 changes: 28 additions & 33 deletions ext/polars/src/apply/dataframe.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use magnus::{class, IntoValue, RArray, TryConvert, Value};
use magnus::{class, prelude::*, typed_data::Obj, IntoValue, RArray, TryConvert, Value};
use polars::prelude::*;
use polars_core::frame::row::{rows_to_schema_first_non_null, Row};
use polars_core::series::SeriesIter;
Expand Down Expand Up @@ -34,48 +34,48 @@ pub fn apply_lambda_unknown<'a>(
null_count += 1;
continue;
} else if out.is_kind_of(class::true_class()) || out.is_kind_of(class::false_class()) {
let first_value = out.try_convert::<bool>().ok();
let first_value = bool::try_convert(out).ok();
return Ok((
RbSeries::new(
Obj::wrap(RbSeries::new(
apply_lambda_with_bool_out_type(df, lambda, null_count, first_value)
.into_series(),
)
.into(),
))
.as_value(),
false,
));
} else if out.is_kind_of(class::float()) {
let first_value = out.try_convert::<f64>().ok();
let first_value = f64::try_convert(out).ok();

return Ok((
RbSeries::new(
Obj::wrap(RbSeries::new(
apply_lambda_with_primitive_out_type::<Float64Type>(
df,
lambda,
null_count,
first_value,
)
.into_series(),
)
.into(),
))
.as_value(),
false,
));
} else if out.is_kind_of(class::integer()) {
let first_value = out.try_convert::<i64>().ok();
let first_value = i64::try_convert(out).ok();
return Ok((
RbSeries::new(
Obj::wrap(RbSeries::new(
apply_lambda_with_primitive_out_type::<Int64Type>(
df,
lambda,
null_count,
first_value,
)
.into_series(),
)
.into(),
))
.as_value(),
false,
));
// } else if out.is_kind_of(class::string()) {
// let first_value = out.try_convert::<String>().ok();
// let first_value = String::try_convert(out).ok();
// return Ok((
// RbSeries::new(
// apply_lambda_with_utf8_out_type(df, lambda, null_count, first_value)
Expand All @@ -85,25 +85,21 @@ pub fn apply_lambda_unknown<'a>(
// false,
// ));
} else if out.respond_to("_s", true)? {
let rb_rbseries: Value = out.funcall("_s", ()).unwrap();
let series = rb_rbseries
.try_convert::<&RbSeries>()
.unwrap()
.series
.borrow();
let rb_rbseries: Obj<RbSeries> = out.funcall("_s", ()).unwrap();
let series = rb_rbseries.series.borrow();
let dt = series.dtype();
return Ok((
RbSeries::new(
Obj::wrap(RbSeries::new(
apply_lambda_with_list_out_type(df, lambda, null_count, Some(&series), dt)?
.into_series(),
)
.into(),
))
.as_value(),
false,
));
} else if out.try_convert::<Wrap<Row<'a>>>().is_ok() {
let first_value = out.try_convert::<Wrap<Row<'a>>>().unwrap().0;
} else if Wrap::<Row<'a>>::try_convert(out).is_ok() {
let first_value = Wrap::<Row<'a>>::try_convert(out).unwrap().0;
return Ok((
RbDataFrame::from(
Obj::wrap(RbDataFrame::from(
apply_lambda_with_rows_output(
df,
lambda,
Expand All @@ -112,8 +108,8 @@ pub fn apply_lambda_unknown<'a>(
inference_size,
)
.map_err(RbPolarsErr::from)?,
)
.into(),
))
.as_value(),
true,
));
} else if out.is_kind_of(class::array()) {
Expand Down Expand Up @@ -143,7 +139,7 @@ where
let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap()));
let tpl = (RArray::from_iter(iter),);
match lambda.funcall::<_, _, Value>("call", tpl) {
Ok(val) => val.try_convert::<T>().ok(),
Ok(val) => T::try_convert(val).ok(),
Err(e) => panic!("ruby function failed {}", e),
}
})
Expand Down Expand Up @@ -219,8 +215,7 @@ pub fn apply_lambda_with_list_out_type(
let tpl = (RArray::from_iter(iter),);
match lambda.funcall::<_, _, Value>("call", tpl) {
Ok(val) => match val.funcall::<_, _, Value>("_s", ()) {
Ok(val) => val
.try_convert::<&RbSeries>()
Ok(val) => Obj::<RbSeries>::try_convert(val)
.ok()
.map(|ps| ps.series.borrow().clone()),
Err(_) => {
Expand Down Expand Up @@ -257,11 +252,11 @@ pub fn apply_lambda_with_rows_output<'a>(
let tpl = (RArray::from_iter(iter),);
match lambda.funcall::<_, _, Value>("call", tpl) {
Ok(val) => {
match val.try_convert::<RArray>().ok() {
match RArray::try_convert(val).ok() {
Some(tuple) => {
row_buf.0.clear();
for v in tuple.each() {
let v = v.unwrap().try_convert::<Wrap<AnyValue>>().unwrap().0;
let v = Wrap::<AnyValue>::try_convert(v.unwrap()).unwrap().0;
row_buf.0.push(v);
}
let ptr = &row_buf as *const Row;
Expand Down
6 changes: 3 additions & 3 deletions ext/polars/src/apply/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ pub mod dataframe;
pub mod lazy;
pub mod series;

use magnus::{RHash, Value};
use magnus::{prelude::*, RHash, Value};
use polars::chunked_array::builder::get_list_builder;
use polars::prelude::*;
use polars_core::export::rayon::prelude::*;
Expand Down Expand Up @@ -68,7 +68,7 @@ fn iterator_to_struct(
}
}
Some(dict) => {
let dict = dict.try_convert::<RHash>()?;
let dict = RHash::try_convert(dict)?;
if dict.len() != struct_width {
return Err(crate::error::ComputeError::new_err(
format!("Cannot create struct type.\n> The struct dtype expects {} fields, but it got a dict with {} fields.", struct_width, dict.len())
Expand All @@ -78,7 +78,7 @@ fn iterator_to_struct(
// the first item determines the output name
todo!()
// for ((_, val), field_items) in dict.iter().zip(&mut items) {
// let item = val.try_convert::<Wrap<AnyValue>>()?;
// let item = Wrap::<AnyValue>::try_convert(val)?;
// field_items.push(item.0)
// }
}
Expand Down
28 changes: 12 additions & 16 deletions ext/polars/src/apply/series.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use magnus::{class, IntoValue, RHash, TryConvert, Value};
use magnus::{class, prelude::*, typed_data::Obj, IntoValue, RHash, TryConvert, Value};
use polars::prelude::*;

use super::*;
Expand All @@ -14,12 +14,12 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>(
null_count: usize,
) -> RbResult<RbSeries> {
if out.is_kind_of(class::true_class()) || out.is_kind_of(class::false_class()) {
let first_value = out.try_convert::<bool>().unwrap();
let first_value = bool::try_convert(out).unwrap();
applyer
.apply_lambda_with_bool_out_type(lambda, null_count, Some(first_value))
.map(|ca| ca.into_series().into())
} else if out.is_kind_of(class::float()) {
let first_value = out.try_convert::<f64>().unwrap();
let first_value = f64::try_convert(out).unwrap();
applyer
.apply_lambda_with_primitive_out_type::<Float64Type>(
lambda,
Expand All @@ -28,7 +28,7 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>(
)
.map(|ca| ca.into_series().into())
} else if out.is_kind_of(class::string()) {
let first_value = out.try_convert::<String>().unwrap();
let first_value = String::try_convert(out).unwrap();
applyer
.apply_lambda_with_utf8_out_type(lambda, null_count, Some(first_value.as_str()))
.map(|ca| ca.into_series().into())
Expand All @@ -37,21 +37,21 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>(
} else if out.is_kind_of(class::array()) {
todo!()
} else if out.is_kind_of(class::hash()) {
let first = out.try_convert::<Wrap<AnyValue<'_>>>()?;
let first = Wrap::<AnyValue<'_>>::try_convert(out)?;
applyer.apply_to_struct(lambda, null_count, first.0)
}
// this succeeds for numpy ints as well, where checking if it is pyint fails
// we do this later in the chain so that we don't extract integers from string chars.
else if out.try_convert::<i64>().is_ok() {
let first_value = out.try_convert::<i64>().unwrap();
else if i64::try_convert(out).is_ok() {
let first_value = i64::try_convert(out).unwrap();
applyer
.apply_lambda_with_primitive_out_type::<Int64Type>(
lambda,
null_count,
Some(first_value),
)
.map(|ca| ca.into_series().into())
} else if let Ok(av) = out.try_convert::<Wrap<AnyValue>>() {
} else if let Ok(av) = Wrap::<AnyValue>::try_convert(out) {
applyer
.apply_extract_any_values(lambda, null_count, av.0)
.map(|s| s.into())
Expand Down Expand Up @@ -141,7 +141,7 @@ where
S: TryConvert,
{
match call_lambda(lambda, in_val) {
Ok(out) => out.try_convert::<S>(),
Ok(out) => S::try_convert(out),
Err(e) => panic!("ruby function failed {}", e),
}
}
Expand All @@ -151,13 +151,9 @@ where
T: IntoValue,
{
let out: Value = lambda.funcall("call", (in_val,))?;
let py_series: Value = out.funcall("_s", ())?;
Ok(py_series
.try_convert::<&RbSeries>()
.unwrap()
.series
.borrow()
.clone())
let py_series: Obj<RbSeries> = out.funcall("_s", ())?;
let tmp = py_series.series.borrow();
Ok(tmp.clone())
}

impl<'a> ApplyLambda<'a> for BooleanChunked {
Expand Down
50 changes: 25 additions & 25 deletions ext/polars/src/batched_csv.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use magnus::{RArray, Value};
use magnus::{prelude::*, RArray, Value};
use polars::io::mmap::MmapBytesReader;
use polars::io::RowCount;
use polars::prelude::read_impl::OwnedBatchedCsvReader;
Expand All @@ -24,31 +24,31 @@ impl RbBatchedCsv {
pub fn new(arguments: &[Value]) -> RbResult<Self> {
// start arguments
// this pattern is needed for more than 16
let infer_schema_length: Option<usize> = arguments[0].try_convert()?;
let chunk_size: usize = arguments[1].try_convert()?;
let has_header: bool = arguments[2].try_convert()?;
let ignore_errors: bool = arguments[3].try_convert()?;
let n_rows: Option<usize> = arguments[4].try_convert()?;
let skip_rows: usize = arguments[5].try_convert()?;
let projection: Option<Vec<usize>> = arguments[6].try_convert()?;
let sep: String = arguments[7].try_convert()?;
let rechunk: bool = arguments[8].try_convert()?;
let columns: Option<Vec<String>> = arguments[9].try_convert()?;
let encoding: Wrap<CsvEncoding> = arguments[10].try_convert()?;
let n_threads: Option<usize> = arguments[11].try_convert()?;
let path: PathBuf = arguments[12].try_convert()?;
let overwrite_dtype: Option<Vec<(String, Wrap<DataType>)>> = arguments[13].try_convert()?;
let infer_schema_length = Option::<usize>::try_convert(arguments[0])?;
let chunk_size = usize::try_convert(arguments[1])?;
let has_header = bool::try_convert(arguments[2])?;
let ignore_errors = bool::try_convert(arguments[3])?;
let n_rows = Option::<usize>::try_convert(arguments[4])?;
let skip_rows = usize::try_convert(arguments[5])?;
let projection = Option::<Vec<usize>>::try_convert(arguments[6])?;
let sep = String::try_convert(arguments[7])?;
let rechunk = bool::try_convert(arguments[8])?;
let columns = Option::<Vec<String>>::try_convert(arguments[9])?;
let encoding = Wrap::<CsvEncoding>::try_convert(arguments[10])?;
let n_threads = Option::<usize>::try_convert(arguments[11])?;
let path = PathBuf::try_convert(arguments[12])?;
let overwrite_dtype = Option::<Vec<(String, Wrap<DataType>)>>::try_convert(arguments[13])?;
// TODO fix
let overwrite_dtype_slice: Option<Vec<Wrap<DataType>>> = None; // arguments[14].try_convert()?;
let low_memory: bool = arguments[15].try_convert()?;
let comment_char: Option<String> = arguments[16].try_convert()?;
let quote_char: Option<String> = arguments[17].try_convert()?;
let null_values: Option<Wrap<NullValues>> = arguments[18].try_convert()?;
let try_parse_dates: bool = arguments[19].try_convert()?;
let skip_rows_after_header: usize = arguments[20].try_convert()?;
let row_count: Option<(String, IdxSize)> = arguments[21].try_convert()?;
let sample_size: usize = arguments[22].try_convert()?;
let eol_char: String = arguments[23].try_convert()?;
let overwrite_dtype_slice = Option::<Vec<Wrap<DataType>>>::None; // Option::<Vec<Wrap<DataType>>>::try_convert(arguments[14])?;
let low_memory = bool::try_convert(arguments[15])?;
let comment_char = Option::<String>::try_convert(arguments[16])?;
let quote_char = Option::<String>::try_convert(arguments[17])?;
let null_values = Option::<Wrap<NullValues>>::try_convert(arguments[18])?;
let try_parse_dates = bool::try_convert(arguments[19])?;
let skip_rows_after_header = usize::try_convert(arguments[20])?;
let row_count = Option::<(String, IdxSize)>::try_convert(arguments[21])?;
let sample_size = usize::try_convert(arguments[22])?;
let eol_char = String::try_convert(arguments[23])?;
// end arguments

let null_values = null_values.map(|w| w.0);
Expand Down
Loading
Loading