diff --git a/Cargo.lock b/Cargo.lock index 2f0d09be9..090ca6901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,7 +147,7 @@ name = "async-jsonrpc-client" version = "0.1.0" source = "git+https://github.com/witnet/async-jsonrpc-client?branch=fix-tcp-leak#600a2d696af265511868f77c01e448c1d678650e" dependencies = [ - "error-chain", + "error-chain 0.12.4", "futures 0.1.31", "jsonrpc-core 10.1.0", "log 0.4.19", @@ -242,9 +242,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "bech32" @@ -929,6 +929,15 @@ dependencies = [ "libc", ] +[[package]] +name = "error-chain" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" +dependencies = [ + "backtrace", +] + [[package]] name = "error-chain" version = "0.12.4" @@ -1753,6 +1762,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonpath" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8061db09019f1879ba586685694fe18279f597b1b3a9dd308f35e596be6cdf7d" +dependencies = [ + "error-chain 0.11.0", + "pest", + "pest_derive", + "serde", + "serde_json", +] + [[package]] name = "jsonrpc-core" version = "10.1.0" @@ -2744,6 +2766,23 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "pest" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fce5d8b5cc33983fc74f78ad552b5522ab41442c4ca91606e4236eb4b5ceefc" + +[[package]] +name = "pest_derive" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3294f437119209b084c797604295f40227cffa35c57220b1e99a6ff3bf8ee4" +dependencies = [ + "pest", + "quote 0.3.15", + "syn 0.11.11", +] + [[package]] name = "pin-project" version = "1.1.0" @@ -2968,6 +3007,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" + [[package]] name = "quote" version = "0.6.13" @@ -3252,7 +3297,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.2", + "base64 0.21.5", "bytes 1.4.0", "encoding_rs", "futures-core", @@ -3719,6 +3764,12 @@ dependencies = [ "autocfg 1.1.0", ] +[[package]] +name = "slicestring" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7b0ad4ee74da25b3ff80bc5aa633bf8b1e0bbaf577c6f44529a26ad2d9df75" + [[package]] name = "sluice" version = "0.5.5" @@ -3820,6 +3871,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +dependencies = [ + "quote 0.3.15", + "synom", + "unicode-xid 0.0.4", +] + [[package]] name = "syn" version = "0.15.44" @@ -3853,6 +3915,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" +dependencies = [ + "unicode-xid 0.0.4", +] + [[package]] name = "synstructure" version = "0.12.6" @@ -4466,6 +4537,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" + [[package]] name = "unicode-xid" version = "0.1.0" @@ -4490,7 +4567,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b11c96ac7ee530603dcdf68ed1557050f374ce55a5a07193ebf8cbc9f8927e9" dependencies = [ - "base64 0.21.2", + "base64 0.21.5", "log 0.4.19", "native-tls", "once_cell", @@ -4958,7 +5035,7 @@ dependencies = [ [[package]] name = "witnet" -version = "1.6.7" +version = "2.0.0" dependencies = [ "ansi_term", "bytecount", @@ -5005,7 +5082,7 @@ dependencies = [ [[package]] name = "witnet-centralized-ethereum-bridge" -version = "1.6.7" +version = "2.0.0" dependencies = [ "actix", "async-jsonrpc-client", @@ -5069,7 +5146,7 @@ dependencies = [ [[package]] name = "witnet_data_structures" -version = "1.6.7" +version = "2.0.0" dependencies = [ "bech32", "bencher", @@ -5134,7 +5211,7 @@ dependencies = [ [[package]] name = "witnet_node" -version = "1.6.7" +version = "2.0.0" dependencies = [ "actix", "ansi_term", @@ -5198,22 +5275,26 @@ dependencies = [ [[package]] name = "witnet_rad" -version = "0.3.2" +version = "0.3.3" dependencies = [ + "base64 0.21.5", "cbor-codec", "failure", "futures 0.3.28", "hex", "http", "if_rust_version", + "jsonpath", "log 0.4.19", "minidom", "num_enum", "ordered-float", "rand 0.7.3", + "regex", "serde", "serde_cbor", "serde_json", + "slicestring", "url", "witnet_config", "witnet_crypto", @@ -5241,7 +5322,7 @@ dependencies = [ [[package]] name = "witnet_toolkit" -version = "1.6.7" +version = "2.0.0" dependencies = [ "failure", "hex", @@ -5284,7 +5365,7 @@ dependencies = [ [[package]] name = "witnet_wallet" -version = "1.6.7" +version = "2.0.0" dependencies = [ "actix", "async-jsonrpc-client", diff --git a/Cargo.toml b/Cargo.toml index 0a428c2fb..21a934c11 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet" -version = "1.6.7" +version = "2.0.0" authors = ["Witnet Foundation "] publish = false repository = "witnet/witnet-rust" diff --git a/bridges/centralized-ethereum/Cargo.toml b/bridges/centralized-ethereum/Cargo.toml index 5492feb9e..6e3a4f7a0 100644 --- a/bridges/centralized-ethereum/Cargo.toml +++ b/bridges/centralized-ethereum/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet-centralized-ethereum-bridge" -version = "1.6.7" +version = "2.0.0" authors = ["Witnet Foundation "] edition = "2018" diff --git a/data_structures/Cargo.toml b/data_structures/Cargo.toml index fa801502b..9fd896c71 100644 --- a/data_structures/Cargo.toml +++ b/data_structures/Cargo.toml @@ -3,7 +3,7 @@ authors = ["Witnet Foundation "] description = "data structures component" edition = "2021" name = "witnet_data_structures" -version = "1.6.7" +version = "2.0.0" workspace = ".." [features] diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs index 34910b8a5..8b9b3d729 100644 --- a/data_structures/src/chain/mod.rs +++ b/data_structures/src/chain/mod.rs @@ -1645,11 +1645,17 @@ pub enum RADType { /// HTTP POST request #[serde(rename = "HTTP-POST")] HttpPost, + /// HTTP HEAD request + #[serde(rename = "HTTP-HEAD")] + HttpHead, } impl RADType { pub fn is_http(&self) -> bool { - matches!(self, RADType::HttpGet | RADType::HttpPost) + matches!( + self, + RADType::HttpGet | RADType::HttpPost | RADType::HttpHead + ) } } @@ -1701,7 +1707,7 @@ pub struct RADRetrieve { pub script: Vec, /// Body of a HTTP-POST request pub body: Vec, - /// Extra headers of a HTTP-GET or HTTP-POST request + /// Extra headers of a HTTP-GET, HTTP-POST or HTTP-HEAD request pub headers: Vec<(String, String)>, } @@ -1810,6 +1816,9 @@ impl RADRetrieve { &[Field::Body, Field::Headers], ) } + RADType::HttpHead => { + check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers]) + } } } @@ -2681,7 +2690,7 @@ impl TransactionsPool { for input in &vt_tx.body.inputs { self.output_pointer_map .entry(input.output_pointer) - .or_insert_with(Vec::new) + .or_default() .push(vt_tx.hash()); } @@ -2706,7 +2715,7 @@ impl TransactionsPool { for input in &dr_tx.body.inputs { self.output_pointer_map .entry(input.output_pointer) - .or_insert_with(Vec::new) + .or_default() .push(dr_tx.hash()); } diff --git a/data_structures/src/data_request.rs b/data_structures/src/data_request.rs index 6dd3493a1..fc3823037 100644 --- a/data_structures/src/data_request.rs +++ b/data_structures/src/data_request.rs @@ -143,7 +143,7 @@ impl DataRequestPool { self.data_requests_by_epoch .entry(epoch) - .or_insert_with(HashSet::new) + .or_default() .insert(dr_hash); self.data_request_pool.insert(dr_hash, dr_state); diff --git a/data_structures/src/proto/mod.rs b/data_structures/src/proto/mod.rs index 4d681d8b9..c1bb007ca 100644 --- a/data_structures/src/proto/mod.rs +++ b/data_structures/src/proto/mod.rs @@ -51,6 +51,7 @@ impl ProtobufConvert for chain::RADType { chain::RADType::HttpGet => witnet::DataRequestOutput_RADRequest_RADType::HttpGet, chain::RADType::Rng => witnet::DataRequestOutput_RADRequest_RADType::Rng, chain::RADType::HttpPost => witnet::DataRequestOutput_RADRequest_RADType::HttpPost, + chain::RADType::HttpHead => witnet::DataRequestOutput_RADRequest_RADType::HttpHead, } } @@ -60,6 +61,7 @@ impl ProtobufConvert for chain::RADType { witnet::DataRequestOutput_RADRequest_RADType::HttpGet => chain::RADType::HttpGet, witnet::DataRequestOutput_RADRequest_RADType::Rng => chain::RADType::Rng, witnet::DataRequestOutput_RADRequest_RADType::HttpPost => chain::RADType::HttpPost, + witnet::DataRequestOutput_RADRequest_RADType::HttpHead => chain::RADType::HttpHead, }) } } diff --git a/data_structures/src/radon_error.rs b/data_structures/src/radon_error.rs index 7d3176629..6fdf30d26 100644 --- a/data_structures/src/radon_error.rs +++ b/data_structures/src/radon_error.rs @@ -34,6 +34,8 @@ pub enum RadonErrors { HTTPError = 0x30, /// Al least one of the sources could not be retrieved, timeout reached. RetrieveTimeout = 0x31, + /// Value cannot be extracted from binary buffer + BufferIsNotValue = 0x32, // Math errors /// Math operator caused an underflow. Underflow = 0x40, diff --git a/data_structures/src/serialization_helpers.rs b/data_structures/src/serialization_helpers.rs index 7f7d7d4e6..a8d59ee69 100644 --- a/data_structures/src/serialization_helpers.rs +++ b/data_structures/src/serialization_helpers.rs @@ -360,7 +360,7 @@ struct RADRetrieveSerializationHelperJson { /// Body of a HTTP-POST request #[serde(default, skip_serializing_if = "Vec::is_empty")] pub body: Vec, - /// Extra headers of a HTTP-GET or HTTP-POST request + /// Extra headers of a HTTP-GET, HTTP-HEAD or HTTP-POST request #[serde(default, skip_serializing_if = "Vec::is_empty")] pub headers: Vec<(String, String)>, } @@ -377,7 +377,7 @@ struct RADRetrieveSerializationHelperBincode { pub script: Vec, /// Body of a HTTP-POST request pub body: Vec, - /// Extra headers of a HTTP-GET or HTTP-POST request + /// Extra headers of a HTTP-GET, HTTP-HEAD or HTTP-POST request pub headers: Vec<(String, String)>, } diff --git a/net/src/client/http/mod.rs b/net/src/client/http/mod.rs index 55ae33e9e..3fca5533b 100644 --- a/net/src/client/http/mod.rs +++ b/net/src/client/http/mod.rs @@ -151,7 +151,7 @@ pub struct WitnetHttpResponse { impl WitnetHttpResponse { #[inline] - /// Simple wrapper around `isahc::Response::status`. + /// Simple wrapper around `isahc::Response`. pub fn inner(self) -> isahc::Response { self.res } diff --git a/node/Cargo.toml b/node/Cargo.toml index 796ca2737..57fe55a7a 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_node" -version = "1.6.7" +version = "2.0.0" authors = ["Witnet Foundation "] workspace = ".." description = "node component" diff --git a/rad/Cargo.toml b/rad/Cargo.toml index 806b62527..4cd5b3080 100644 --- a/rad/Cargo.toml +++ b/rad/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_rad" -version = "0.3.2" +version = "0.3.3" authors = ["Witnet Foundation "] edition = "2021" workspace = ".." @@ -10,6 +10,7 @@ description = "RAD component" rocksdb-backend = ["witnet_data_structures/rocksdb-backend"] [dependencies] +base64 = "0.21.5" cbor-codec = { git = "https://github.com/witnet/cbor-codec.git", branch = "feat/ldexpf-shim" } failure = "0.1.8" futures = "0.3.4" @@ -18,14 +19,17 @@ if_rust_version = "1.0.0" # the http crate is used to perform additional validations before passing arguments to the surf http client # the version of http must be kept in sync with the version used by surf http = "0.2.1" +jsonpath = "0.1.1" log = "0.4.8" minidom = { git = "https://github.com/witnet/xmpp-rs", rev = "bc8a33ff5da95ee4039ad7ee3376c100d9e35c74" } num_enum = "0.4.2" ordered-float = "3.0" rand = "0.7.3" +regex = "1.4.2" serde = "1.0.111" serde_cbor = "0.11.2" serde_json = "1.0.96" +slicestring = "0.3.2" # the url crate is used to perform additional validations before passing arguments to the surf http client # the version of url must be kept in sync with the version used by surf in the `witnet_net` crate url = "2.1.1" diff --git a/rad/src/error.rs b/rad/src/error.rs index a413f47c4..e27834450 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -38,6 +38,15 @@ pub enum RadError { description )] JsonParse { description: String }, + /// The given JSON path is not present in a JSON-stringified object + #[fail(display = "Failed to find JSON path `{}` from RadonString", path)] + JsonPathNotFound { path: String }, + /// Failed to parse a JSON path selector from a string value + #[fail( + display = "Failed to parse a JSON path from a string value: {:?}", + description + )] + JsonPathParse { description: String }, /// Failed to parse an object from a XML buffer #[fail( display = "Failed to parse an object from a XML buffer: {:?}", @@ -404,7 +413,10 @@ impl RadError { }) } } - None => Err(RadError::DecodeRadonErrorEmptyArray), + None => { + println!("None"); + Err(RadError::DecodeRadonErrorEmptyArray) + } } } @@ -431,6 +443,10 @@ impl RadError { } Ok(RadonError::new(match kind { + RadonErrors::BufferIsNotValue => { + let (description,) = deserialize_args(error_args)?; + RadError::BufferIsNotValue { description } + } RadonErrors::RequestTooManySources => RadError::RequestTooManySources, RadonErrors::ScriptTooManyCalls => RadError::ScriptTooManyCalls, RadonErrors::Overflow => RadError::Overflow, @@ -548,6 +564,7 @@ impl RadError { }; Some(serialize_args((message,))?) } + RadError::BufferIsNotValue { description } => Some(serialize_args((description,))?), _ => None, }; @@ -574,6 +591,7 @@ impl RadError { pub fn try_into_error_code(&self) -> Result { Ok(match self { RadError::Unknown => RadonErrors::Unknown, + RadError::BufferIsNotValue { .. } => RadonErrors::BufferIsNotValue, RadError::SourceScriptNotCBOR => RadonErrors::SourceScriptNotCBOR, RadError::SourceScriptNotArray => RadonErrors::SourceScriptNotArray, RadError::SourceScriptNotRADON => RadonErrors::SourceScriptNotRADON, @@ -753,6 +771,9 @@ mod tests { inner: None, message: Some("Only the message field is serialized".to_string()), }, + RadonErrors::BufferIsNotValue => RadError::BufferIsNotValue { + description: "Buffer is no value".to_string(), + }, // If this panics after adding a new `RadonTypes`, add a new example above _ => panic!("No example for {:?}", radon_errors), } diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 64c826780..c62706c34 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -78,7 +78,12 @@ pub fn try_data_request( .iter() .zip(inputs.iter()) .map(|(retrieve, input)| { - run_retrieval_with_data_report(retrieve, input, &mut retrieval_context, settings) + run_retrieval_with_data_report( + retrieve, + RadonTypes::from(RadonString::from(*input)), + &mut retrieval_context, + settings, + ) }) .collect() } else { @@ -160,42 +165,27 @@ pub fn try_data_request( } } -/// Handle HTTP-GET and HTTP-POST response with data, and return a `RadonReport`. -fn string_response_with_data_report( +/// Execute Radon Script using as input the RadonTypes value deserialized from a retrieval response +fn handle_response_with_data_report( retrieve: &RADRetrieve, - response: &str, + response: RadonTypes, context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { - let input = RadonTypes::from(RadonString::from(response)); let radon_script = unpack_radon_script(&retrieve.script)?; - - execute_radon_script(input, &radon_script, context, settings) -} - -/// Handle Rng response with data report -fn rng_response_with_data_report( - response: &str, - context: &mut ReportContext, -) -> Result> { - let response_bytes = response.as_bytes(); - let result = RadonTypes::from(RadonBytes::from(response_bytes.to_vec())); - - Ok(RadonReport::from_result(Ok(result), context)) + execute_radon_script(response, &radon_script, context, settings) } /// Run retrieval without performing any external network requests, return `Result`. pub fn run_retrieval_with_data_report( retrieve: &RADRetrieve, - response: &str, + response: RadonTypes, context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { match retrieve.kind { - RADType::HttpGet => string_response_with_data_report(retrieve, response, context, settings), - RADType::Rng => rng_response_with_data_report(response, context), - RADType::HttpPost => { - string_response_with_data_report(retrieve, response, context, settings) + RADType::HttpGet | RADType::HttpPost | RADType::HttpHead | RADType::Rng => { + handle_response_with_data_report(retrieve, response, context, settings) } _ => Err(RadError::UnknownRetrieval), } @@ -204,7 +194,7 @@ pub fn run_retrieval_with_data_report( /// Run retrieval without performing any external network requests, return `Result`. pub fn run_retrieval_with_data( retrieve: &RADRetrieve, - response: &str, + response: RadonTypes, settings: RadonScriptExecutionSettings, active_wips: ActiveWips, ) -> Result { @@ -214,7 +204,7 @@ pub fn run_retrieval_with_data( .map(RadonReport::into_inner) } -/// Handle generic HTTP (GET/POST) response +/// Handle generic HTTP (GET/POST/HEAD) response async fn http_response( retrieve: &RADRetrieve, context: &mut ReportContext, @@ -259,6 +249,10 @@ async fn http_response( WitnetHttpBody::from(retrieve.body.clone()), ) } + RADType::HttpHead => ( + builder.method("HEAD").uri(&retrieve.url), + WitnetHttpBody::empty(), + ), _ => panic!( "Called http_response with invalid retrieval kind {:?}", retrieve.kind @@ -282,6 +276,7 @@ async fn http_response( }) })?; + let start_ts = std::time::SystemTime::now(); let response = client .send(request) .await @@ -296,18 +291,51 @@ async fn http_response( }); } - // If at some point we want to support the retrieval of non-UTF8 data (e.g. raw bytes), this is - // where we need to decide how to read the response body - let (_parts, mut body) = response.into_parts(); - let mut response_string = String::default(); - body.read_to_string(&mut response_string) - .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), - })?; + let (parts, mut body) = response.into_parts(); - let result = run_retrieval_with_data_report(retrieve, &response_string, context, settings); + let response: RadonTypes; + if retrieve.kind != RADType::HttpHead && parts.headers.contains_key("accept-ranges") { + // http response is a binary stream + let mut response_bytes = Vec::::default(); + // todo: before reading the response buffer, an error should be thrown if it was too big + body.read_to_end(&mut response_bytes) + .await + .map_err(|x| RadError::HttpOther { + message: x.to_string(), + })?; + response = RadonTypes::from(RadonBytes::from(response_bytes)); + } else { + // response is a string + let mut response_string = String::default(); + match retrieve.kind { + RADType::HttpHead => { + response_string = format!("{:?}", parts.headers); + } + _ => { + body.read_to_string(&mut response_string) + .await + .map_err(|x| RadError::HttpOther { + message: x.to_string(), + })?; + } + } + response = RadonTypes::from(RadonString::from(response_string)); + } + + let result = + handle_response_with_data_report(retrieve, response, context, settings).map(|report| { + let completion_ts = std::time::SystemTime::now(); + RadonReport { + context: ReportContext { + start_time: Some(start_ts), + completion_time: Some(completion_ts), + ..report.context + }, + running_time: completion_ts.duration_since(start_ts).unwrap_or_default(), + ..report + } + }); match &result { Ok(report) => { log::debug!( @@ -318,7 +346,6 @@ async fn http_response( } Err(e) => log::debug!("Failed result for source {}: {:?}", retrieve.url, e), } - result } @@ -354,9 +381,10 @@ pub async fn run_retrieval_report( context.set_active_wips(active_wips); match retrieve.kind { - RADType::HttpGet => http_response(retrieve, context, settings, client).await, + RADType::HttpGet | RADType::HttpHead | RADType::HttpPost => { + http_response(retrieve, context, settings, client).await + } RADType::Rng => rng_response(context, settings).await, - RADType::HttpPost => http_response(retrieve, context, settings, client).await, _ => Err(RadError::UnknownRetrieval), } } @@ -649,7 +677,7 @@ fn validate_header(name: &str, value: &str) -> Result<()> { Err(RadError::InvalidHttpHeader { name: name.to_string(), value: value.to_string(), - error: error_message.to_string(), + error: error_message, }) } else { Ok(()) @@ -829,6 +857,39 @@ mod tests { use super::*; + #[test] + fn test_run_http_head_retrieval() { + let script_r = Value::Array(vec![ + Value::Integer(RadonOpCodes::StringParseJSONMap as i128), + Value::Array(vec![ + Value::Integer(RadonOpCodes::MapGetString as i128), + Value::Text("etag".to_string()), + ]), + ]); + let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); + + let retrieve = RADRetrieve { + kind: RADType::HttpHead, + url: "https://en.wikipedia.org/static/images/icons/wikipedia.png".to_string(), + script: packed_script_r, + body: vec![], + headers: vec![], + }; + let response_string = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#; + let result = run_retrieval_with_data( + &retrieve, + RadonTypes::from(RadonString::from(response_string)), + RadonScriptExecutionSettings::disable_all(), + current_active_wips(), + ) + .unwrap(); + + match result { + RadonTypes::String(_) => {} + err => panic!("Error in run_retrieval: {:?}", err), + } + } + #[test] fn test_run_retrieval() { let script_r = Value::Array(vec![ @@ -851,11 +912,10 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":500,"main":"Rain","description":"light rain","icon":"10d"}],"base":"stations","main":{"temp":17.59,"pressure":1022,"humidity":67,"temp_min":15,"temp_max":20},"visibility":10000,"wind":{"speed":3.6,"deg":260},"rain":{"1h":0.51},"clouds":{"all":20},"dt":1567501321,"sys":{"type":1,"id":1275,"message":0.0089,"country":"DE","sunrise":1567484402,"sunset":1567533129},"timezone":7200,"id":2950159,"name":"Berlin","cod":200}"#; - + let response_string = r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":500,"main":"Rain","description":"light rain","icon":"10d"}],"base":"stations","main":{"temp":17.59,"pressure":1022,"humidity":67,"temp_min":15,"temp_max":20},"visibility":10000,"wind":{"speed":3.6,"deg":260},"rain":{"1h":0.51},"clouds":{"all":20},"dt":1567501321,"sys":{"type":1,"id":1275,"message":0.0089,"country":"DE","sunrise":1567484402,"sunset":1567533129},"timezone":7200,"id":2950159,"name":"Berlin","cod":200}"#; let result = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -910,7 +970,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = "84"; + let response_string = "84"; let expected = RadonTypes::Float(RadonFloat::from(84)); let aggregate = RADAggregate { @@ -925,7 +985,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -948,7 +1008,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = "307"; + let response_string = "307"; let expected = RadonTypes::Float(RadonFloat::from(307)); let aggregate = RADAggregate { @@ -962,7 +1022,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1001,7 +1061,7 @@ mod tests { headers: vec![], }; // This response was modified because the original was about 100KB. - let response = r#"[{"estacion_nombre":"Pza. de España","estacion_numero":4,"fecha":"03092019","hora0":{"estado":"Pasado","valor":"00008"}}]"#; + let response_string = r#"[{"estacion_nombre":"Pza. de España","estacion_numero":4,"fecha":"03092019","hora0":{"estado":"Pasado","valor":"00008"}}]"#; let expected = RadonTypes::Float(RadonFloat::from(8)); let aggregate = RADAggregate { @@ -1015,7 +1075,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1045,7 +1105,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"PSOE":123,"PP":66,"Cs":57,"UP":42,"VOX":24,"ERC-SOBIRANISTES":15,"JxCAT-JUNTS":7,"PNV":6,"EH Bildu":4,"CCa-PNC":2,"NA+":2,"COMPROMÍS 2019":1,"PRC":1,"PACMA":0,"FRONT REPUBLICÀ":0,"BNG":0,"RECORTES CERO-GV":0,"NCa":0,"PACT":0,"ARA-MES-ESQUERRA":0,"GBAI":0,"PUM+J":0,"EN MAREA":0,"PCTE":0,"EL PI":0,"AxSI":0,"PCOE":0,"PCPE":0,"AVANT ADELANTE LOS VERDES":0,"EB":0,"CpM":0,"SOMOS REGIÓN":0,"PCPA":0,"PH":0,"UIG-SOM-CUIDES":0,"ERPV":0,"IZQP":0,"PCPC":0,"AHORA CANARIAS":0,"CxG":0,"PPSO":0,"CNV":0,"PREPAL":0,"C.Ex-C.R.Ex-P.R.Ex":0,"PR+":0,"P-LIB":0,"CILU-LINARES":0,"ANDECHA ASTUR":0,"JF":0,"PYLN":0,"FIA":0,"FE de las JONS":0,"SOLIDARIA":0,"F8":0,"DPL":0,"UNIÓN REGIONALISTA":0,"centrados":0,"DP":0,"VOU":0,"PDSJE-UDEC":0,"IZAR":0,"RISA":0,"C 21":0,"+MAS+":0,"UDT":0}"#; + let response_string = r#"{"PSOE":123,"PP":66,"Cs":57,"UP":42,"VOX":24,"ERC-SOBIRANISTES":15,"JxCAT-JUNTS":7,"PNV":6,"EH Bildu":4,"CCa-PNC":2,"NA+":2,"COMPROMÍS 2019":1,"PRC":1,"PACMA":0,"FRONT REPUBLICÀ":0,"BNG":0,"RECORTES CERO-GV":0,"NCa":0,"PACT":0,"ARA-MES-ESQUERRA":0,"GBAI":0,"PUM+J":0,"EN MAREA":0,"PCTE":0,"EL PI":0,"AxSI":0,"PCOE":0,"PCPE":0,"AVANT ADELANTE LOS VERDES":0,"EB":0,"CpM":0,"SOMOS REGIÓN":0,"PCPA":0,"PH":0,"UIG-SOM-CUIDES":0,"ERPV":0,"IZQP":0,"PCPC":0,"AHORA CANARIAS":0,"CxG":0,"PPSO":0,"CNV":0,"PREPAL":0,"C.Ex-C.R.Ex-P.R.Ex":0,"PR+":0,"P-LIB":0,"CILU-LINARES":0,"ANDECHA ASTUR":0,"JF":0,"PYLN":0,"FIA":0,"FE de las JONS":0,"SOLIDARIA":0,"F8":0,"DPL":0,"UNIÓN REGIONALISTA":0,"centrados":0,"DP":0,"VOU":0,"PDSJE-UDEC":0,"IZAR":0,"RISA":0,"C 21":0,"+MAS+":0,"UDT":0}"#; let expected = RadonTypes::Float(RadonFloat::from(123)); let aggregate = RADAggregate { @@ -1059,7 +1119,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1100,10 +1160,10 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"event":{"homeTeam":{"name":"Ryazan-VDV","slug":"ryazan-vdv","gender":"F","national":false,"id":171120,"shortName":"Ryazan-VDV","subTeams":[]},"awayTeam":{"name":"Olympique Lyonnais","slug":"olympique-lyonnais","gender":"F","national":false,"id":26245,"shortName":"Lyon","subTeams":[]},"homeScore":{"current":0,"display":0,"period1":0,"normaltime":0},"awayScore":{"current":9,"display":9,"period1":5,"normaltime":9}}}"#; + let response_string = r#"{"event":{"homeTeam":{"name":"Ryazan-VDV","slug":"ryazan-vdv","gender":"F","national":false,"id":171120,"shortName":"Ryazan-VDV","subTeams":[]},"awayTeam":{"name":"Olympique Lyonnais","slug":"olympique-lyonnais","gender":"F","national":false,"id":26245,"shortName":"Lyon","subTeams":[]},"homeScore":{"current":0,"display":0,"period1":0,"normaltime":0},"awayScore":{"current":9,"display":9,"period1":5,"normaltime":9}}}"#; let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1526,6 +1586,58 @@ mod tests { } } + #[test] + fn test_try_data_request_http_get_non_ascii_header_key() { + let script_r = Value::Array(vec![]); + let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); + let body = Vec::from(String::from("")); + let headers = vec![("ñ", "value")]; + let headers = headers + .into_iter() + .map(|(a, b)| (a.to_string(), b.to_string())) + .collect(); + let request = RADRequest { + time_lock: 0, + retrieve: vec![RADRetrieve { + kind: RADType::HttpGet, + url: String::from("http://127.0.0.1"), + script: packed_script_r, + body, + headers, + }], + aggregate: RADAggregate { + filters: vec![], + reducer: RadonReducers::Mode as u32, + }, + tally: RADTally { + filters: vec![], + reducer: RadonReducers::Mode as u32, + }, + }; + let report = try_data_request( + &request, + RadonScriptExecutionSettings::enable_all(), + None, + None, + ); + let tally_result = report.tally.into_inner(); + + assert_eq!( + tally_result, + RadonTypes::RadonError( + RadonError::try_from(RadError::UnhandledIntercept { + inner: Some(Box::new(RadError::InvalidHttpHeader { + name: "ñ".to_string(), + value: "value".to_string(), + error: "invalid HTTP header name".to_string() + })), + message: None + }) + .unwrap() + ) + ); + } + #[test] fn test_try_data_request_http_post_non_ascii_header_key() { let script_r = Value::Array(vec![]); diff --git a/rad/src/operators/array.rs b/rad/src/operators/array.rs index b4aac9674..1f3c809cd 100644 --- a/rad/src/operators/array.rs +++ b/rad/src/operators/array.rs @@ -1,18 +1,20 @@ use std::{ clone::Clone, convert::{TryFrom, TryInto}, - iter, }; use serde_cbor::value::{from_value, Value}; -use witnet_data_structures::radon_report::{RadonReport, ReportContext, Stage}; +use witnet_data_structures::radon_report::ReportContext; use crate::{ error::RadError, filters::{self, RadonFilters}, - operators::{string, RadonOpCodes}, + operators::string, reducers::{self, RadonReducers}, - script::{execute_radon_script, unpack_subscript, RadonCall, RadonScriptExecutionSettings}, + script::{ + execute_radon_script, partial_results_extract, unpack_subscript, + RadonScriptExecutionSettings, + }, types::{array::RadonArray, integer::RadonInteger, string::RadonString, RadonType, RadonTypes}, }; @@ -109,6 +111,38 @@ fn get_numeric_string(input: &RadonArray, args: &[Value]) -> Result Result { + // Join not applicable if the input array is not homogeneous + if !input.is_homogeneous() { + return Err(RadError::UnsupportedOpNonHomogeneous { + operator: "ArrayJoin".to_string(), + }); + } + let separator = if !args.is_empty() { + from_value::(args[0].to_owned()).unwrap_or_default() + } else { + String::from("") + }; + match input.value().first() { + Some(RadonTypes::String(_)) => { + let string_list: Vec = input + .value() + .into_iter() + .map(|item| RadonString::try_from(item).unwrap_or_default().value()) + .collect(); + Ok(RadonTypes::from(RadonString::from( + string_list.join(separator.as_str()), + ))) + } + Some(first_item) => Err(RadError::UnsupportedOperator { + input_type: first_item.radon_type_name().to_string(), + operator: "ArrayJoin".to_string(), + args: Some(args.to_vec()), + }), + _ => Err(RadError::EmptyArray), + } +} + pub fn map( input: &RadonArray, args: &[Value], @@ -221,6 +255,52 @@ pub fn filter( } } +pub fn pick( + input: &RadonArray, + args: &[Value], + _context: &mut ReportContext, +) -> Result { + let not_found = |index: usize| RadError::ArrayIndexOutOfBounds { + index: i32::try_from(index).unwrap(), + }; + + let wrong_args = || RadError::WrongArguments { + input_type: RadonArray::radon_type_name(), + operator: "Pick".to_string(), + args: args.to_vec(), + }; + + let mut indexes = vec![]; + if args.len() > 1 { + return Err(wrong_args()); + } else { + let first_arg = args.get(0).ok_or_else(wrong_args)?; + match first_arg { + Value::Array(values) => { + for value in values.iter() { + let index = from_value::(value.clone()).map_err(|_| wrong_args())?; + indexes.push(index); + } + } + Value::Integer(_) => { + let index = from_value::(first_arg.clone()).map_err(|_| wrong_args())?; + indexes.push(index); + } + _ => return Err(wrong_args()), + }; + } + + let mut output_vec: Vec = vec![]; + for index in indexes { + if let Some(value) = input.value().get(index) { + output_vec.push(value.clone()); + } else { + return Err(not_found(index)); + } + } + Ok(RadonTypes::from(RadonArray::from(output_vec))) +} + pub fn sort( input: &RadonArray, args: &[Value], @@ -299,27 +379,6 @@ pub fn sort( Ok(RadonArray::from(result).into()) } -fn partial_results_extract( - subscript: &[RadonCall], - reports: &[RadonReport], - context: &mut ReportContext, -) { - if let Stage::Retrieval(metadata) = &mut context.stage { - metadata.subscript_partial_results.push(subscript.iter().chain(iter::once(&(RadonOpCodes::Fail, None))).enumerate().map(|(index, _)| - reports - .iter() - .map(|report| - report.partial_results - .as_ref() - .expect("Execution reports from applying subscripts are expected to contain partial results") - .get(index) - .expect("Execution reports from applying same subscript on multiple values should contain the same number of partial results") - .clone() - ).collect::>() - ).collect::>>()); - } -} - pub fn transpose(input: &RadonArray) -> Result { let mut v = vec![]; let mut prev_len = None; @@ -406,15 +465,15 @@ pub mod legacy { mod tests { use std::collections::BTreeMap; - use witnet_data_structures::radon_report::RetrievalMetadata; + use witnet_data_structures::radon_report::{RetrievalMetadata, Stage}; use crate::{ error::RadError, operators::{ Operable, RadonOpCodes::{ - IntegerGreaterThan, IntegerMultiply, MapGetBoolean, MapGetFloat, MapGetInteger, - MapGetString, + self, IntegerGreaterThan, IntegerMultiply, MapGetBoolean, MapGetFloat, + MapGetInteger, MapGetString, }, }, types::{ diff --git a/rad/src/operators/bytes.rs b/rad/src/operators/bytes.rs index 174e38b4d..65ecce64f 100644 --- a/rad/src/operators/bytes.rs +++ b/rad/src/operators/bytes.rs @@ -1,14 +1,31 @@ +use base64::Engine; use serde_cbor::value::{from_value, Value}; use std::convert::TryFrom; use crate::{ error::RadError, hash_functions::{self, RadonHashFunctions}, - types::{bytes::RadonBytes, string::RadonString, RadonType}, + types::{ + bytes::{RadonBytes, RadonBytesEncoding}, + integer::RadonInteger, + string::RadonString, + RadonType, + }, }; -pub fn to_string(input: &RadonBytes) -> Result { - RadonString::try_from(Value::Text(hex::encode(input.value()))) +pub fn as_integer(input: &RadonBytes) -> Result { + let input_value_len = input.value().len(); + match input_value_len { + 1..=16 => { + let mut bytes_array = [0u8; 16]; + bytes_array[16 - input_value_len..].copy_from_slice(&input.value()); + Ok(RadonInteger::from(i128::from_be_bytes(bytes_array))) + } + 17.. => Err(RadError::ParseInt { + message: "Input buffer too big".to_string(), + }), + _ => Err(RadError::EmptyArray), + } } pub fn hash(input: &RadonBytes, args: &[Value]) -> Result { @@ -27,14 +44,70 @@ pub fn hash(input: &RadonBytes, args: &[Value]) -> Result Ok(RadonBytes::from(digest)) } + +pub fn length(input: &RadonBytes) -> RadonInteger { + RadonInteger::from(input.value().len() as i128) +} + +pub fn slice(input: &RadonBytes, args: &[Value]) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "BytesSlice".to_string(), + args: args.to_vec(), + }; + let end_index = input.value().len(); + if end_index > 0 { + let start_index = from_value::(args[0].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; + let mut slice = input.value().as_slice().split_at(start_index).1.to_vec(); + if args.len() == 2 { + let end_index = from_value::(args[1].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; + slice.truncate(end_index - start_index); + } + Ok(RadonBytes::from(slice)) + } else { + Err(wrong_args()) + } +} + +pub fn stringify(input: &RadonBytes, args: &Option>) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "Stringify".to_string(), + args: args.to_owned().unwrap_or_default().to_vec(), + }; + let mut bytes_encoding = RadonBytesEncoding::Hex; + match args { + Some(args) => { + if !args.is_empty() { + let arg = args.first().ok_or_else(wrong_args)?.to_owned(); + let bytes_encoding_u8 = from_value::(arg).map_err(|_| wrong_args())?; + bytes_encoding = + RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; + } + } + _ => (), + } + match bytes_encoding { + RadonBytesEncoding::Hex => RadonString::try_from(Value::Text(hex::encode(input.value()))), + RadonBytesEncoding::Base64 => RadonString::try_from(Value::Text( + base64::engine::general_purpose::STANDARD.encode(input.value()), + )), + } +} + #[cfg(test)] mod tests { use super::*; #[test] - fn test_bytes_to_string() { + fn test_bytes_stringify() { let input = RadonBytes::from(vec![0x01, 0x02, 0x03]); - let output = to_string(&input).unwrap().value(); + let valid_args = Some(vec![Value::from(0x00)]); + let output = stringify(&input, &valid_args).unwrap().value(); let valid_expected = "010203".to_string(); diff --git a/rad/src/operators/integer.rs b/rad/src/operators/integer.rs index 407522dfa..5fda41039 100644 --- a/rad/src/operators/integer.rs +++ b/rad/src/operators/integer.rs @@ -5,8 +5,8 @@ use serde_cbor::value::{from_value, Value}; use crate::{ error::RadError, types::{ - boolean::RadonBoolean, float::RadonFloat, integer::RadonInteger, string::RadonString, - RadonType, + boolean::RadonBoolean, bytes::RadonBytes, float::RadonFloat, integer::RadonInteger, + string::RadonString, RadonType, }, }; @@ -20,6 +20,20 @@ pub fn absolute(input: &RadonInteger) -> Result { } } +pub fn to_bytes(input: RadonInteger) -> Result { + let mut bytes_array = [0u8; 16]; + bytes_array.copy_from_slice(&input.value().to_be_bytes()); + let mut leading_zeros = 0; + for i in 0..bytes_array.len() { + if bytes_array[i] != 0u8 { + break; + } else { + leading_zeros += 1; + } + } + Ok(RadonBytes::from(bytes_array[leading_zeros..].to_vec())) +} + pub fn to_float(input: RadonInteger) -> Result { RadonFloat::try_from(Value::Integer(input.value())) } diff --git a/rad/src/operators/map.rs b/rad/src/operators/map.rs index c81da9eef..b399c2584 100644 --- a/rad/src/operators/map.rs +++ b/rad/src/operators/map.rs @@ -1,13 +1,88 @@ -use std::convert::TryInto; +use std::{collections::BTreeMap, convert::TryInto}; use serde_cbor::value::{from_value, Value}; +use witnet_data_structures::radon_report::ReportContext; use crate::{ error::RadError, operators::string, + script::{ + execute_radon_script, partial_results_extract, unpack_subscript, + RadonScriptExecutionSettings, + }, types::{array::RadonArray, map::RadonMap, string::RadonString, RadonType, RadonTypes}, }; +pub fn alter( + input: &RadonMap, + args: &[Value], + context: &mut ReportContext, +) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonMap::radon_type_name(), + operator: "Alter".to_string(), + args: args.to_vec(), + }; + + let first_arg = args.get(0).ok_or_else(wrong_args)?; + let mut input_keys = vec![]; + match first_arg { + Value::Array(keys) => { + for key in keys.iter() { + let key_string = from_value::(key.to_owned()).map_err(|_| wrong_args())?; + input_keys.push(key_string); + } + } + Value::Text(key) => { + input_keys.push(key.clone()); + } + _ => return Err(wrong_args()), + }; + + let subscript = args.get(1).ok_or_else(wrong_args)?; + match subscript { + Value::Array(_arg) => { + let subscript_err = |e| RadError::Subscript { + input_type: "RadonMap".to_string(), + operator: "Alter".to_string(), + inner: Box::new(e), + }; + let subscript = unpack_subscript(subscript).map_err(subscript_err)?; + + let not_found = |key_str: &str| RadError::MapKeyNotFound { + key: String::from(key_str), + }; + + let input_map = input.value(); + let mut output_map = input.value().clone(); + let mut reports = vec![]; + + let settings = RadonScriptExecutionSettings::tailored_to_stage(&context.stage); + for key in input_keys { + let value = input_map + .get(key.as_str()) + .ok_or_else(|| not_found(key.as_str()))?; + let report = + execute_radon_script(value.clone(), subscript.as_slice(), context, settings)?; + // If there is an error while altering value, short-circuit and bubble up the error as it comes + // from the radon script execution + if let RadonTypes::RadonError(error) = &report.result { + return Err(error.clone().into_inner()); + } else { + output_map.insert(key, report.result.clone()); + } + reports.push(report); + } + + // Extract the partial results from the reports and put them in the execution context if needed + partial_results_extract(&subscript, &reports, context); + + Ok(RadonMap::from(output_map)) + } + _ => Err(wrong_args()), + } +} + fn inner_get(input: &RadonMap, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonMap::radon_type_name(), @@ -81,6 +156,56 @@ pub fn values(input: &RadonMap) -> RadonArray { RadonArray::from(v) } +pub fn pick(input: &RadonMap, args: &[Value]) -> Result { + let not_found = |key_str: &str| RadError::MapKeyNotFound { + key: String::from(key_str), + }; + + let wrong_args = || RadError::WrongArguments { + input_type: RadonMap::radon_type_name(), + operator: "Pick".to_string(), + args: args.to_vec(), + }; + + let mut input_keys = vec![]; + if args.len() > 1 { + return Err(wrong_args()); + } else { + let first_arg = args.get(0).ok_or_else(wrong_args)?; + match first_arg { + Value::Array(keys) => { + for key in keys.iter() { + let key_string = + from_value::(key.to_owned()).map_err(|_| wrong_args())?; + input_keys.push(key_string); + } + } + Value::Text(key) => { + input_keys.push(key.clone()); + } + _ => return Err(wrong_args()), + }; + } + + let mut output_map = BTreeMap::::default(); + for key in input_keys { + if let Some(value) = input.value().get(&key) { + output_map.insert(key, value.clone()); + } else { + return Err(not_found(key.as_str())); + } + } + Ok(RadonMap::from(output_map)) +} + +pub fn stringify(input: &RadonMap) -> Result { + let json_string = serde_json::to_string(&input.value()).map_err(|_| RadError::Decode { + from: "RadonMap", + to: "RadonString", + })?; + Ok(RadonString::from(json_string)) +} + /// This module was introduced for encapsulating the interim legacy logic before WIP-0024 is /// introduced, for the sake of maintainability. /// diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 6964f2623..f169c2e2d 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -29,7 +29,7 @@ pub enum RadonOpCodes { // Array operator codes (start at 0x10) ArrayCount = 0x10, ArrayFilter = 0x11, - // ArrayFlatten = 0x12, + ArrayJoin = 0x12, ArrayGetArray = 0x13, ArrayGetBoolean = 0x14, ArrayGetBytes = 0x15, @@ -41,49 +41,50 @@ pub enum RadonOpCodes { ArrayReduce = 0x1B, // ArraySome = 0x1C, ArraySort = 0x1D, - // ArrayTake = 0x1E, + ArrayPick = 0x1E, /////////////////////////////////////////////////////////////////////// // Boolean operator codes (start at 0x20) - BooleanAsString = 0x20, // BooleanMatch = 0x21, BooleanNegate = 0x22, + BooleanToString = 0x20, /////////////////////////////////////////////////////////////////////// // Bytes operator codes (start at 0x30) - BytesAsString = 0x30, + BytesAsInteger = 0x32, BytesHash = 0x31, + BytesLength = 0x34, + BytesSlice = 0x3C, + BytesStringify = 0x30, /////////////////////////////////////////////////////////////////////// // Integer operator codes (start at 0x40) IntegerAbsolute = 0x40, - IntegerAsFloat = 0x41, - IntegerAsString = 0x42, IntegerGreaterThan = 0x43, IntegerLessThan = 0x44, - // IntegerMatch = 0x45, IntegerModulo = 0x46, IntegerMultiply = 0x47, IntegerNegate = 0x48, IntegerPower = 0x49, - // IntegerReciprocal = 0x4A, + IntegerToBytes = 0x4A, + IntegerToFloat = 0x41, + IntegerToString = 0x42, // IntegerSum = 0x4B, /////////////////////////////////////////////////////////////////////// // Float operator codes (start at 0x50) FloatAbsolute = 0x50, - FloatAsString = 0x51, FloatCeiling = 0x52, - FloatGreaterThan = 0x53, FloatFloor = 0x54, + FloatGreaterThan = 0x53, FloatLessThan = 0x55, FloatModulo = 0x56, FloatMultiply = 0x57, FloatNegate = 0x58, FloatPower = 0x59, - // FloatReciprocal = 0x5A, FloatRound = 0x5B, + FloatToString = 0x51, // FloatSum = 0x5C, FloatTruncate = 0x5D, /////////////////////////////////////////////////////////////////////// // Map operator codes (start at 0x60) - // MapEntries = 0x60, + MapStringify = 0x60, MapGetArray = 0x61, MapGetBoolean = 0x62, MapGetBytes = 0x63, @@ -93,10 +94,13 @@ pub enum RadonOpCodes { MapGetString = 0x67, MapKeys = 0x68, MapValues = 0x69, + //MapEntries = 0x6A, + MapAlter = 0x6B, + MapPick = 0x6E, /////////////////////////////////////////////////////////////////////// // String operator codes (start at 0x70) StringAsBoolean = 0x70, - // StringAsBytes = 0x71, + StringAsBytes = 0x71, StringAsFloat = 0x72, StringAsInteger = 0x73, StringLength = 0x74, @@ -104,6 +108,9 @@ pub enum RadonOpCodes { StringParseJSONArray = 0x76, StringParseJSONMap = 0x77, StringParseXMLMap = 0x78, + StringReplace = 0x7B, + StringSlice = 0x7C, + StringSplit = 0x7D, StringToLowerCase = 0x79, StringToUpperCase = 0x7A, } diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index e23914e99..b05b28f49 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -4,15 +4,27 @@ use std::{ str::FromStr, }; +use base64::Engine; +use jsonpath::Selector; use serde_cbor::value::{from_value, Value}; use serde_json::Value as JsonValue; +use regex::Regex; +use slicestring::Slice; +use witnet_data_structures::radon_error::RadonError; + use crate::{ error::RadError, hash_functions::{self, RadonHashFunctions}, types::{ - array::RadonArray, boolean::RadonBoolean, bytes::RadonBytes, float::RadonFloat, - integer::RadonInteger, map::RadonMap, string::RadonString, RadonType, RadonTypes, + array::RadonArray, + boolean::RadonBoolean, + bytes::{RadonBytes, RadonBytesEncoding}, + float::RadonFloat, + integer::RadonInteger, + map::RadonMap, + string::RadonString, + RadonType, RadonTypes, }, }; @@ -20,24 +32,239 @@ const MAX_DEPTH: u8 = 20; const DEFAULT_THOUSANDS_SEPARATOR: &str = ","; const DEFAULT_DECIMAL_SEPARATOR: &str = "."; +pub fn as_bool(input: &RadonString) -> Result { + let str_value = radon_trim(input); + bool::from_str(&str_value) + .map(RadonBoolean::from) + .map_err(Into::into) +} + +pub fn as_bytes(input: &RadonString, args: &Option>) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "AsBytes".to_string(), + args: args.to_owned().unwrap_or_default().to_vec(), + }; + let mut input_string = input.value(); + if input_string.starts_with("0x") { + input_string = input_string.slice(2..); + } + if input_string.len() % 2 != 0 { + input_string.insert(0, '0'); + } + let mut bytes_encoding = RadonBytesEncoding::Hex; + match args { + Some(args) => { + if !args.is_empty() { + let arg = args.first().ok_or_else(wrong_args)?.to_owned(); + let bytes_encoding_u8 = from_value::(arg).map_err(|_| wrong_args())?; + bytes_encoding = + RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; + } + } + _ => (), + } + match bytes_encoding { + RadonBytesEncoding::Hex => Ok(RadonBytes::from( + hex::decode(input_string.as_str()).map_err(|_err| RadError::Decode { + from: "RadonString", + to: "RadonBytes", + })?, + )), + RadonBytesEncoding::Base64 => Ok(RadonBytes::from( + base64::engine::general_purpose::STANDARD + .decode(input.value()) + .map_err(|_err| RadError::Decode { + from: "RadonString", + to: "RadonBytes", + })?, + )), + } +} + +/// Converts a `RadonString` into a `RadonFloat`, provided that the input string actually represents +/// a valid floating point number. +pub fn as_float(input: &RadonString, args: &Option>) -> Result { + f64::from_str(&as_numeric_string( + input, + args.as_deref().unwrap_or_default(), + )) + .map(RadonFloat::from) + .map_err(Into::into) +} + +/// Converts a `RadonString` into a `RadonFloat`, provided that the input string actually represents +/// a valid integer number. +pub fn as_integer( + input: &RadonString, + args: &Option>, +) -> Result { + i128::from_str(&as_numeric_string( + input, + args.as_deref().unwrap_or_default(), + )) + .map(RadonInteger::from) + .map_err(Into::into) +} + +/// Converts a `RadonString` into a `String` containing a numeric value, provided that the input +/// string actually represents a valid number. +pub fn as_numeric_string(input: &RadonString, args: &[Value]) -> String { + let str_value = radon_trim(input); + let (thousands_separator, decimal_separator) = read_separators_from_args(args); + + replace_separators(str_value, thousands_separator, decimal_separator) +} + +pub fn hash(input: &RadonString, args: &[Value]) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "Hash".to_string(), + args: args.to_vec(), + }; + + let input_string = input.value(); + let input_bytes = input_string.as_bytes(); + + let arg = args.first().ok_or_else(wrong_args)?.to_owned(); + let hash_function_integer = from_value::(arg).map_err(|_| wrong_args())?; + let hash_function_code = + RadonHashFunctions::try_from(hash_function_integer).map_err(|_| wrong_args())?; + + let digest = hash_functions::hash(input_bytes, hash_function_code)?; + let hex_string = hex::encode(digest); + + Ok(RadonString::from(hex_string)) +} + +pub fn length(input: &RadonString) -> RadonInteger { + RadonInteger::from(input.value().len() as i128) +} + /// Parse `RadonTypes` from a JSON-encoded `RadonString`. pub fn parse_json(input: &RadonString) -> Result { let json_value: JsonValue = serde_json::from_str(&input.value()).map_err(|err| RadError::JsonParse { description: err.to_string(), })?; - RadonTypes::try_from(json_value) } -pub fn parse_json_map(input: &RadonString) -> Result { - let item = parse_json(input)?; - item.try_into() +pub fn parse_json_map( + input: &RadonString, + args: &Option>, +) -> Result { + let not_found = |json_path: &str| RadError::JsonPathNotFound { + path: String::from(json_path), + }; + + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "ParseJsonMap".to_string(), + args: args.to_owned().unwrap_or_default(), + }; + + let json_input: JsonValue = + serde_json::from_str(&input.value()).map_err(|err| RadError::JsonParse { + description: err.to_string(), + })?; + + match args.to_owned().unwrap_or_default().get(0) { + Some(Value::Text(json_path)) => { + let selector = + Selector::new(json_path.as_str()).map_err(|err| RadError::JsonPathParse { + description: err.to_string(), + })?; + let item = selector + .find(&json_input) + .next() + .ok_or_else(|| not_found(json_path.as_str()))?; + RadonTypes::try_from(item.to_owned())?.try_into() + } + None => RadonTypes::try_from(json_input)?.try_into(), + _ => Err(wrong_args()), + } } -pub fn parse_json_array(input: &RadonString) -> Result { - let item = parse_json(input)?; - item.try_into() +pub fn parse_json_array( + input: &RadonString, + args: &Option>, +) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "ParseJsonArray".to_string(), + args: args.to_owned().unwrap_or_default(), + }; + + let json_input: JsonValue = + serde_json::from_str(&input.value()).map_err(|err| RadError::JsonParse { + description: err.to_string(), + })?; + + match args.to_owned().unwrap_or_default().get(0) { + Some(Value::Array(values)) => { + let mut items: Vec = vec![]; + for path in values { + if let Value::Text(json_path) = path { + let selector = Selector::new(json_path.as_str()).map_err(|err| { + RadError::JsonPathParse { + description: err.to_string(), + } + })?; + let mut subitems: Vec = + selector.find(&json_input).map(into_radon_types).collect(); + if subitems.len() > 1 { + items.insert(items.len(), RadonArray::from(subitems).into()); + } else { + items.append(subitems.as_mut()); + } + } else { + return Err(wrong_args()); + } + } + Ok(RadonArray::from(items)) + } + Some(Value::Text(json_path)) => { + let selector = + Selector::new(json_path.as_str()).map_err(|err| RadError::JsonPathParse { + description: err.to_string(), + })?; + let items: Vec = selector.find(&json_input).map(into_radon_types).collect(); + Ok(RadonArray::from(items)) + } + None => RadonTypes::try_from(json_input)?.try_into(), + _ => Err(wrong_args()), + } +} + +fn into_radon_types(value: &serde_json::Value) -> RadonTypes { + match value { + serde_json::Value::Number(value) => { + if value.is_f64() { + RadonTypes::from(RadonFloat::from(value.as_f64().unwrap_or_default())) + } else { + RadonTypes::from(RadonInteger::from( + value.as_i64().unwrap_or_default() as i128 + )) + } + } + serde_json::Value::Bool(value) => RadonTypes::from(RadonBoolean::from(*value)), + serde_json::Value::String(value) => RadonTypes::from(RadonString::from(value.clone())), + serde_json::Value::Object(entries) => { + let mut object: BTreeMap = BTreeMap::new(); + for (key, value) in entries { + object.insert(key.clone(), into_radon_types(value)); + } + RadonTypes::from(RadonMap::from(object)) + } + serde_json::Value::Array(values) => { + let items: Vec = values.iter().map(into_radon_types).collect(); + RadonTypes::from(RadonArray::from(items)) + } + _ => RadonTypes::from(RadonError::new(RadError::JsonParse { + description: value.to_string(), + })), + } } fn add_children( @@ -150,84 +377,70 @@ pub fn radon_trim(input: &RadonString) -> String { } } -pub fn to_bool(input: &RadonString) -> Result { - let str_value = radon_trim(input); - bool::from_str(&str_value) - .map(RadonBoolean::from) - .map_err(Into::into) -} - -/// Converts a `RadonString` into a `RadonFloat`, provided that the input string actually represents -/// a valid floating point number. -pub fn as_float(input: &RadonString, args: &Option>) -> Result { - f64::from_str(&as_numeric_string( - input, - args.as_deref().unwrap_or_default(), - )) - .map(RadonFloat::from) - .map_err(Into::into) -} - -/// Converts a `RadonString` into a `RadonFloat`, provided that the input string actually represents -/// a valid integer number. -pub fn as_integer( - input: &RadonString, - args: &Option>, -) -> Result { - i128::from_str(&as_numeric_string( - input, - args.as_deref().unwrap_or_default(), - )) - .map(RadonInteger::from) - .map_err(Into::into) -} - -/// Converts a `RadonString` into a `String` containing a numeric value, provided that the input -/// string actually represents a valid number. -pub fn as_numeric_string(input: &RadonString, args: &[Value]) -> String { - let str_value = radon_trim(input); - let (thousands_separator, decimal_separator) = read_separators_from_args(args); - - replace_separators(str_value, thousands_separator, decimal_separator) -} - -pub fn length(input: &RadonString) -> RadonInteger { - RadonInteger::from(input.value().len() as i128) -} - -pub fn to_lowercase(input: &RadonString) -> RadonString { - RadonString::from(input.value().as_str().to_lowercase()) +pub fn replace(input: &RadonString, args: &[Value]) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "StringReplace".to_string(), + args: args.to_vec(), + }; + let regex = RadonString::try_from(args.first().ok_or_else(wrong_args)?.to_owned())?; + let replacement = RadonString::try_from(args.get(1).ok_or_else(wrong_args)?.to_owned())?; + Ok(RadonString::from(input.value().as_str().replace( + regex.value().as_str(), + replacement.value().as_str(), + ))) } -pub fn to_uppercase(input: &RadonString) -> RadonString { - RadonString::from(input.value().as_str().to_uppercase()) +pub fn slice(input: &RadonString, args: &[Value]) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "StringSlice".to_string(), + args: args.to_vec(), + }; + let mut end_index: usize = input.value().len(); + match args.len() { + 2 => { + let start_index = from_value::(args[0].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; + end_index = from_value::(args[1].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; + Ok(RadonString::from( + input.value().as_str().slice(start_index..end_index), + )) + } + 1 => { + let start_index = from_value::(args[0].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; + Ok(RadonString::from( + input.value().as_str().slice(start_index..end_index), + )) + } + _ => Err(wrong_args()), + } } -pub fn hash(input: &RadonString, args: &[Value]) -> Result { +pub fn split(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), - operator: "Hash".to_string(), + operator: "StringSplit".to_string(), args: args.to_vec(), }; - - let input_string = input.value(); - let input_bytes = input_string.as_bytes(); - - let arg = args.first().ok_or_else(wrong_args)?.to_owned(); - let hash_function_integer = from_value::(arg).map_err(|_| wrong_args())?; - let hash_function_code = - RadonHashFunctions::try_from(hash_function_integer).map_err(|_| wrong_args())?; - - let digest = hash_functions::hash(input_bytes, hash_function_code)?; - let hex_string = hex::encode(digest); - - Ok(RadonString::from(hex_string)) + let pattern = RadonString::try_from(args.first().ok_or_else(wrong_args)?.to_owned())?; + let parts: Vec = Regex::new(pattern.value().as_str()) + .unwrap() + .split(input.value().as_str()) + .map(|part| RadonTypes::from(RadonString::from(part))) + .collect(); + Ok(RadonArray::from(parts)) } pub fn string_match(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), - operator: "String match".to_string(), + operator: "StringMatch".to_string(), args: args.to_vec(), }; @@ -281,6 +494,14 @@ pub fn string_match(input: &RadonString, args: &[Value]) -> Result RadonString { + RadonString::from(input.value().as_str().to_lowercase()) +} + +pub fn to_uppercase(input: &RadonString) -> RadonString { + RadonString::from(input.value().as_str().to_uppercase()) +} + /// Replace thousands and decimals separators in a `String`. #[inline] pub fn replace_separators( @@ -357,7 +578,7 @@ mod tests { #[test] fn test_parse_json_map() { let json_map = RadonString::from(r#"{ "Hello": "world" }"#); - let output = parse_json_map(&json_map).unwrap(); + let output = parse_json_map(&json_map, &None).unwrap(); let key = "Hello"; let value = RadonTypes::String(RadonString::from("world")); @@ -517,7 +738,7 @@ mod tests { fn test_parse_json_map_with_null_entries() { // When parsing a JSON map, any keys with value `null` are ignored let json_map = RadonString::from(r#"{ "Hello": "world", "Bye": null }"#); - let output = parse_json_map(&json_map).unwrap(); + let output = parse_json_map(&json_map, &None).unwrap(); let key = "Hello"; let value = RadonTypes::String(RadonString::from("world")); @@ -531,7 +752,7 @@ mod tests { #[test] fn test_parse_json_map_fail() { let invalid_json = RadonString::from(r#"{ "Hello": }"#); - let output = parse_json_map(&invalid_json).unwrap_err(); + let output = parse_json_map(&invalid_json, &None).unwrap_err(); let expected_err = RadError::JsonParse { description: "expected value at line 1 column 13".to_string(), @@ -539,7 +760,7 @@ mod tests { assert_eq!(output, expected_err); let json_array = RadonString::from(r#"[1,2,3]"#); - let output = parse_json_map(&json_array).unwrap_err(); + let output = parse_json_map(&json_array, &None).unwrap_err(); let expected_err = RadError::Decode { from: "cbor::value::Value", to: RadonMap::radon_type_name(), @@ -550,7 +771,7 @@ mod tests { #[test] fn test_parse_json_array() { let json_array = RadonString::from(r#"[1,2,3]"#); - let output = parse_json_array(&json_array).unwrap(); + let output = parse_json_array(&json_array, &None).unwrap(); let expected_output = RadonArray::from(vec![ RadonTypes::Integer(RadonInteger::from(1)), @@ -565,7 +786,7 @@ mod tests { fn test_parse_json_array_with_null_entries() { // When parsing a JSON array, any elements with value `null` are ignored let json_array = RadonString::from(r#"[null, 1, null, null, 2, 3, null]"#); - let output = parse_json_array(&json_array).unwrap(); + let output = parse_json_array(&json_array, &None).unwrap(); let expected_output = RadonArray::from(vec![ RadonTypes::Integer(RadonInteger::from(1)), @@ -579,7 +800,7 @@ mod tests { #[test] fn test_parse_json_array_fail() { let invalid_json = RadonString::from(r#"{ "Hello": }"#); - let output = parse_json_array(&invalid_json).unwrap_err(); + let output = parse_json_array(&invalid_json, &None).unwrap_err(); let expected_err = RadError::JsonParse { description: "expected value at line 1 column 13".to_string(), @@ -587,7 +808,7 @@ mod tests { assert_eq!(output, expected_err); let json_map = RadonString::from(r#"{ "Hello": "world" }"#); - let output = parse_json_array(&json_map).unwrap_err(); + let output = parse_json_array(&json_map, &None).unwrap_err(); let expected_err = RadError::Decode { from: "cbor::value::Value", to: RadonArray::radon_type_name(), @@ -731,7 +952,7 @@ mod tests { let rad_float = RadonBoolean::from(false); let rad_string: RadonString = RadonString::from("false"); - assert_eq!(to_bool(&rad_string).unwrap(), rad_float); + assert_eq!(as_bool(&rad_string).unwrap(), rad_float); } #[test] @@ -1115,7 +1336,7 @@ mod tests { let args = vec![Value::Map(map)]; let result = string_match(&input_key, &args); - assert_eq!(result.unwrap_err().to_string(), "Wrong `RadonString::String match()` arguments: `[Map({Text(\"key1\"): Float(1.0), Text(\"key2\"): Float(2.0)})]`"); + assert_eq!(result.unwrap_err().to_string(), "Wrong `RadonString::StringMatch()` arguments: `[Map({Text(\"key1\"): Float(1.0), Text(\"key2\"): Float(2.0)})]`"); } #[test] diff --git a/rad/src/script.rs b/rad/src/script.rs index c395b757a..33150cc66 100644 --- a/rad/src/script.rs +++ b/rad/src/script.rs @@ -1,4 +1,4 @@ -use std::convert::TryFrom; +use std::{clone::Clone, convert::TryFrom, iter}; use serde_cbor::{ self as cbor, @@ -280,6 +280,27 @@ pub fn create_radon_script_from_filters_and_reducer( Ok(radoncall_vec) } +pub fn partial_results_extract( + subscript: &[RadonCall], + reports: &[RadonReport], + context: &mut ReportContext, +) { + if let Stage::Retrieval(metadata) = &mut context.stage { + metadata.subscript_partial_results.push(subscript.iter().chain(iter::once(&(RadonOpCodes::Fail, None))).enumerate().map(|(index, _)| + reports + .iter() + .map(|report| + report.partial_results + .as_ref() + .expect("Execution reports from applying subscripts are expected to contain partial results") + .get(index) + .expect("Execution reports from applying same subscript on multiple values should contain the same number of partial results") + .clone() + ).collect::>() + ).collect::>>()); + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -299,7 +320,7 @@ mod tests { r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":600,"main":"Snow","description":"light snow","icon":"13n"}],"base":"stations","main":{"temp":-4,"pressure":1013,"humidity":73,"temp_min":-4,"temp_max":-4},"visibility":10000,"wind":{"speed":2.6,"deg":90},"clouds":{"all":75},"dt":1548346800,"sys":{"type":1,"id":1275,"message":0.0038,"country":"DE","sunrise":1548313160,"sunset":1548344298},"id":2950159,"name":"Berlin","cod":200}"#, )); let script = vec![ - (RadonOpCodes::StringParseJSONMap, None), + (RadonOpCodes::StringParseJSONMap, Some(vec![])), ( RadonOpCodes::MapGetMap, Some(vec![Value::Text(String::from("main"))]), diff --git a/rad/src/types/array.rs b/rad/src/types/array.rs index 1c769e308..744f7a827 100644 --- a/rad/src/types/array.rs +++ b/rad/src/types/array.rs @@ -166,9 +166,11 @@ impl Operable for RadonArray { array_operators::get::(self, args).map(RadonTypes::from) } (RadonOpCodes::ArrayFilter, Some(args)) => array_operators::filter(self, args, context), + (RadonOpCodes::ArrayJoin, Some(args)) => array_operators::join(self, args), (RadonOpCodes::ArrayMap, Some(args)) => array_operators::map(self, args, context), (RadonOpCodes::ArrayReduce, Some(args)) => array_operators::reduce(self, args, context), (RadonOpCodes::ArraySort, Some(args)) => array_operators::sort(self, args, context), + (RadonOpCodes::ArrayPick, Some(args)) => array_operators::pick(self, args, context), (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_ARRAY_TYPE_NAME.to_string(), operator: op_code.to_string(), diff --git a/rad/src/types/boolean.rs b/rad/src/types/boolean.rs index f6af1fc64..3f64a3c34 100644 --- a/rad/src/types/boolean.rs +++ b/rad/src/types/boolean.rs @@ -80,7 +80,7 @@ impl Operable for RadonBoolean { match call { (RadonOpCodes::Identity, None) => identity(RadonTypes::from(self.clone())), (RadonOpCodes::BooleanNegate, None) => Ok(boolean_operators::negate(self).into()), - (RadonOpCodes::BooleanAsString, None) => boolean_operators::to_string(self.clone()) + (RadonOpCodes::BooleanToString, None) => boolean_operators::to_string(self.clone()) .map(RadonTypes::from) .map_err(Into::into), (op_code, args) => Err(RadError::UnsupportedOperator { diff --git a/rad/src/types/bytes.rs b/rad/src/types/bytes.rs index fe5c19198..03f02f6e2 100644 --- a/rad/src/types/bytes.rs +++ b/rad/src/types/bytes.rs @@ -4,6 +4,8 @@ use crate::{ script::RadonCall, types::{RadonType, RadonTypes}, }; +use num_enum::TryFromPrimitive; +use serde::Serialize; use serde_cbor::value::Value; use std::{ convert::{TryFrom, TryInto}, @@ -13,6 +15,15 @@ use witnet_data_structures::radon_report::ReportContext; const RADON_BYTES_TYPE_NAME: &str = "RadonBytes"; +/// List of support string-encoding algorithms for buffers +#[derive(Debug, Default, PartialEq, Eq, Serialize, TryFromPrimitive)] +#[repr(u8)] +pub enum RadonBytesEncoding { + #[default] + Hex = 0, + Base64 = 1, +} + #[derive(Clone, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] pub struct RadonBytes { value: Vec, @@ -84,12 +95,21 @@ impl Operable for RadonBytes { match call { // Identity (RadonOpCodes::Identity, None) => identity(RadonTypes::from(self.clone())), - (RadonOpCodes::BytesAsString, None) => bytes_operators::to_string(self) + (RadonOpCodes::BytesAsInteger, None) => bytes_operators::as_integer(self) .map(RadonTypes::from) .map_err(Into::into), (RadonOpCodes::BytesHash, Some(args)) => bytes_operators::hash(self, args.as_slice()) .map(RadonTypes::from) .map_err(Into::into), + (RadonOpCodes::BytesLength, None) => { + Ok(RadonTypes::from(bytes_operators::length(self))) + } + (RadonOpCodes::BytesSlice, Some(args)) => bytes_operators::slice(self, args.as_slice()) + .map(RadonTypes::from) + .map_err(Into::into), + (RadonOpCodes::BytesStringify, args) => bytes_operators::stringify(self, args) + .map(RadonTypes::from) + .map_err(Into::into), // Unsupported / unimplemented (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_BYTES_TYPE_NAME.to_string(), diff --git a/rad/src/types/float.rs b/rad/src/types/float.rs index 3e81fb5d6..736c3d7f6 100644 --- a/rad/src/types/float.rs +++ b/rad/src/types/float.rs @@ -116,26 +116,22 @@ impl Operable for RadonFloat { (RadonOpCodes::FloatAbsolute, None) => { Ok(RadonTypes::from(float_operators::absolute(self))) } - (RadonOpCodes::FloatAsString, None) => float_operators::to_string(self.clone()) - .map(RadonTypes::from) - .map_err(Into::into), (RadonOpCodes::FloatCeiling, None) => { Ok(RadonTypes::from(float_operators::ceiling(self))) } + (RadonOpCodes::FloatFloor, None) => Ok(RadonTypes::from(float_operators::floor(self))), (RadonOpCodes::FloatGreaterThan, Some(args)) => { float_operators::greater_than(self, args).map(Into::into) } (RadonOpCodes::FloatLessThan, Some(args)) => { float_operators::less_than(self, args).map(Into::into) } - (RadonOpCodes::FloatMultiply, Some(args)) => { - float_operators::multiply(self, args.as_slice()).map(Into::into) - } (RadonOpCodes::FloatModulo, Some(args)) => { float_operators::modulo(self, args.as_slice()).map(Into::into) } - (RadonOpCodes::FloatFloor, None) => Ok(RadonTypes::from(float_operators::floor(self))), - + (RadonOpCodes::FloatMultiply, Some(args)) => { + float_operators::multiply(self, args.as_slice()).map(Into::into) + } (RadonOpCodes::FloatNegate, None) => { Ok(RadonTypes::from(float_operators::negate(self))) } @@ -143,6 +139,9 @@ impl Operable for RadonFloat { float_operators::power(self, args.as_slice()).map(Into::into) } (RadonOpCodes::FloatRound, None) => Ok(RadonTypes::from(float_operators::round(self))), + (RadonOpCodes::FloatToString, None) => float_operators::to_string(self.clone()) + .map(RadonTypes::from) + .map_err(Into::into), (RadonOpCodes::FloatTruncate, None) => { Ok(RadonTypes::from(float_operators::truncate(self))) } diff --git a/rad/src/types/integer.rs b/rad/src/types/integer.rs index 845fd7478..254ff70d9 100644 --- a/rad/src/types/integer.rs +++ b/rad/src/types/integer.rs @@ -105,12 +105,6 @@ impl Operable for RadonInteger { (RadonOpCodes::IntegerAbsolute, None) => integer_operators::absolute(self) .map(RadonTypes::from) .map_err(Into::into), - (RadonOpCodes::IntegerAsFloat, None) => integer_operators::to_float(self.clone()) - .map(RadonTypes::from) - .map_err(Into::into), - (RadonOpCodes::IntegerAsString, None) => integer_operators::to_string(self.clone()) - .map(RadonTypes::from) - .map_err(Into::into), (RadonOpCodes::IntegerGreaterThan, Some(args)) => { integer_operators::greater_than(self, args).map(Into::into) } @@ -129,6 +123,15 @@ impl Operable for RadonInteger { (RadonOpCodes::IntegerPower, Some(args)) => { integer_operators::power(self, args.as_slice()).map(Into::into) } + (RadonOpCodes::IntegerToBytes, None) => integer_operators::to_bytes(self.clone()) + .map(RadonTypes::from) + .map_err(Into::into), + (RadonOpCodes::IntegerToFloat, None) => integer_operators::to_float(self.clone()) + .map(RadonTypes::from) + .map_err(Into::into), + (RadonOpCodes::IntegerToString, None) => integer_operators::to_string(self.clone()) + .map(RadonTypes::from) + .map_err(Into::into), // Unsupported / unimplemented (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_INTEGER_TYPE_NAME.to_string(), diff --git a/rad/src/types/map.rs b/rad/src/types/map.rs index 41e6d4380..ec9963d15 100644 --- a/rad/src/types/map.rs +++ b/rad/src/types/map.rs @@ -161,6 +161,15 @@ impl Operable for RadonMap { } (RadonOpCodes::MapKeys, None) => Ok(RadonTypes::from(map_operators::keys(self))), (RadonOpCodes::MapValues, None) => Ok(RadonTypes::from(map_operators::values(self))), + (RadonOpCodes::MapAlter, Some(args)) => { + map_operators::alter(self, args, context).map(RadonTypes::from) + } + (RadonOpCodes::MapPick, Some(args)) => { + map_operators::pick(self, args).map(RadonTypes::from) + } + (RadonOpCodes::MapStringify, None) => { + map_operators::stringify(self).map(RadonTypes::from) + } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_MAP_TYPE_NAME.to_string(), operator: op_code.to_string(), diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index 16eb9ccf3..be0f66982 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -105,6 +105,12 @@ impl Operable for RadonString { match call { (RadonOpCodes::Identity, None) => identity(RadonTypes::from(self.clone())), + (RadonOpCodes::StringAsBoolean, None) => string_operators::as_bool(self) + .map(RadonTypes::from) + .map_err(Into::into), + (RadonOpCodes::StringAsBytes, args) => string_operators::as_bytes(self, args) + .map(RadonTypes::from) + .map_err(Into::into), (RadonOpCodes::StringAsFloat, args) => if wip0024 { string_operators::as_float(self, args) } else { @@ -119,30 +125,46 @@ impl Operable for RadonString { } .map(RadonTypes::from) .map_err(Into::into), - (RadonOpCodes::StringAsBoolean, None) => string_operators::to_bool(self) - .map(RadonTypes::from) - .map_err(Into::into), - (RadonOpCodes::StringParseJSONArray, None) => string_operators::parse_json_array(self) - .map(RadonTypes::from) - .map_err(Into::into), - (RadonOpCodes::StringParseJSONMap, None) => string_operators::parse_json_map(self) - .map(RadonTypes::from) - .map_err(Into::into), + (RadonOpCodes::StringLength, None) => { + Ok(RadonTypes::from(string_operators::length(self))) + } (RadonOpCodes::StringMatch, Some(args)) => { string_operators::string_match(self, args.as_slice()).map(RadonTypes::from) } - (RadonOpCodes::StringLength, None) => { - Ok(RadonTypes::from(string_operators::length(self))) + (RadonOpCodes::StringParseJSONArray, args) => { + string_operators::parse_json_array(self, args) + .map(RadonTypes::from) + .map_err(Into::into) + } + (RadonOpCodes::StringParseJSONMap, args) => { + string_operators::parse_json_map(self, args) + .map(RadonTypes::from) + .map_err(Into::into) } + (RadonOpCodes::StringParseXMLMap, None) => string_operators::parse_xml_map(self) + .map(RadonTypes::from) + .map_err(Into::into), (RadonOpCodes::StringToLowerCase, None) => { Ok(RadonTypes::from(string_operators::to_lowercase(self))) } (RadonOpCodes::StringToUpperCase, None) => { Ok(RadonTypes::from(string_operators::to_uppercase(self))) } - (RadonOpCodes::StringParseXMLMap, None) => string_operators::parse_xml_map(self) - .map(RadonTypes::from) - .map_err(Into::into), + (RadonOpCodes::StringReplace, Some(args)) => { + string_operators::replace(self, args.as_slice()) + .map(RadonTypes::from) + .map_err(Into::into) + } + (RadonOpCodes::StringSlice, Some(args)) => { + string_operators::slice(self, args.as_slice()) + .map(RadonTypes::from) + .map_err(Into::into) + } + (RadonOpCodes::StringSplit, Some(args)) => { + string_operators::split(self, args.as_slice()) + .map(RadonTypes::from) + .map_err(Into::into) + } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_STRING_TYPE_NAME.to_string(), operator: op_code.to_string(), diff --git a/schemas/witnet/witnet.proto b/schemas/witnet/witnet.proto index 64b1b04e0..842009744 100644 --- a/schemas/witnet/witnet.proto +++ b/schemas/witnet/witnet.proto @@ -121,6 +121,7 @@ message DataRequestOutput { HttpGet = 1; Rng = 2; HttpPost = 3; + HttpHead = 4; } message RADFilter { uint32 op = 1; @@ -133,7 +134,7 @@ message DataRequestOutput { bytes script = 3; // Body of HTTP-POST request bytes body = 4; - // Extra headers for HTTP-GET and HTTP-POST requests + // Extra headers for HTTP-GET, HTTP-HEAD and HTTP-POST requests repeated StringPair headers = 5; } message RADAggregate { diff --git a/toolkit/Cargo.toml b/toolkit/Cargo.toml index eb4a5096d..7ac7b72d4 100644 --- a/toolkit/Cargo.toml +++ b/toolkit/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_toolkit" -version = "1.6.7" +version = "2.0.0" authors = ["Adán SDPC "] edition = "2021" diff --git a/wallet/Cargo.toml b/wallet/Cargo.toml index 4bf3bb766..5a7221a5f 100644 --- a/wallet/Cargo.toml +++ b/wallet/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Witnet Foundation "] edition = "2021" name = "witnet_wallet" -version = "1.6.7" +version = "2.0.0" workspace = ".." [dependencies]