From 5bbea5017103f3fc117e7f13d21d49a07ee9d532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 10 Oct 2023 12:48:47 +0200 Subject: [PATCH 01/20] feat: implement HttpHead as new RADType --- data_structures/src/chain/mod.rs | 10 ++++-- data_structures/src/proto/mod.rs | 2 ++ data_structures/src/serialization_helpers.rs | 4 +-- rad/src/lib.rs | 36 +++++++++++++++++--- schemas/witnet/witnet.proto | 3 +- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs index 34910b8a5..757f6c46e 100644 --- a/data_structures/src/chain/mod.rs +++ b/data_structures/src/chain/mod.rs @@ -1645,11 +1645,14 @@ pub enum RADType { /// HTTP POST request #[serde(rename = "HTTP-POST")] HttpPost, + /// HTTP HEAD request + #[serde(rename = "HTTP-HEAD")] + HttpHead, } impl RADType { pub fn is_http(&self) -> bool { - matches!(self, RADType::HttpGet | RADType::HttpPost) + matches!(self, RADType::HttpGet | RADType::HttpPost | RADType::HttpHead) } } @@ -1701,7 +1704,7 @@ pub struct RADRetrieve { pub script: Vec, /// Body of a HTTP-POST request pub body: Vec, - /// Extra headers of a HTTP-GET or HTTP-POST request + /// Extra headers of a HTTP-GET, HTTP-POST or HTTP-HEAD request pub headers: Vec<(String, String)>, } @@ -1809,7 +1812,8 @@ impl RADRetrieve { &[Field::Kind, Field::Url, Field::Script], &[Field::Body, Field::Headers], ) - } + }, + RADType::HttpHead => check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers]) } } diff --git a/data_structures/src/proto/mod.rs b/data_structures/src/proto/mod.rs index 4d681d8b9..c1bb007ca 100644 --- a/data_structures/src/proto/mod.rs +++ b/data_structures/src/proto/mod.rs @@ -51,6 +51,7 @@ impl ProtobufConvert for chain::RADType { chain::RADType::HttpGet => witnet::DataRequestOutput_RADRequest_RADType::HttpGet, chain::RADType::Rng => witnet::DataRequestOutput_RADRequest_RADType::Rng, chain::RADType::HttpPost => witnet::DataRequestOutput_RADRequest_RADType::HttpPost, + chain::RADType::HttpHead => witnet::DataRequestOutput_RADRequest_RADType::HttpHead, } } @@ -60,6 +61,7 @@ impl ProtobufConvert for chain::RADType { witnet::DataRequestOutput_RADRequest_RADType::HttpGet => chain::RADType::HttpGet, witnet::DataRequestOutput_RADRequest_RADType::Rng => chain::RADType::Rng, witnet::DataRequestOutput_RADRequest_RADType::HttpPost => chain::RADType::HttpPost, + witnet::DataRequestOutput_RADRequest_RADType::HttpHead => chain::RADType::HttpHead, }) } } diff --git a/data_structures/src/serialization_helpers.rs b/data_structures/src/serialization_helpers.rs index 7f7d7d4e6..a8d59ee69 100644 --- a/data_structures/src/serialization_helpers.rs +++ b/data_structures/src/serialization_helpers.rs @@ -360,7 +360,7 @@ struct RADRetrieveSerializationHelperJson { /// Body of a HTTP-POST request #[serde(default, skip_serializing_if = "Vec::is_empty")] pub body: Vec, - /// Extra headers of a HTTP-GET or HTTP-POST request + /// Extra headers of a HTTP-GET, HTTP-HEAD or HTTP-POST request #[serde(default, skip_serializing_if = "Vec::is_empty")] pub headers: Vec<(String, String)>, } @@ -377,7 +377,7 @@ struct RADRetrieveSerializationHelperBincode { pub script: Vec, /// Body of a HTTP-POST request pub body: Vec, - /// Extra headers of a HTTP-GET or HTTP-POST request + /// Extra headers of a HTTP-GET, HTTP-HEAD or HTTP-POST request pub headers: Vec<(String, String)>, } diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 64c826780..a3781df42 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -25,10 +25,11 @@ use crate::{ create_radon_script_from_filters_and_reducer, execute_radon_script, unpack_radon_script, RadonScriptExecutionSettings, }, - types::{array::RadonArray, bytes::RadonBytes, string::RadonString, RadonTypes}, + types::{array::RadonArray, bytes::RadonBytes, map::RadonMap, string::RadonString, RadonTypes}, user_agents::UserAgent, }; use core::convert::From; +use std::collections::BTreeMap; use witnet_net::client::http::{WitnetHttpBody, WitnetHttpRequest}; pub mod conditions; @@ -173,6 +174,25 @@ fn string_response_with_data_report( execute_radon_script(input, &radon_script, context, settings) } +/// Handle HTTP-HEAD response with data, and return a `RadonReport`. +fn headers_response_with_data_report( + retrieve: &RADRetrieve, + response: &str, + context: &mut ReportContext, + settings: RadonScriptExecutionSettings, +) -> Result> { + let headers: BTreeMap = response.split("\r\n").map(|line| { + let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect(); + // todo: check there are two parts, and two parts only + // todo: make sure that values from repeated keys get appended within a RadonArray + (String::from(parts[0]), RadonTypes::from(RadonString::from(parts[1]))) + }).collect(); + let input = RadonTypes::from(RadonMap::from(headers)); + let radon_script = unpack_radon_script(&retrieve.script)?; + + execute_radon_script(input, &radon_script, context, settings) +} + /// Handle Rng response with data report fn rng_response_with_data_report( response: &str, @@ -196,7 +216,10 @@ pub fn run_retrieval_with_data_report( RADType::Rng => rng_response_with_data_report(response, context), RADType::HttpPost => { string_response_with_data_report(retrieve, response, context, settings) - } + }, + RADType::HttpHead => { + headers_response_with_data_report(retrieve, response, context, settings) + }, _ => Err(RadError::UnknownRetrieval), } } @@ -214,7 +237,7 @@ pub fn run_retrieval_with_data( .map(RadonReport::into_inner) } -/// Handle generic HTTP (GET/POST) response +/// Handle generic HTTP (GET/POST/HEAD) response async fn http_response( retrieve: &RADRetrieve, context: &mut ReportContext, @@ -258,7 +281,11 @@ async fn http_response( builder.method("POST").uri(&retrieve.url), WitnetHttpBody::from(retrieve.body.clone()), ) - } + }, + RADType::HttpHead => ( + builder.method("HEAD").uri(&retrieve.url), + WitnetHttpBody::empty(), + ), _ => panic!( "Called http_response with invalid retrieval kind {:?}", retrieve.kind @@ -357,6 +384,7 @@ pub async fn run_retrieval_report( RADType::HttpGet => http_response(retrieve, context, settings, client).await, RADType::Rng => rng_response(context, settings).await, RADType::HttpPost => http_response(retrieve, context, settings, client).await, + RADType::HttpHead => http_response(retrieve, context, settings, client).await, _ => Err(RadError::UnknownRetrieval), } } diff --git a/schemas/witnet/witnet.proto b/schemas/witnet/witnet.proto index 64b1b04e0..842009744 100644 --- a/schemas/witnet/witnet.proto +++ b/schemas/witnet/witnet.proto @@ -121,6 +121,7 @@ message DataRequestOutput { HttpGet = 1; Rng = 2; HttpPost = 3; + HttpHead = 4; } message RADFilter { uint32 op = 1; @@ -133,7 +134,7 @@ message DataRequestOutput { bytes script = 3; // Body of HTTP-POST request bytes body = 4; - // Extra headers for HTTP-GET and HTTP-POST requests + // Extra headers for HTTP-GET, HTTP-HEAD and HTTP-POST requests repeated StringPair headers = 5; } message RADAggregate { From 3fdf8f973dcbb168149faafad727cbc084b6ed67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 10 Oct 2023 13:26:31 +0200 Subject: [PATCH 02/20] chore: cargo fmt --all --- data_structures/src/chain/mod.rs | 11 ++++++++--- rad/src/lib.rs | 22 ++++++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs index 757f6c46e..7ed2e38e0 100644 --- a/data_structures/src/chain/mod.rs +++ b/data_structures/src/chain/mod.rs @@ -1652,7 +1652,10 @@ pub enum RADType { impl RADType { pub fn is_http(&self) -> bool { - matches!(self, RADType::HttpGet | RADType::HttpPost | RADType::HttpHead) + matches!( + self, + RADType::HttpGet | RADType::HttpPost | RADType::HttpHead + ) } } @@ -1812,8 +1815,10 @@ impl RADRetrieve { &[Field::Kind, Field::Url, Field::Script], &[Field::Body, Field::Headers], ) - }, - RADType::HttpHead => check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers]) + } + RADType::HttpHead => { + check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers]) + } } } diff --git a/rad/src/lib.rs b/rad/src/lib.rs index a3781df42..dfa623910 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -181,12 +181,18 @@ fn headers_response_with_data_report( context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { - let headers: BTreeMap = response.split("\r\n").map(|line| { - let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect(); - // todo: check there are two parts, and two parts only - // todo: make sure that values from repeated keys get appended within a RadonArray - (String::from(parts[0]), RadonTypes::from(RadonString::from(parts[1]))) - }).collect(); + let headers: BTreeMap = response + .split("\r\n") + .map(|line| { + let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect(); + // todo: check there are two parts, and two parts only + // todo: make sure that values from repeated keys get appended within a RadonArray + ( + String::from(parts[0]), + RadonTypes::from(RadonString::from(parts[1])), + ) + }) + .collect(); let input = RadonTypes::from(RadonMap::from(headers)); let radon_script = unpack_radon_script(&retrieve.script)?; @@ -216,10 +222,10 @@ pub fn run_retrieval_with_data_report( RADType::Rng => rng_response_with_data_report(response, context), RADType::HttpPost => { string_response_with_data_report(retrieve, response, context, settings) - }, + } RADType::HttpHead => { headers_response_with_data_report(retrieve, response, context, settings) - }, + } _ => Err(RadError::UnknownRetrieval), } } From 721347a1b316e385bdc0225fdc3862d5ff69b6b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:45:08 +0200 Subject: [PATCH 03/20] feat: handle repeated headers within http/head response --- rad/src/lib.rs | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index dfa623910..532a67668 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -181,18 +181,33 @@ fn headers_response_with_data_report( context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { - let headers: BTreeMap = response - .split("\r\n") - .map(|line| { - let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect(); - // todo: check there are two parts, and two parts only - // todo: make sure that values from repeated keys get appended within a RadonArray - ( - String::from(parts[0]), - RadonTypes::from(RadonString::from(parts[1])), - ) + let mut headers: BTreeMap> = BTreeMap::new(); + + for line in response.split("\r\n") { + if let Some(first_colon_index) = line.find(":") { + // key: trim spaces and lower case all ascii chars left to the first colon character + let key = String::from(line[0..first_colon_index].trim().to_ascii_lowercase()); + // value: trim spaces on the substring after the first colon character + let value = RadonTypes::from(RadonString::from(line[first_colon_index + 1..].trim())); + headers.entry(key).or_default().push(value); + } + } + + let headers: BTreeMap = BTreeMap::from_iter( + headers.iter().map(|(key, value)| { + match value.len() { + len if len > 1 => ( + key.clone(), + RadonTypes::from(RadonArray::from(value.to_vec())) + ), + _ => ( + key.clone(), + value[0].clone() + ) + } }) - .collect(); + ); + let input = RadonTypes::from(RadonMap::from(headers)); let radon_script = unpack_radon_script(&retrieve.script)?; From ef307337cc7b11b16b4c92a41839cbe2493229b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:47:30 +0200 Subject: [PATCH 04/20] test: run valid http head retrieval --- rad/src/lib.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 532a67668..d28a579ae 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -878,6 +878,38 @@ mod tests { use super::*; + #[test] + fn test_run_http_head_retrieval() { + let script_r = Value::Array(vec![ + Value::Array(vec![ + Value::Integer(RadonOpCodes::MapGetString as i128), + Value::Text("etag".to_string()), + ]), + ]); + let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); + println!("{:?}", packed_script_r); + + let retrieve = RADRetrieve { + kind: RADType::HttpHead, + url: "https://witnet.io/_nuxt/img/dragon_reading.a37f8cb.png".to_string(), + script: packed_script_r, + body: vec![], + headers: vec![], + }; + let response = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 498219\r\nContent-Length: 123456\r\neTag: \"64eca181-79a2b\"\r\n"; + let result = run_retrieval_with_data( + &retrieve, + response, + RadonScriptExecutionSettings::disable_all(), + current_active_wips(), + ).unwrap(); + + match result { + RadonTypes::String(_) => {} + err => panic!("Error in run_retrieval: {:?}", err), + } + } + #[test] fn test_run_retrieval() { let script_r = Value::Array(vec![ From a6653977a7d3aa97da4bf1dcc19d07d239050b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:48:31 +0200 Subject: [PATCH 05/20] test: try http-head data request w/ invalid request header --- rad/src/lib.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index d28a579ae..e07bc4b32 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -302,7 +302,7 @@ async fn http_response( builder.method("POST").uri(&retrieve.url), WitnetHttpBody::from(retrieve.body.clone()), ) - }, + } RADType::HttpHead => ( builder.method("HEAD").uri(&retrieve.url), WitnetHttpBody::empty(), @@ -1607,6 +1607,58 @@ mod tests { } } + #[test] + fn test_try_data_request_http_get_non_ascii_header_key() { + let script_r = Value::Array(vec![]); + let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); + let body = Vec::from(String::from("")); + let headers = vec![("ñ", "value")]; + let headers = headers + .into_iter() + .map(|(a, b)| (a.to_string(), b.to_string())) + .collect(); + let request = RADRequest { + time_lock: 0, + retrieve: vec![RADRetrieve { + kind: RADType::HttpGet, + url: String::from("http://127.0.0.1"), + script: packed_script_r, + body, + headers, + }], + aggregate: RADAggregate { + filters: vec![], + reducer: RadonReducers::Mode as u32, + }, + tally: RADTally { + filters: vec![], + reducer: RadonReducers::Mode as u32, + }, + }; + let report = try_data_request( + &request, + RadonScriptExecutionSettings::enable_all(), + None, + None, + ); + let tally_result = report.tally.into_inner(); + + assert_eq!( + tally_result, + RadonTypes::RadonError( + RadonError::try_from(RadError::UnhandledIntercept { + inner: Some(Box::new(RadError::InvalidHttpHeader { + name: "ñ".to_string(), + value: "value".to_string(), + error: "invalid HTTP header name".to_string() + })), + message: None + }) + .unwrap() + ) + ); + } + #[test] fn test_try_data_request_http_post_non_ascii_header_key() { let script_r = Value::Array(vec![]); From 198e5b8f99f208f2a2662dcb81babc2eacf1d27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:49:09 +0200 Subject: [PATCH 06/20] chore: bump package versions --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- bridges/centralized-ethereum/Cargo.toml | 2 +- data_structures/Cargo.toml | 2 +- node/Cargo.toml | 2 +- rad/Cargo.toml | 2 +- toolkit/Cargo.toml | 2 +- wallet/Cargo.toml | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f0d09be9..4571cdfa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4958,7 +4958,7 @@ dependencies = [ [[package]] name = "witnet" -version = "1.6.7" +version = "1.7.0" dependencies = [ "ansi_term", "bytecount", @@ -5005,7 +5005,7 @@ dependencies = [ [[package]] name = "witnet-centralized-ethereum-bridge" -version = "1.6.7" +version = "1.7.0" dependencies = [ "actix", "async-jsonrpc-client", @@ -5069,7 +5069,7 @@ dependencies = [ [[package]] name = "witnet_data_structures" -version = "1.6.7" +version = "1.7.0" dependencies = [ "bech32", "bencher", @@ -5134,7 +5134,7 @@ dependencies = [ [[package]] name = "witnet_node" -version = "1.6.7" +version = "1.7.0" dependencies = [ "actix", "ansi_term", @@ -5198,7 +5198,7 @@ dependencies = [ [[package]] name = "witnet_rad" -version = "0.3.2" +version = "0.3.3" dependencies = [ "cbor-codec", "failure", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "witnet_toolkit" -version = "1.6.7" +version = "1.7.0" dependencies = [ "failure", "hex", @@ -5284,7 +5284,7 @@ dependencies = [ [[package]] name = "witnet_wallet" -version = "1.6.7" +version = "1.7.0" dependencies = [ "actix", "async-jsonrpc-client", diff --git a/Cargo.toml b/Cargo.toml index 0a428c2fb..4a1b64b65 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet" -version = "1.6.7" +version = "1.7.0" authors = ["Witnet Foundation "] publish = false repository = "witnet/witnet-rust" diff --git a/bridges/centralized-ethereum/Cargo.toml b/bridges/centralized-ethereum/Cargo.toml index 5492feb9e..b87084520 100644 --- a/bridges/centralized-ethereum/Cargo.toml +++ b/bridges/centralized-ethereum/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet-centralized-ethereum-bridge" -version = "1.6.7" +version = "1.7.0" authors = ["Witnet Foundation "] edition = "2018" diff --git a/data_structures/Cargo.toml b/data_structures/Cargo.toml index fa801502b..f09cf9699 100644 --- a/data_structures/Cargo.toml +++ b/data_structures/Cargo.toml @@ -3,7 +3,7 @@ authors = ["Witnet Foundation "] description = "data structures component" edition = "2021" name = "witnet_data_structures" -version = "1.6.7" +version = "1.7.0" workspace = ".." [features] diff --git a/node/Cargo.toml b/node/Cargo.toml index 796ca2737..88a31f51a 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_node" -version = "1.6.7" +version = "1.7.0" authors = ["Witnet Foundation "] workspace = ".." description = "node component" diff --git a/rad/Cargo.toml b/rad/Cargo.toml index 806b62527..d9bef65f3 100644 --- a/rad/Cargo.toml +++ b/rad/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_rad" -version = "0.3.2" +version = "0.3.3" authors = ["Witnet Foundation "] edition = "2021" workspace = ".." diff --git a/toolkit/Cargo.toml b/toolkit/Cargo.toml index eb4a5096d..5ee64aecd 100644 --- a/toolkit/Cargo.toml +++ b/toolkit/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_toolkit" -version = "1.6.7" +version = "1.7.0" authors = ["Adán SDPC "] edition = "2021" diff --git a/wallet/Cargo.toml b/wallet/Cargo.toml index 4bf3bb766..ec59fb12b 100644 --- a/wallet/Cargo.toml +++ b/wallet/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Witnet Foundation "] edition = "2021" name = "witnet_wallet" -version = "1.6.7" +version = "1.7.0" workspace = ".." [dependencies] From 9bae448eb61a266b2fe36ebe6386fcd8a940e08e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:52:59 +0200 Subject: [PATCH 07/20] chore: cargo fmt --all --- rad/src/lib.rs | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index e07bc4b32..d21e663ca 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -192,21 +192,15 @@ fn headers_response_with_data_report( headers.entry(key).or_default().push(value); } } - - let headers: BTreeMap = BTreeMap::from_iter( - headers.iter().map(|(key, value)| { - match value.len() { - len if len > 1 => ( - key.clone(), - RadonTypes::from(RadonArray::from(value.to_vec())) - ), - _ => ( - key.clone(), - value[0].clone() - ) - } - }) - ); + + let headers: BTreeMap = + BTreeMap::from_iter(headers.iter().map(|(key, value)| match value.len() { + len if len > 1 => ( + key.clone(), + RadonTypes::from(RadonArray::from(value.to_vec())), + ), + _ => (key.clone(), value[0].clone()), + })); let input = RadonTypes::from(RadonMap::from(headers)); let radon_script = unpack_radon_script(&retrieve.script)?; @@ -880,12 +874,10 @@ mod tests { #[test] fn test_run_http_head_retrieval() { - let script_r = Value::Array(vec![ - Value::Array(vec![ - Value::Integer(RadonOpCodes::MapGetString as i128), - Value::Text("etag".to_string()), - ]), - ]); + let script_r = Value::Array(vec![Value::Array(vec![ + Value::Integer(RadonOpCodes::MapGetString as i128), + Value::Text("etag".to_string()), + ])]); let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); println!("{:?}", packed_script_r); @@ -902,7 +894,8 @@ mod tests { response, RadonScriptExecutionSettings::disable_all(), current_active_wips(), - ).unwrap(); + ) + .unwrap(); match result { RadonTypes::String(_) => {} From af5ff8643a735a534fd322dc9423e5b4c9668cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 11:59:33 +0200 Subject: [PATCH 08/20] chore: solve clippy warnings --- rad/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index d21e663ca..5fa1e8979 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -184,9 +184,9 @@ fn headers_response_with_data_report( let mut headers: BTreeMap> = BTreeMap::new(); for line in response.split("\r\n") { - if let Some(first_colon_index) = line.find(":") { + if let Some(first_colon_index) = line.find(':') { // key: trim spaces and lower case all ascii chars left to the first colon character - let key = String::from(line[0..first_colon_index].trim().to_ascii_lowercase()); + let key = line[0..first_colon_index].trim().to_ascii_lowercase(); // value: trim spaces on the substring after the first colon character let value = RadonTypes::from(RadonString::from(line[first_colon_index + 1..].trim())); headers.entry(key).or_default().push(value); @@ -692,7 +692,7 @@ fn validate_header(name: &str, value: &str) -> Result<()> { Err(RadError::InvalidHttpHeader { name: name.to_string(), value: value.to_string(), - error: error_message.to_string(), + error: error_message, }) } else { Ok(()) From ac57deddef050bba326bce951bc6f49ca592afb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 17:51:28 +0200 Subject: [PATCH 09/20] fix: use http::Response::headers(&self) to transform response to HTTP/HEAD into a JSON string Instead or parsing http::Response::body into a RadonMap --- net/src/client/http/mod.rs | 2 +- rad/src/lib.rs | 84 +++++++++++++------------------------- 2 files changed, 29 insertions(+), 57 deletions(-) diff --git a/net/src/client/http/mod.rs b/net/src/client/http/mod.rs index 55ae33e9e..3fca5533b 100644 --- a/net/src/client/http/mod.rs +++ b/net/src/client/http/mod.rs @@ -151,7 +151,7 @@ pub struct WitnetHttpResponse { impl WitnetHttpResponse { #[inline] - /// Simple wrapper around `isahc::Response::status`. + /// Simple wrapper around `isahc::Response`. pub fn inner(self) -> isahc::Response { self.res } diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 5fa1e8979..3c3990dea 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -25,11 +25,10 @@ use crate::{ create_radon_script_from_filters_and_reducer, execute_radon_script, unpack_radon_script, RadonScriptExecutionSettings, }, - types::{array::RadonArray, bytes::RadonBytes, map::RadonMap, string::RadonString, RadonTypes}, + types::{array::RadonArray, bytes::RadonBytes, string::RadonString, RadonTypes}, user_agents::UserAgent, }; use core::convert::From; -use std::collections::BTreeMap; use witnet_net::client::http::{WitnetHttpBody, WitnetHttpRequest}; pub mod conditions; @@ -174,40 +173,6 @@ fn string_response_with_data_report( execute_radon_script(input, &radon_script, context, settings) } -/// Handle HTTP-HEAD response with data, and return a `RadonReport`. -fn headers_response_with_data_report( - retrieve: &RADRetrieve, - response: &str, - context: &mut ReportContext, - settings: RadonScriptExecutionSettings, -) -> Result> { - let mut headers: BTreeMap> = BTreeMap::new(); - - for line in response.split("\r\n") { - if let Some(first_colon_index) = line.find(':') { - // key: trim spaces and lower case all ascii chars left to the first colon character - let key = line[0..first_colon_index].trim().to_ascii_lowercase(); - // value: trim spaces on the substring after the first colon character - let value = RadonTypes::from(RadonString::from(line[first_colon_index + 1..].trim())); - headers.entry(key).or_default().push(value); - } - } - - let headers: BTreeMap = - BTreeMap::from_iter(headers.iter().map(|(key, value)| match value.len() { - len if len > 1 => ( - key.clone(), - RadonTypes::from(RadonArray::from(value.to_vec())), - ), - _ => (key.clone(), value[0].clone()), - })); - - let input = RadonTypes::from(RadonMap::from(headers)); - let radon_script = unpack_radon_script(&retrieve.script)?; - - execute_radon_script(input, &radon_script, context, settings) -} - /// Handle Rng response with data report fn rng_response_with_data_report( response: &str, @@ -227,14 +192,10 @@ pub fn run_retrieval_with_data_report( settings: RadonScriptExecutionSettings, ) -> Result> { match retrieve.kind { - RADType::HttpGet => string_response_with_data_report(retrieve, response, context, settings), - RADType::Rng => rng_response_with_data_report(response, context), - RADType::HttpPost => { + RADType::HttpGet | RADType::HttpPost | RADType::HttpHead => { string_response_with_data_report(retrieve, response, context, settings) } - RADType::HttpHead => { - headers_response_with_data_report(retrieve, response, context, settings) - } + RADType::Rng => rng_response_with_data_report(response, context), _ => Err(RadError::UnknownRetrieval), } } @@ -340,13 +301,21 @@ async fn http_response( // If at some point we want to support the retrieval of non-UTF8 data (e.g. raw bytes), this is // where we need to decide how to read the response body - let (_parts, mut body) = response.into_parts(); let mut response_string = String::default(); - body.read_to_string(&mut response_string) - .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), - })?; + + let (parts, mut body) = response.into_parts(); + match retrieve.kind { + RADType::HttpHead => { + response_string = format!("{:?}", parts.headers); + } + _ => { + body.read_to_string(&mut response_string) + .await + .map_err(|x| RadError::HttpOther { + message: x.to_string(), + })?; + } + } let result = run_retrieval_with_data_report(retrieve, &response_string, context, settings); @@ -396,10 +365,10 @@ pub async fn run_retrieval_report( context.set_active_wips(active_wips); match retrieve.kind { - RADType::HttpGet => http_response(retrieve, context, settings, client).await, + RADType::HttpGet | RADType::HttpHead | RADType::HttpPost => { + http_response(retrieve, context, settings, client).await + } RADType::Rng => rng_response(context, settings).await, - RADType::HttpPost => http_response(retrieve, context, settings, client).await, - RADType::HttpHead => http_response(retrieve, context, settings, client).await, _ => Err(RadError::UnknownRetrieval), } } @@ -874,10 +843,13 @@ mod tests { #[test] fn test_run_http_head_retrieval() { - let script_r = Value::Array(vec![Value::Array(vec![ - Value::Integer(RadonOpCodes::MapGetString as i128), - Value::Text("etag".to_string()), - ])]); + let script_r = Value::Array(vec![ + Value::Integer(RadonOpCodes::StringParseJSONMap as i128), + Value::Array(vec![ + Value::Integer(RadonOpCodes::MapGetString as i128), + Value::Text("etag".to_string()), + ]), + ]); let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); println!("{:?}", packed_script_r); @@ -888,7 +860,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 498219\r\nContent-Length: 123456\r\neTag: \"64eca181-79a2b\"\r\n"; + let response = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#; let result = run_retrieval_with_data( &retrieve, response, From 5e073f9ac26d577bc5cdbe4f619039f1f280ef6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Fri, 20 Oct 2023 09:52:56 +0200 Subject: [PATCH 10/20] chore: attend pr review comments --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- bridges/centralized-ethereum/Cargo.toml | 2 +- data_structures/Cargo.toml | 2 +- node/Cargo.toml | 2 +- rad/src/lib.rs | 3 +-- toolkit/Cargo.toml | 2 +- wallet/Cargo.toml | 2 +- 8 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4571cdfa6..8580a36aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4958,7 +4958,7 @@ dependencies = [ [[package]] name = "witnet" -version = "1.7.0" +version = "2.0.0" dependencies = [ "ansi_term", "bytecount", @@ -5005,7 +5005,7 @@ dependencies = [ [[package]] name = "witnet-centralized-ethereum-bridge" -version = "1.7.0" +version = "2.0.0" dependencies = [ "actix", "async-jsonrpc-client", @@ -5069,7 +5069,7 @@ dependencies = [ [[package]] name = "witnet_data_structures" -version = "1.7.0" +version = "2.0.0" dependencies = [ "bech32", "bencher", @@ -5134,7 +5134,7 @@ dependencies = [ [[package]] name = "witnet_node" -version = "1.7.0" +version = "2.0.0" dependencies = [ "actix", "ansi_term", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "witnet_toolkit" -version = "1.7.0" +version = "2.0.0" dependencies = [ "failure", "hex", @@ -5284,7 +5284,7 @@ dependencies = [ [[package]] name = "witnet_wallet" -version = "1.7.0" +version = "2.0.0" dependencies = [ "actix", "async-jsonrpc-client", diff --git a/Cargo.toml b/Cargo.toml index 4a1b64b65..21a934c11 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet" -version = "1.7.0" +version = "2.0.0" authors = ["Witnet Foundation "] publish = false repository = "witnet/witnet-rust" diff --git a/bridges/centralized-ethereum/Cargo.toml b/bridges/centralized-ethereum/Cargo.toml index b87084520..6e3a4f7a0 100644 --- a/bridges/centralized-ethereum/Cargo.toml +++ b/bridges/centralized-ethereum/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet-centralized-ethereum-bridge" -version = "1.7.0" +version = "2.0.0" authors = ["Witnet Foundation "] edition = "2018" diff --git a/data_structures/Cargo.toml b/data_structures/Cargo.toml index f09cf9699..9fd896c71 100644 --- a/data_structures/Cargo.toml +++ b/data_structures/Cargo.toml @@ -3,7 +3,7 @@ authors = ["Witnet Foundation "] description = "data structures component" edition = "2021" name = "witnet_data_structures" -version = "1.7.0" +version = "2.0.0" workspace = ".." [features] diff --git a/node/Cargo.toml b/node/Cargo.toml index 88a31f51a..57fe55a7a 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_node" -version = "1.7.0" +version = "2.0.0" authors = ["Witnet Foundation "] workspace = ".." description = "node component" diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 3c3990dea..e7eb65541 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -851,11 +851,10 @@ mod tests { ]), ]); let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); - println!("{:?}", packed_script_r); let retrieve = RADRetrieve { kind: RADType::HttpHead, - url: "https://witnet.io/_nuxt/img/dragon_reading.a37f8cb.png".to_string(), + url: "https://en.wikipedia.org/static/images/icons/wikipedia.png".to_string(), script: packed_script_r, body: vec![], headers: vec![], diff --git a/toolkit/Cargo.toml b/toolkit/Cargo.toml index 5ee64aecd..7ac7b72d4 100644 --- a/toolkit/Cargo.toml +++ b/toolkit/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_toolkit" -version = "1.7.0" +version = "2.0.0" authors = ["Adán SDPC "] edition = "2021" diff --git a/wallet/Cargo.toml b/wallet/Cargo.toml index ec59fb12b..5a7221a5f 100644 --- a/wallet/Cargo.toml +++ b/wallet/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Witnet Foundation "] edition = "2021" name = "witnet_wallet" -version = "1.7.0" +version = "2.0.0" workspace = ".." [dependencies] From 6c4d1ac2006da442b55fa12cd2b8f7991be44547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Fri, 20 Oct 2023 12:44:25 +0200 Subject: [PATCH 11/20] feat(rad): add support to binary sources --- rad/src/lib.rs | 88 ++++++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 38 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index e7eb65541..6f9b8f693 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -78,7 +78,12 @@ pub fn try_data_request( .iter() .zip(inputs.iter()) .map(|(retrieve, input)| { - run_retrieval_with_data_report(retrieve, input, &mut retrieval_context, settings) + run_retrieval_with_data_report( + retrieve, + RadonTypes::from(RadonString::from(*input)), + &mut retrieval_context, + settings, + ) }) .collect() } else { @@ -160,42 +165,28 @@ pub fn try_data_request( } } -/// Handle HTTP-GET and HTTP-POST response with data, and return a `RadonReport`. -fn string_response_with_data_report( +/// Execute Radon Script using as input the RadonTypes value deserialized from a retrieval response +fn handle_response_with_data_report( retrieve: &RADRetrieve, - response: &str, + response: RadonTypes, context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { - let input = RadonTypes::from(RadonString::from(response)); let radon_script = unpack_radon_script(&retrieve.script)?; - - execute_radon_script(input, &radon_script, context, settings) -} - -/// Handle Rng response with data report -fn rng_response_with_data_report( - response: &str, - context: &mut ReportContext, -) -> Result> { - let response_bytes = response.as_bytes(); - let result = RadonTypes::from(RadonBytes::from(response_bytes.to_vec())); - - Ok(RadonReport::from_result(Ok(result), context)) + execute_radon_script(response, &radon_script, context, settings) } /// Run retrieval without performing any external network requests, return `Result`. pub fn run_retrieval_with_data_report( retrieve: &RADRetrieve, - response: &str, + response: RadonTypes, context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { match retrieve.kind { - RADType::HttpGet | RADType::HttpPost | RADType::HttpHead => { - string_response_with_data_report(retrieve, response, context, settings) + RADType::HttpGet | RADType::HttpPost | RADType::HttpHead | RADType::Rng => { + handle_response_with_data_report(retrieve, response, context, settings) } - RADType::Rng => rng_response_with_data_report(response, context), _ => Err(RadError::UnknownRetrieval), } } @@ -203,7 +194,7 @@ pub fn run_retrieval_with_data_report( /// Run retrieval without performing any external network requests, return `Result`. pub fn run_retrieval_with_data( retrieve: &RADRetrieve, - response: &str, + response: RadonTypes, settings: RadonScriptExecutionSettings, active_wips: ActiveWips, ) -> Result { @@ -299,26 +290,48 @@ async fn http_response( }); } - // If at some point we want to support the retrieval of non-UTF8 data (e.g. raw bytes), this is - // where we need to decide how to read the response body - let mut response_string = String::default(); - let (parts, mut body) = response.into_parts(); - match retrieve.kind { - RADType::HttpHead => { - response_string = format!("{:?}", parts.headers); + + let response: RadonTypes; + match parts.headers.get("accept-ranges") { + Some(_) => { + // http response is a binary stream + let mut response_bytes = Vec::::default(); + match retrieve.kind { + RADType::HttpHead => { + // todo: assert http-head responses should never return binary streams + } + _ => { + // todo: before reading the response buffer, an error should thrown it was too big + body.read_to_end(&mut response_bytes).await.map_err(|x| { + RadError::HttpOther { + message: x.to_string(), + } + })?; + } + } + response = RadonTypes::from(RadonBytes::from(response_bytes)); } _ => { - body.read_to_string(&mut response_string) - .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), - })?; + // response is a string + let mut response_string = String::default(); + match retrieve.kind { + RADType::HttpHead => { + response_string = format!("{:?}", parts.headers); + } + _ => { + body.read_to_string(&mut response_string) + .await + .map_err(|x| RadError::HttpOther { + message: x.to_string(), + })?; + } + } + response = RadonTypes::from(RadonString::from(response_string)); } } - let result = run_retrieval_with_data_report(retrieve, &response_string, context, settings); - + let result = handle_response_with_data_report(retrieve, response, context, settings); match &result { Ok(report) => { log::debug!( @@ -329,7 +342,6 @@ async fn http_response( } Err(e) => log::debug!("Failed result for source {}: {:?}", retrieve.url, e), } - result } From 0cfd81d4aad31b2212ee0b354fc45b6175d8940a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Fri, 20 Oct 2023 12:45:00 +0200 Subject: [PATCH 12/20] chore(rad): refactor existing tests --- rad/src/lib.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 6f9b8f693..2af893083 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -871,10 +871,10 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#; + let response_string = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#; let result = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -908,11 +908,10 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":500,"main":"Rain","description":"light rain","icon":"10d"}],"base":"stations","main":{"temp":17.59,"pressure":1022,"humidity":67,"temp_min":15,"temp_max":20},"visibility":10000,"wind":{"speed":3.6,"deg":260},"rain":{"1h":0.51},"clouds":{"all":20},"dt":1567501321,"sys":{"type":1,"id":1275,"message":0.0089,"country":"DE","sunrise":1567484402,"sunset":1567533129},"timezone":7200,"id":2950159,"name":"Berlin","cod":200}"#; - + let response_string = r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":500,"main":"Rain","description":"light rain","icon":"10d"}],"base":"stations","main":{"temp":17.59,"pressure":1022,"humidity":67,"temp_min":15,"temp_max":20},"visibility":10000,"wind":{"speed":3.6,"deg":260},"rain":{"1h":0.51},"clouds":{"all":20},"dt":1567501321,"sys":{"type":1,"id":1275,"message":0.0089,"country":"DE","sunrise":1567484402,"sunset":1567533129},"timezone":7200,"id":2950159,"name":"Berlin","cod":200}"#; let result = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -967,7 +966,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = "84"; + let response_string = "84"; let expected = RadonTypes::Float(RadonFloat::from(84)); let aggregate = RADAggregate { @@ -982,7 +981,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1005,7 +1004,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = "307"; + let response_string = "307"; let expected = RadonTypes::Float(RadonFloat::from(307)); let aggregate = RADAggregate { @@ -1019,7 +1018,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1058,7 +1057,7 @@ mod tests { headers: vec![], }; // This response was modified because the original was about 100KB. - let response = r#"[{"estacion_nombre":"Pza. de España","estacion_numero":4,"fecha":"03092019","hora0":{"estado":"Pasado","valor":"00008"}}]"#; + let response_string = r#"[{"estacion_nombre":"Pza. de España","estacion_numero":4,"fecha":"03092019","hora0":{"estado":"Pasado","valor":"00008"}}]"#; let expected = RadonTypes::Float(RadonFloat::from(8)); let aggregate = RADAggregate { @@ -1072,7 +1071,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1102,7 +1101,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"PSOE":123,"PP":66,"Cs":57,"UP":42,"VOX":24,"ERC-SOBIRANISTES":15,"JxCAT-JUNTS":7,"PNV":6,"EH Bildu":4,"CCa-PNC":2,"NA+":2,"COMPROMÍS 2019":1,"PRC":1,"PACMA":0,"FRONT REPUBLICÀ":0,"BNG":0,"RECORTES CERO-GV":0,"NCa":0,"PACT":0,"ARA-MES-ESQUERRA":0,"GBAI":0,"PUM+J":0,"EN MAREA":0,"PCTE":0,"EL PI":0,"AxSI":0,"PCOE":0,"PCPE":0,"AVANT ADELANTE LOS VERDES":0,"EB":0,"CpM":0,"SOMOS REGIÓN":0,"PCPA":0,"PH":0,"UIG-SOM-CUIDES":0,"ERPV":0,"IZQP":0,"PCPC":0,"AHORA CANARIAS":0,"CxG":0,"PPSO":0,"CNV":0,"PREPAL":0,"C.Ex-C.R.Ex-P.R.Ex":0,"PR+":0,"P-LIB":0,"CILU-LINARES":0,"ANDECHA ASTUR":0,"JF":0,"PYLN":0,"FIA":0,"FE de las JONS":0,"SOLIDARIA":0,"F8":0,"DPL":0,"UNIÓN REGIONALISTA":0,"centrados":0,"DP":0,"VOU":0,"PDSJE-UDEC":0,"IZAR":0,"RISA":0,"C 21":0,"+MAS+":0,"UDT":0}"#; + let response_string = r#"{"PSOE":123,"PP":66,"Cs":57,"UP":42,"VOX":24,"ERC-SOBIRANISTES":15,"JxCAT-JUNTS":7,"PNV":6,"EH Bildu":4,"CCa-PNC":2,"NA+":2,"COMPROMÍS 2019":1,"PRC":1,"PACMA":0,"FRONT REPUBLICÀ":0,"BNG":0,"RECORTES CERO-GV":0,"NCa":0,"PACT":0,"ARA-MES-ESQUERRA":0,"GBAI":0,"PUM+J":0,"EN MAREA":0,"PCTE":0,"EL PI":0,"AxSI":0,"PCOE":0,"PCPE":0,"AVANT ADELANTE LOS VERDES":0,"EB":0,"CpM":0,"SOMOS REGIÓN":0,"PCPA":0,"PH":0,"UIG-SOM-CUIDES":0,"ERPV":0,"IZQP":0,"PCPC":0,"AHORA CANARIAS":0,"CxG":0,"PPSO":0,"CNV":0,"PREPAL":0,"C.Ex-C.R.Ex-P.R.Ex":0,"PR+":0,"P-LIB":0,"CILU-LINARES":0,"ANDECHA ASTUR":0,"JF":0,"PYLN":0,"FIA":0,"FE de las JONS":0,"SOLIDARIA":0,"F8":0,"DPL":0,"UNIÓN REGIONALISTA":0,"centrados":0,"DP":0,"VOU":0,"PDSJE-UDEC":0,"IZAR":0,"RISA":0,"C 21":0,"+MAS+":0,"UDT":0}"#; let expected = RadonTypes::Float(RadonFloat::from(123)); let aggregate = RADAggregate { @@ -1116,7 +1115,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1157,10 +1156,10 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"event":{"homeTeam":{"name":"Ryazan-VDV","slug":"ryazan-vdv","gender":"F","national":false,"id":171120,"shortName":"Ryazan-VDV","subTeams":[]},"awayTeam":{"name":"Olympique Lyonnais","slug":"olympique-lyonnais","gender":"F","national":false,"id":26245,"shortName":"Lyon","subTeams":[]},"homeScore":{"current":0,"display":0,"period1":0,"normaltime":0},"awayScore":{"current":9,"display":9,"period1":5,"normaltime":9}}}"#; + let response_string = r#"{"event":{"homeTeam":{"name":"Ryazan-VDV","slug":"ryazan-vdv","gender":"F","national":false,"id":171120,"shortName":"Ryazan-VDV","subTeams":[]},"awayTeam":{"name":"Olympique Lyonnais","slug":"olympique-lyonnais","gender":"F","national":false,"id":26245,"shortName":"Lyon","subTeams":[]},"homeScore":{"current":0,"display":0,"period1":0,"normaltime":0},"awayScore":{"current":9,"display":9,"period1":5,"normaltime":9}}}"#; let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) From 68c7648180863c17c4271cdbeaca9a8a9ed344d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Thu, 26 Oct 2023 13:24:08 +0200 Subject: [PATCH 13/20] chore: attend pr review comments --- rad/src/lib.rs | 68 ++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 2af893083..a30cd0309 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -13,7 +13,7 @@ use witnet_data_structures::{ RADAggregate, RADRequest, RADRetrieve, RADTally, RADType, }, radon_report::{RadonReport, ReportContext, RetrievalMetadata, Stage, TallyMetaData}, - witnessing::WitnessingConfig, + witnessing::WitnessingConfig, radon_error::RadonError, }; use witnet_net::client::http::WitnetHttpClient; pub use witnet_net::Uri; @@ -293,42 +293,44 @@ async fn http_response( let (parts, mut body) = response.into_parts(); let response: RadonTypes; - match parts.headers.get("accept-ranges") { - Some(_) => { - // http response is a binary stream - let mut response_bytes = Vec::::default(); - match retrieve.kind { - RADType::HttpHead => { - // todo: assert http-head responses should never return binary streams - } - _ => { - // todo: before reading the response buffer, an error should thrown it was too big - body.read_to_end(&mut response_bytes).await.map_err(|x| { - RadError::HttpOther { - message: x.to_string(), - } - })?; - } + if parts.headers.contains_key("accept-ranges") { + // http response is a binary stream + let mut response_bytes = Vec::::default(); + match retrieve.kind { + RADType::HttpHead => { + response = RadonTypes::RadonError( + RadonError::try_from(RadError::BufferIsNotValue { + description: String::from("Unsupported binary streams from HTTP/HEAD sources") + }) + .unwrap() + ); + } + _ => { + // todo: before reading the response buffer, an error should be thrown if it was too big + body.read_to_end(&mut response_bytes).await.map_err(|x| { + RadError::HttpOther { + message: x.to_string(), + } + })?; + response = RadonTypes::from(RadonBytes::from(response_bytes)); } - response = RadonTypes::from(RadonBytes::from(response_bytes)); } - _ => { - // response is a string - let mut response_string = String::default(); - match retrieve.kind { - RADType::HttpHead => { - response_string = format!("{:?}", parts.headers); - } - _ => { - body.read_to_string(&mut response_string) - .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), - })?; - } + } else { + // response is a string + let mut response_string = String::default(); + match retrieve.kind { + RADType::HttpHead => { + response_string = format!("{:?}", parts.headers); + } + _ => { + body.read_to_string(&mut response_string) + .await + .map_err(|x| RadError::HttpOther { + message: x.to_string(), + })?; } - response = RadonTypes::from(RadonString::from(response_string)); } + response = RadonTypes::from(RadonString::from(response_string)); } let result = handle_response_with_data_report(retrieve, response, context, settings); From 184f768850a16a201679fc25f67b051f7d415f0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Thu, 26 Oct 2023 13:27:44 +0200 Subject: [PATCH 14/20] chore: cargo clippy --fix --- data_structures/src/chain/mod.rs | 4 ++-- data_structures/src/data_request.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs index 7ed2e38e0..8b9b3d729 100644 --- a/data_structures/src/chain/mod.rs +++ b/data_structures/src/chain/mod.rs @@ -2690,7 +2690,7 @@ impl TransactionsPool { for input in &vt_tx.body.inputs { self.output_pointer_map .entry(input.output_pointer) - .or_insert_with(Vec::new) + .or_default() .push(vt_tx.hash()); } @@ -2715,7 +2715,7 @@ impl TransactionsPool { for input in &dr_tx.body.inputs { self.output_pointer_map .entry(input.output_pointer) - .or_insert_with(Vec::new) + .or_default() .push(dr_tx.hash()); } diff --git a/data_structures/src/data_request.rs b/data_structures/src/data_request.rs index 6dd3493a1..fc3823037 100644 --- a/data_structures/src/data_request.rs +++ b/data_structures/src/data_request.rs @@ -143,7 +143,7 @@ impl DataRequestPool { self.data_requests_by_epoch .entry(epoch) - .or_insert_with(HashSet::new) + .or_default() .insert(dr_hash); self.data_request_pool.insert(dr_hash, dr_state); From cd4641c84aae3593d2395dd3f63147c7875a2f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Thu, 26 Oct 2023 19:14:34 +0200 Subject: [PATCH 15/20] feat(rad): add new RadonErrors::BufferIsNotValue --- data_structures/src/radon_error.rs | 2 ++ rad/src/error.rs | 5 +++++ rad/src/lib.rs | 13 +++++-------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/data_structures/src/radon_error.rs b/data_structures/src/radon_error.rs index 7d3176629..6fdf30d26 100644 --- a/data_structures/src/radon_error.rs +++ b/data_structures/src/radon_error.rs @@ -34,6 +34,8 @@ pub enum RadonErrors { HTTPError = 0x30, /// Al least one of the sources could not be retrieved, timeout reached. RetrieveTimeout = 0x31, + /// Value cannot be extracted from binary buffer + BufferIsNotValue = 0x32, // Math errors /// Math operator caused an underflow. Underflow = 0x40, diff --git a/rad/src/error.rs b/rad/src/error.rs index a413f47c4..b9f2086b6 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -431,6 +431,10 @@ impl RadError { } Ok(RadonError::new(match kind { + RadonErrors::BufferIsNotValue => { + let (description,) = deserialize_args(error_args)?; + RadError::BufferIsNotValue { description } + } RadonErrors::RequestTooManySources => RadError::RequestTooManySources, RadonErrors::ScriptTooManyCalls => RadError::ScriptTooManyCalls, RadonErrors::Overflow => RadError::Overflow, @@ -574,6 +578,7 @@ impl RadError { pub fn try_into_error_code(&self) -> Result { Ok(match self { RadError::Unknown => RadonErrors::Unknown, + RadError::BufferIsNotValue { .. } => RadonErrors::BufferIsNotValue, RadError::SourceScriptNotCBOR => RadonErrors::SourceScriptNotCBOR, RadError::SourceScriptNotArray => RadonErrors::SourceScriptNotArray, RadError::SourceScriptNotRADON => RadonErrors::SourceScriptNotRADON, diff --git a/rad/src/lib.rs b/rad/src/lib.rs index a30cd0309..5de2f1379 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -13,7 +13,7 @@ use witnet_data_structures::{ RADAggregate, RADRequest, RADRetrieve, RADTally, RADType, }, radon_report::{RadonReport, ReportContext, RetrievalMetadata, Stage, TallyMetaData}, - witnessing::WitnessingConfig, radon_error::RadonError, + witnessing::WitnessingConfig, }; use witnet_net::client::http::WitnetHttpClient; pub use witnet_net::Uri; @@ -298,12 +298,9 @@ async fn http_response( let mut response_bytes = Vec::::default(); match retrieve.kind { RADType::HttpHead => { - response = RadonTypes::RadonError( - RadonError::try_from(RadError::BufferIsNotValue { - description: String::from("Unsupported binary streams from HTTP/HEAD sources") - }) - .unwrap() - ); + return Err(RadError::BufferIsNotValue { + description: String::from("Unsupported binary streams from HTTP/HEAD sources") + }); } _ => { // todo: before reading the response buffer, an error should be thrown if it was too big @@ -312,9 +309,9 @@ async fn http_response( message: x.to_string(), } })?; - response = RadonTypes::from(RadonBytes::from(response_bytes)); } } + response = RadonTypes::from(RadonBytes::from(response_bytes)); } else { // response is a string let mut response_string = String::default(); From f1314a4626a1f47bad478b0d4d6964897ba9a689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:36:02 +0100 Subject: [PATCH 16/20] fix(rad): http-head response headers can contain 'accept-ranges' --- rad/src/lib.rs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 5de2f1379..289178f98 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -293,24 +293,16 @@ async fn http_response( let (parts, mut body) = response.into_parts(); let response: RadonTypes; - if parts.headers.contains_key("accept-ranges") { + if retrieve.kind != RADType::HttpHead && parts.headers.contains_key("accept-ranges") { // http response is a binary stream let mut response_bytes = Vec::::default(); - match retrieve.kind { - RADType::HttpHead => { - return Err(RadError::BufferIsNotValue { - description: String::from("Unsupported binary streams from HTTP/HEAD sources") - }); - } - _ => { - // todo: before reading the response buffer, an error should be thrown if it was too big - body.read_to_end(&mut response_bytes).await.map_err(|x| { - RadError::HttpOther { - message: x.to_string(), - } - })?; + + // todo: before reading the response buffer, an error should be thrown if it was too big + body.read_to_end(&mut response_bytes).await.map_err(|x| { + RadError::HttpOther { + message: x.to_string(), } - } + })?; response = RadonTypes::from(RadonBytes::from(response_bytes)); } else { // response is a string From 1f2402b1b953e94e54ae5fb9e07038f765aa8951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:39:29 +0100 Subject: [PATCH 17/20] feat(rad): implement StringReplace --- rad/src/operators/mod.rs | 1 + rad/src/operators/string.rs | 13 +++++++++++++ rad/src/types/string.rs | 3 +++ 3 files changed, 17 insertions(+) diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 6964f2623..1ccceed8f 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -106,6 +106,7 @@ pub enum RadonOpCodes { StringParseXMLMap = 0x78, StringToLowerCase = 0x79, StringToUpperCase = 0x7A, + StringReplace = 0x7B, } impl fmt::Display for RadonOpCodes { diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index e23914e99..a5dd6197b 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -7,6 +7,8 @@ use std::{ use serde_cbor::value::{from_value, Value}; use serde_json::Value as JsonValue; +use regex::Regex; + use crate::{ error::RadError, hash_functions::{self, RadonHashFunctions}, @@ -224,6 +226,17 @@ pub fn hash(input: &RadonString, args: &[Value]) -> Result Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "StringReplace".to_string(), + args: args.to_vec(), + }; + let regex = RadonString::try_from(args.first().ok_or_else(wrong_args)?.to_owned())?; + let replacement = RadonString::try_from(args.get(1).ok_or_else(wrong_args)?.to_owned())?; + Ok(RadonString::from(input.value().as_str().replace(regex.value().as_str(), replacement.value().as_str()))) +} + pub fn string_match(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index 16eb9ccf3..fb42ad9b5 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -143,6 +143,9 @@ impl Operable for RadonString { (RadonOpCodes::StringParseXMLMap, None) => string_operators::parse_xml_map(self) .map(RadonTypes::from) .map_err(Into::into), + (RadonOpCodes::StringReplace, Some(args)) => { + string_operators::string_replace(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_STRING_TYPE_NAME.to_string(), operator: op_code.to_string(), From 533c10367b7cd788e22163d8425b5252358a89ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:42:44 +0100 Subject: [PATCH 18/20] feat(rad): implement StringSlice --- rad/Cargo.toml | 1 + rad/src/operators/mod.rs | 1 + rad/src/operators/string.rs | 22 ++++++++++++++++++++++ rad/src/types/string.rs | 3 +++ 4 files changed, 27 insertions(+) diff --git a/rad/Cargo.toml b/rad/Cargo.toml index d9bef65f3..965e21105 100644 --- a/rad/Cargo.toml +++ b/rad/Cargo.toml @@ -26,6 +26,7 @@ rand = "0.7.3" serde = "1.0.111" serde_cbor = "0.11.2" serde_json = "1.0.96" +slicestring = "0.3.2" # the url crate is used to perform additional validations before passing arguments to the surf http client # the version of url must be kept in sync with the version used by surf in the `witnet_net` crate url = "2.1.1" diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 1ccceed8f..5f483f280 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -107,6 +107,7 @@ pub enum RadonOpCodes { StringToLowerCase = 0x79, StringToUpperCase = 0x7A, StringReplace = 0x7B, + StringSlice = 0x7C, } impl fmt::Display for RadonOpCodes { diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index a5dd6197b..f7560958a 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -7,6 +7,7 @@ use std::{ use serde_cbor::value::{from_value, Value}; use serde_json::Value as JsonValue; +use slicestring::Slice; use regex::Regex; use crate::{ @@ -237,6 +238,27 @@ pub fn string_replace(input: &RadonString, args: &[Value]) -> Result Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "StringSlice".to_string(), + args: args.to_vec(), + }; + let mut end_index: usize = input.value().len(); + match args.len() { + 2 => { + let start_index = from_value::(args[0].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + end_index = from_value::(args[1].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + Ok(RadonString::from(input.value().as_str().slice(start_index..end_index))) + } + 1 => { + let start_index = from_value::(args[0].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + Ok(RadonString::from(input.value().as_str().slice(start_index..end_index))) + } + _ => Err(wrong_args()) + } +} + pub fn string_match(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index fb42ad9b5..fce29c114 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -146,6 +146,9 @@ impl Operable for RadonString { (RadonOpCodes::StringReplace, Some(args)) => { string_operators::string_replace(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) } + (RadonOpCodes::StringSlice, Some(args)) => { + string_operators::string_slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_STRING_TYPE_NAME.to_string(), operator: op_code.to_string(), From b3322b1bdc795f4cf8590de5784ad097c990178d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:43:54 +0100 Subject: [PATCH 19/20] feat(rad): implement StringSplit --- Cargo.lock | 8 ++++++++ rad/Cargo.toml | 1 + rad/src/operators/mod.rs | 1 + rad/src/operators/string.rs | 13 ++++++++++++- rad/src/types/string.rs | 3 +++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 8580a36aa..de24156e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3719,6 +3719,12 @@ dependencies = [ "autocfg 1.1.0", ] +[[package]] +name = "slicestring" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7b0ad4ee74da25b3ff80bc5aa633bf8b1e0bbaf577c6f44529a26ad2d9df75" + [[package]] name = "sluice" version = "0.5.5" @@ -5211,9 +5217,11 @@ dependencies = [ "num_enum", "ordered-float", "rand 0.7.3", + "regex", "serde", "serde_cbor", "serde_json", + "slicestring", "url", "witnet_config", "witnet_crypto", diff --git a/rad/Cargo.toml b/rad/Cargo.toml index 965e21105..2a0d478de 100644 --- a/rad/Cargo.toml +++ b/rad/Cargo.toml @@ -23,6 +23,7 @@ minidom = { git = "https://github.com/witnet/xmpp-rs", rev = "bc8a33ff5da95ee403 num_enum = "0.4.2" ordered-float = "3.0" rand = "0.7.3" +regex = "1.4.2" serde = "1.0.111" serde_cbor = "0.11.2" serde_json = "1.0.96" diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 5f483f280..32d6961c1 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -108,6 +108,7 @@ pub enum RadonOpCodes { StringToUpperCase = 0x7A, StringReplace = 0x7B, StringSlice = 0x7C, + StringSplit = 0x7D, } impl fmt::Display for RadonOpCodes { diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index f7560958a..3149a9cc7 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -259,10 +259,21 @@ pub fn string_slice(input: &RadonString, args: &[Value]) -> Result Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "StringSplit".to_string(), + args: args.to_vec(), + }; + let pattern = RadonString::try_from(args.first().ok_or_else(wrong_args)?.to_owned())?; + let parts: Vec = Regex::new(pattern.value().as_str()).unwrap().split(input.value().as_str()).map(|part| RadonTypes::from(RadonString::from(part))).collect(); + Ok(RadonArray::from(parts)) +} + pub fn string_match(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), - operator: "String match".to_string(), + operator: "StringMatch".to_string(), args: args.to_vec(), }; diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index fce29c114..d12fa79bd 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -149,6 +149,9 @@ impl Operable for RadonString { (RadonOpCodes::StringSlice, Some(args)) => { string_operators::string_slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) } + (RadonOpCodes::StringSplit, Some(args)) => { + string_operators::string_split(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_STRING_TYPE_NAME.to_string(), operator: op_code.to_string(), From 88c867496d8414466a21625b2442d290250580ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:45:18 +0100 Subject: [PATCH 20/20] feat(rad): first approach to ArrayJoin --- rad/src/operators/array.rs | 34 ++++++++++++++++++++++++++++++++++ rad/src/operators/mod.rs | 2 +- rad/src/types/array.rs | 1 + 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/rad/src/operators/array.rs b/rad/src/operators/array.rs index b4aac9674..c86f7778e 100644 --- a/rad/src/operators/array.rs +++ b/rad/src/operators/array.rs @@ -109,6 +109,40 @@ fn get_numeric_string(input: &RadonArray, args: &[Value]) -> Result Result { + // Join not applicable if the input array is not homogeneous + if !input.is_homogeneous() { + return Err(RadError::UnsupportedOpNonHomogeneous { + operator: "ArrayJoin".to_string(), + }); + } + let separator = if args.len() > 0 { + from_value::(args[0].to_owned()).unwrap_or_default() + } else { + String::from("") + }; + match input.value().first() { + Some(RadonTypes::String(_)) => { + let string_list: Vec = input.value().into_iter().map(|item| + RadonString::try_from(item).unwrap_or_default().value() + ).collect(); + Ok(RadonTypes::from(RadonString::from(string_list.join(separator.as_str())))) + Some(first_item) => { + return Err(RadError::UnsupportedOperator { + input_type: first_item.radon_type_name().to_string(), + operator: "ArrayJoin".to_string(), + args: Some(args.to_vec()) + }); + } + _ => { + return Err(RadError::EmptyArray) + } + } +} + pub fn map( input: &RadonArray, args: &[Value], diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 32d6961c1..b1889c241 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -29,7 +29,7 @@ pub enum RadonOpCodes { // Array operator codes (start at 0x10) ArrayCount = 0x10, ArrayFilter = 0x11, - // ArrayFlatten = 0x12, + ArrayJoin = 0x12, ArrayGetArray = 0x13, ArrayGetBoolean = 0x14, ArrayGetBytes = 0x15, diff --git a/rad/src/types/array.rs b/rad/src/types/array.rs index 1c769e308..ce888f7f0 100644 --- a/rad/src/types/array.rs +++ b/rad/src/types/array.rs @@ -166,6 +166,7 @@ impl Operable for RadonArray { array_operators::get::(self, args).map(RadonTypes::from) } (RadonOpCodes::ArrayFilter, Some(args)) => array_operators::filter(self, args, context), + (RadonOpCodes::ArrayJoin, Some(args)) => array_operators::join(self, args), (RadonOpCodes::ArrayMap, Some(args)) => array_operators::map(self, args, context), (RadonOpCodes::ArrayReduce, Some(args)) => array_operators::reduce(self, args, context), (RadonOpCodes::ArraySort, Some(args)) => array_operators::sort(self, args, context),