Merge pull request #131 from HerodotusDev/slr-rm

removing `SLR`

README.md

@@ -35,7 +35,7 @@ The Data Processor CLI serves as an essential tool for developers working with C
 
 ```bash
 # Install with cargo
-❯ cargo install --git https://github.com/HerodotusDev/hdp/ --tag v0.5.0 --locked --force hdp-cli
+❯ cargo install --git https://github.com/HerodotusDev/hdp/ --tag {TAG} --locked --force hdp-cli
 ```
 
 ### Build from source
@@ -75,7 +75,7 @@ Second, run command like below :
 note that this will go though both pre-process -> process step.
 
 ```bash
-hdp run -r ${Request file path} -p ${Pre-processor output} -c ${PIE file after process} -o ${Output file after process}
+hdp run -r ${Request file path} -p ${Program input file path} -b ${Batch proof file path} -c ${PIE file after process}
 ```
 
 For a more comprehensive guide on commands available on `hdp run`:
@@ -84,41 +84,27 @@ For a more comprehensive guide on commands available on `hdp run`:
 ❯ hdp run --help
 Run batch of tasks base on request json file
 
-Usage: hdp run [OPTIONS] --request-file <REQUEST_FILE>
+Usage: hdp run [OPTIONS] --request-file <REQUEST_FILE> --program-input-file <PROGRAM_INPUT_FILE>
 
 Options:
   -r, --request-file <REQUEST_FILE>
           Pass request as json file
-
       --rpc-url <RPC_URL>
-          The RPC URL to fetch the data.
-
-          Can be overwritten by `RPC_URL` environment variable.
-
+          The RPC URL to fetch the data
       --dry-run-cairo-file <DRY_RUN_CAIRO_FILE>
           dry run contract bootloader program. only used for module task
-
-  -p, --preprocessor-output-file <PREPROCESSOR_OUTPUT_FILE>
-          Path to save output file after pre-processing
-
+  -p, --program-input-file <PROGRAM_INPUT_FILE>
+          Path to save program input file after pre-processing
       --cairo-format
-          Set this boolean to true to generate cairo format preprocessor_output_file
-
+          Set this boolean to true to generate cairo format program_input_file
+  -b, --batch-proof-file <BATCH_PROOF_FILE>
+          Path to save batch proof file after pre-processing
       --sound-run-cairo-file <SOUND_RUN_CAIRO_FILE>
           hdp cairo compiled program. main entry point
-
-  -o, --output-file <OUTPUT_FILE>
-          Path to save output file after process
-
-          This will trigger processing(=pie generation) step
-
   -c, --cairo-pie-file <CAIRO_PIE_FILE>
           Path to save pie file
-
-          This will trigger processing(=pie generation) step
-
   -h, --help
-          Print help (see a summary with '-h')
+          Print help (see more with '--help')
 ```
 
 ## Integration Testing
@@ -132,50 +118,47 @@ The core soundness of HDP relies on generating the correct input file and runnin
 ### Supported Aggregate Functions
 
 - **SUM, AVG, MIN, MAX, COUNT**: These functions are supported only for fields with numeric values.
-- **SLR**: Simple linear regression written in Cairo 1. The input array should contain more than 2 elements.
 
 ### Context Required Operation
 
 For a practical example of how to implement context-sensitive operations, refer to the implementation of the `COUNT` function. This example shows how to pass and utilize additional context for operations, which can be particularly useful for conditional processing or complex calculations.
 
-During `SLR` computation, we also need a context to use as the target index for computation. Since `SLR` is not supported during the preprocessing step, we simply pass the encoded task that contains the function context, and the Cairo program will handle this computation based on the provided index.
-
 ### Function Support Matrix
 
 Here is the support matrix indicating which blockchain elements are tested for each aggregate function. The matrix highlights fields where these functions are applicable.
 
-| Field Description                | SUM | AVG | MIN | MAX | COUNT | SLR |
-| -------------------------------- | --- | --- | --- | --- | ----- | --- |
-| `account.nonce`                  | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `account.balance`                | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `account.storage_root`           | -   | -   | -   | -   | -     | -   |
-| `account.code_hash`              | -   | -   | -   | -   | -     | -   |
-| `storage.key` (numeric value)    | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `storage.key` (hash value)       | -   | -   | -   | -   | -     | -   |
-| `header.difficulty`              | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `header.gas_limit`               | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `header.gas_used`                | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `header.timestamp`               | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `header.base_fee_per_gas`        | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `header.blob_gas_used`           | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `header.excess_blob_gas`         | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `header.nonce`                   | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| Other `header` elements          | -   | -   | -   | -   | -     | -   |
-| `tx.nonce`                       | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.gas_price`                   | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.gas_limit`                   | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.value`                       | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.v`                           | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.r`                           | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.s`                           | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.chain_id`                    | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.max_fee_per_gas`             | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.max_priority_fee_per_gas`    | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx.max_fee_per_blob_gas`        | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| Other `tx` elements              | -   | -   | -   | -   | -     | -   |
-| `tx_receipt.success`             | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| `tx_receipt.cumulative_gas_used` | ✅  | ✅  | ✅  | ✅  | ✅    | ✅  |
-| Other `tx_receipt` elements      | -   | -   | -   | -   | -     | -   |
+| Field Description                | SUM | AVG | MIN | MAX | COUNT |
+| -------------------------------- | --- | --- | --- | --- | ----- |
+| `account.nonce`                  | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `account.balance`                | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `account.storage_root`           | -   | -   | -   | -   | -     |
+| `account.code_hash`              | -   | -   | -   | -   | -     |
+| `storage.key` (numeric value)    | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `storage.key` (hash value)       | -   | -   | -   | -   | -     |
+| `header.difficulty`              | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `header.gas_limit`               | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `header.gas_used`                | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `header.timestamp`               | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `header.base_fee_per_gas`        | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `header.blob_gas_used`           | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `header.excess_blob_gas`         | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `header.nonce`                   | ✅  | ✅  | ✅  | ✅  | ✅    |
+| Other `header` elements          | -   | -   | -   | -   | -     |
+| `tx.nonce`                       | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.gas_price`                   | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.gas_limit`                   | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.value`                       | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.v`                           | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.r`                           | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.s`                           | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.chain_id`                    | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.max_fee_per_gas`             | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.max_priority_fee_per_gas`    | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx.max_fee_per_blob_gas`        | ✅  | ✅  | ✅  | ✅  | ✅    |
+| Other `tx` elements              | -   | -   | -   | -   | -     |
+| `tx_receipt.success`             | ✅  | ✅  | ✅  | ✅  | ✅    |
+| `tx_receipt.cumulative_gas_used` | ✅  | ✅  | ✅  | ✅  | ✅    |
+| Other `tx_receipt` elements      | -   | -   | -   | -   | -     |
 
 _Note: Fields marked with "-" are not applicable for the specified aggregate functions because they do not contain numeric data or the data type is not suitable for these calculations._
 

cli/src/cli.rs

@@ -9,7 +9,7 @@ use crate::{
 };
 use anyhow::Result;
 use clap::Parser;
-use hdp::primitives::processed_types::query::ProcessorInput;
+use hdp::primitives::processed_types::cairo_format::query::ProcessorInput;
 use hdp::primitives::request::{SubmitBatchQuery, Task};
 use hdp::processor::{self, Processor};
 use hdp::{
@@ -68,26 +68,23 @@ pub async fn process_entry_run(args: ProcessArgs) -> Result<()> {
     let config = processor::HdpProcessorConfig::init(
         args.sound_run_cairo_file,
         args.input_file,
-        args.output_file,
         args.cairo_pie_file,
     );
-    let input_string = fs::read_to_string(config.input_file)?;
+    let input_string = fs::read_to_string(&config.input_file)?;
     let preprocessor_result: ProcessorInput = serde_json::from_str(&input_string)
         .expect("Input file is not valid. Check if format is correct");
+
+    info!(
+        "processing the data from {}... ",
+        &config.input_file.display()
+    );
     let processor = Processor::new(config.sound_run_program_path.clone());
-    let processor_result = processor
+    processor
         .process(preprocessor_result, &config.cairo_pie_file)
         .await?;
-    fs::write(
-        &config.processor_output_file,
-        serde_json::to_string_pretty(&processor_result)
-            .map_err(|e| anyhow::anyhow!("Failed to serialize processor result: {}", e))?,
-    )
-    .map_err(|e| anyhow::anyhow!("Unable to write output file: {}", e))?;
 
     info!(
-        "finished processing the data, saved the output file in {} and pie file in {}",
-        &config.processor_output_file.display(),
+        "finished processing the data, saved pie file in {}",
         &config.cairo_pie_file.display()
     );
 
@@ -100,10 +97,10 @@ pub async fn module_entry_run(args: RunModuleArgs) -> Result<()> {
         args.chain_id,
         args.dry_run_cairo_file,
         args.sound_run_cairo_file,
-        args.preprocessor_output_file,
+        args.program_input_file,
         args.cairo_format,
         args.save_fetch_keys_file,
-        args.output_file,
+        args.batch_proof_file,
         args.cairo_pie_file,
     );
     let module_registry = ModuleRegistry::new();
@@ -127,10 +124,10 @@ pub async fn datalake_entry_run(args: RunDatalakeArgs) -> Result<()> {
         args.chain_id,
         None,
         args.sound_run_cairo_file,
-        args.preprocessor_output_file,
+        args.program_input_file,
         args.cairo_format,
         None,
-        args.output_file,
+        args.batch_proof_file,
         args.cairo_pie_file,
     );
     let parsed_datalake = match args.datalake {
@@ -182,10 +179,10 @@ pub async fn entry_run(args: RunArgs) -> Result<()> {
         Some(parsed.destination_chain_id),
         args.dry_run_cairo_file,
         args.sound_run_cairo_file,
-        args.preprocessor_output_file,
+        args.program_input_file,
         args.cairo_format,
         None,
-        args.output_file,
+        args.batch_proof_file,
         args.cairo_pie_file,
     );
     let module_registry = ModuleRegistry::new();

cli/src/commands/process.rs

@@ -12,15 +12,7 @@ pub struct ProcessArgs {
     #[arg(long)]
     pub sound_run_cairo_file: Option<PathBuf>,
 
-    /// Path to save output file after process
-    ///
-    /// This will trigger processing(=pie generation) step
-    #[arg(short, long)]
-    pub output_file: PathBuf,
-
     /// Path to save pie file
-    ///
-    /// This will trigger processing(=pie generation) step
     #[arg(short, long)]
     pub cairo_pie_file: PathBuf,
 }

cli/src/commands/run.rs

@@ -19,27 +19,31 @@ pub struct RunArgs {
     #[arg(long)]
     pub dry_run_cairo_file: Option<PathBuf>,
 
-    /// Path to save output file after pre-processing.
+    /// Path to save program input file after pre-processing.
+    ///
+    /// This will be input data for cairo program
     #[arg(short, long)]
-    pub preprocessor_output_file: Option<PathBuf>,
+    pub program_input_file: PathBuf,
 
-    /// Set this boolean to true to generate cairo format preprocessor_output_file
-    #[arg(long, default_value_t = false, requires("preprocessor_output_file"))]
+    /// Set this boolean to true to generate cairo format program_input_file
+    ///
+    /// By default, program_input_file is generated in cairo format. If you dont want, set this to false.
+    #[arg(long, default_value_t = true)]
     pub cairo_format: bool,
 
+    /// Path to save batch proof file after pre-processing.
+    ///
+    /// This will be used to verify the batch proof on-chain
+    #[arg(short, long, requires("program_input_file"))]
+    pub batch_proof_file: Option<PathBuf>,
+
     /// hdp cairo compiled program. main entry point
     #[arg(long)]
     pub sound_run_cairo_file: Option<PathBuf>,
 
-    /// Path to save output file after process
-    ///
-    /// This will trigger processing(=pie generation) step
-    #[arg(short, long, requires("preprocessor_output_file"))]
-    pub output_file: Option<PathBuf>,
-
     /// Path to save pie file
     ///
     /// This will trigger processing(=pie generation) step
-    #[arg(short, long, requires("preprocessor_output_file"))]
+    #[arg(short, long, requires("program_input_file"))]
     pub cairo_pie_file: Option<PathBuf>,
 }

cli/src/commands/run_datalake.rs

@@ -29,37 +29,39 @@ pub struct RunDatalakeArgs {
     /// The chain id to fetch the datalake
     pub chain_id: Option<ChainId>,
 
-    /// Path to save output file after pre-processing.
+    /// Path to save program input file after pre-processing.
     ///
-    /// This will trigger pre-processing step
+    /// This will be input data for cairo program
     #[arg(short, long)]
-    pub preprocessor_output_file: Option<PathBuf>,
+    pub program_input_file: PathBuf,
 
-    /// Set this boolean to true to generate cairo format preprocessor_output_file
-    #[arg(long, default_value_t = false, requires("preprocessor_output_file"))]
+    /// Set this boolean to true to generate cairo format program_input_file
+    ///
+    /// By default, program_input_file is generated in cairo format. If you dont want, set this to false.
+    #[arg(long, default_value_t = true)]
     pub cairo_format: bool,
 
+    /// Path to save batch proof file after pre-processing.
+    ///
+    /// This will be used to verify the batch proof on-chain
+    #[arg(short, long, requires("program_input_file"))]
+    pub batch_proof_file: Option<PathBuf>,
+
     /// hdp cairo compiled program. main entry point
     #[arg(long)]
     pub sound_run_cairo_file: Option<PathBuf>,
 
-    /// Path to save output file after process
-    ///
-    /// This will trigger processing(=pie generation) step
-    #[arg(short, long, requires("preprocessor_output_file"))]
-    pub output_file: Option<PathBuf>,
-
     /// Path to save pie file
     ///
     /// This will trigger processing(=pie generation) step
-    #[arg(short, long, requires("preprocessor_output_file"))]
+    #[arg(short, long, requires("program_input_file"))]
     pub cairo_pie_file: Option<PathBuf>,
 }
 
 #[derive(Subcommand, Clone, Debug, PartialEq, Eq)]
 pub enum DataLakeCommands {
     #[command(arg_required_else_help = true)]
-    #[command(short_flag = 'b')]
+    #[command(short_flag = 's')]
     BlockSampled {
         /// Block number range start (inclusive)
         block_range_start: BlockNumber,

cli/src/commands/run_module.rs

@@ -47,29 +47,31 @@ pub struct RunModuleArgs {
     #[arg(long)]
     pub dry_run_cairo_file: Option<PathBuf>,
 
-    /// Path to save output file after pre-processing.
+    /// Path to save program input file after pre-processing.
     ///
-    /// This will trigger pre-processing step
+    /// This will be input data for cairo program
     #[arg(short, long)]
-    pub preprocessor_output_file: Option<PathBuf>,
+    pub program_input_file: PathBuf,
 
-    /// Set this boolean to true to generate cairo format preprocessor_output_file
-    #[arg(long, default_value_t = false, requires("preprocessor_output_file"))]
+    /// Set this boolean to true to generate cairo format program_input_file
+    ///
+    /// By default, program_input_file is generated in cairo format. If you dont want, set this to false.
+    #[arg(long, default_value_t = true)]
     pub cairo_format: bool,
 
+    /// Path to save batch proof file after pre-processing.
+    ///
+    /// This will be used to verify the batch proof on-chain
+    #[arg(short, long, requires("program_input_file"))]
+    pub batch_proof_file: Option<PathBuf>,
+
     /// hdp cairo compiled program. main entry point
     #[arg(long)]
     pub sound_run_cairo_file: Option<PathBuf>,
 
-    /// Path to save output file after process
-    ///
-    /// This will trigger processing(=pie generation) step
-    #[arg(short, long, requires("preprocessor_output_file"))]
-    pub output_file: Option<PathBuf>,
-
     /// Path to save pie file
     ///
     /// This will trigger processing(=pie generation) step
-    #[arg(short, long, requires("preprocessor_output_file"))]
+    #[arg(short, long, requires("program_input_file"))]
     pub cairo_pie_file: Option<PathBuf>,
 }