feat!: module task support fully with improved cli

.env.example

@@ -3,5 +3,5 @@ CHAIN_ID=11155111
 RPC_URL=https://goerli.infura.io/v3/your-infura-api-key
 # Optional
 RPC_CHUNK_SIZE=2000
-DATALAKES=0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000001800000000000000000000000000000000000000000000000000000000000000280000000000000000000000000000000000000000000000000000000000000038000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009eb0f600000000000000000000000000000000000000000000000000000000009eb100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000002010f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009eb0f600000000000000000000000000000000000000000000000000000000009eb100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000002010f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009eb0f600000000000000000000000000000000000000000000000000000000009eb100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000002010f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009eb0f600000000000000000000000000000000000000000000000000000000009eb100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000002010f000000000000000000000000000000000000000000000000000000000000
-TASKS=0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000018000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000060617667000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006073756d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000606d696e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000606d6178000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000
+DRY_RUN_CAIRO_PATH= # path for dry run cairo
+SOUND_RUN_CAIRO_PATH= # path for sound run cairo

.gitignore

@@ -65,6 +65,4 @@ tools/
 
 # Ignore specific directories and files
 hdp-cairo/
-build/
 *.pie
-*.json

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "crates/pre-processor",
     "crates/processor",
     "crates/cairo-runner",
+    "crates/hdp",
 ]
 
 [workspace.package]

README.md

@@ -58,61 +58,54 @@ Welcome to Herodotus Data Processor interactive CLI! 🛰️
 
 ## Usage Examples
 
-The following examples demonstrate how to use the HDP CLI to encode various blockchain data elements into a format suitable for processing. Each command highlights a different function of the HDP system, from averaging values to counting values based on specific criteria.
+First locate `.env` file like the one in [example](./.env.example).
 
-Header value with `AVG`:
+Second, run command like below :
 
-```
-hdp encode -a -c {input.file} "avg" -b 4952100 4952110 "header.base_fee_per_gas" 1
+note that this will go though both pre-process -> process step.
+
+```bash
+hdp run -r ${Request file path} -p ${Pre-processor output} -c ${PIE file after process} -o ${Output file after process}
 ```
 
-Account value with `SUM`:
+For a more comprehensive guide on commands available on `hdp run`:
 
-```
-hdp encode -a -c {input.file} "sum" -b 4952100 4952110 "account.0x7f2c6f930306d3aa736b3a6c6a98f512f74036d4.nonce" 2
-```
+```console
+❯ hdp run --help
+Run batch of tasks base on request json file
 
-Storage value with `AVG`:
+Usage: hdp run [OPTIONS] --request-file <REQUEST_FILE> --preprocessor-output-file <PREPROCESSOR_OUTPUT_FILE>
 
-```
-hdp encode -a -c {input.file} "avg" -b 5382810 5382820 "storage.0x75CeC1db9dCeb703200EAa6595f66885C962B920.0x0000000000000000000000000000000000000000000000000000000000000002" 1
-```
+Options:
+  -r, --request-file <REQUEST_FILE>
+          Pass request as json file
 
-Account value with `COUNT`:
+      --rpc-url <RPC_URL>
+          The RPC URL to fetch the data.
 
-```
-hdp encode -a -c {input.file} "count" "gt.1000" -b 4952100 4952110 "account.0x7f2c6f930306d3aa736b3a6c6a98f512f74036d4.nonce" 2
-```
+          Can be overwritten by `RPC_URL` environment variable.
 
-After encoding, you can directly run processing tasks using environmental configurations for RPC and Chain ID, as shown below:
+      --dry-run-cairo-file <DRY_RUN_CAIRO_FILE>
+          dry run contract bootloader program. only used for module task
 
-```bash
-# pro tip: run herodotus data processing with `.env`
-hdp run
+  -p, --preprocessor-output-file <PREPROCESSOR_OUTPUT_FILE>
+          Path to save output file after pre-processing
 
-# run herodotus data processing
-hdp run ${Encoded Task} ${Encoded Datalake} ${Input your RPC Provider -- this example is Etherum Sepolia} ${Input Chain ID that you are target on}
-```
+      --sound-run-cairo-file <SOUND_RUN_CAIRO_FILE>
+          hdp cairo compiled program. main entry point
 
-For a more comprehensive guide on commands available within HDP CLI:
+  -o, --output-file <OUTPUT_FILE>
+          Path to save output file after process
 
-```console
-❯ hdp --help
-Interact Herodotus Data Processor via CLI
+          This will trigger processing(=pie generation) step
 
-Usage: hdp <COMMAND>
+  -c, --cairo-pie-file <CAIRO_PIE_FILE>
+          Path to save pie file
 
-Commands:
-  start       New to the HDP CLI? Start here!
-  encode      Encode the task and datalake in batched format test purposes
-  decode      Decode batch tasks and datalakes
-  decode-one  Decode one task and one datalake (not batched format)
-  run         Run the evaluator
-  help        Print this message or the help of the given subcommand(s)
+          This will trigger processing(=pie generation) step
 
-Options:
-  -h, --help     Print help
-  -V, --version  Print version
+  -h, --help
+          Print help (see a summary with '-h')
 ```
 
 ## Integration Testing
@@ -128,6 +121,12 @@ The core soundness of HDP relies on generating the correct input file and runnin
 - **SUM, AVG, MIN, MAX, COUNT**: These functions are supported only for fields with numeric values.
 - **SLR**: Simple linear regression written in Cairo 1. The input array should contain more than 2 elements.
 
+### Context Required Operation
+
+For a practical example of how to implement context-sensitive operations, refer to the implementation of the `COUNT` function. This example shows how to pass and utilize additional context for operations, which can be particularly useful for conditional processing or complex calculations.
+
+During `SLR` computation, we also need a context to use as the target index for computation. Since `SLR` is not supported during the preprocessing step, we simply pass the encoded task that contains the function context, and the Cairo program will handle this computation based on the provided index.
+
 ### Function Support Matrix
 
 Here is the support matrix indicating which blockchain elements are tested for each aggregate function. The matrix highlights fields where these functions are applicable.
@@ -179,39 +178,14 @@ _Note: Fields marked with "-" are not applicable for the specified aggregate fun
 cargo make run-ci-flow
 ```
 
-## Defining Your Own Module
-
-For developers interested in extending the functionality of HDP by adding new modules, follow the steps outlined below. This guide assumes a basic understanding of Rust and its module system.
-
-### Getting Started
-
-1. **Module Location**: Start by creating a new module within the `aggregate_fn` directory. You can find this at [aggregation_fn/mod.rs](./crates/primitives/src/aggregate_fn).
-
-2. **Define Enum**: Define your new function as an enum in the [file](./crates/primitives/src/aggregate_fn). Make sure to add match arms for the new enum variants in the implementation.
-
-3. **Handle Data Types**: Depending on the expected input type for your function:
-   - **Integer Inputs**: Use [`U256`](https://docs.rs/alloy-primitives/latest/alloy_primitives/index.html#reexport.U256) for handling large integers compatible with Ethereum's numeric constraints.
-   - **String Inputs**: Use Rust's standard `String` type for text data.
-
-### Context Required Operation
-
-For a practical example of how to implement context-sensitive operations, refer to the implementation of the `COUNT` function. This example shows how to pass and utilize additional context for operations, which can be particularly useful for conditional processing or complex calculations.
-
-During `SLR` computation, we also need a context to use as the target index for computation. Since `SLR` is not supported during the preprocessing step, we simply pass the encoded task that contains the function context, and the Cairo program will handle this computation based on the provided index.
-
-### Testing Your Module
-
-After implementing your new function, it's crucial to verify its functionality:
-
-- **Create Unit Tests**: Add tests in the corresponding test file in the `tests` directory. Ensure your tests cover all new logic to maintain stability and reliability.
-- **Test for Integer Types**: Pay special attention to functions that handle integer types, ensuring they correctly process and output values fitting within a `bytes32` length, reflecting Ethereum's data type constraints.
-
 ### Local Run
 
 To run HDP in a stable environment locally, you need to have `cairo-run` installed with the necessary tools in the correct path and locate the compiled Cairo program. If these steps sound tricky to you, just use the Docker image.
 
 To mount in a container environment, you need to create empty `input.json`, `output.json`, and `cairo.pie` files in the root directory of the host machine before running it.
 
+And locate `requeset.json` file on root that contains intended request format.
+
 ```sh
 docker-compose build
 

cli/Cargo.toml

@@ -18,17 +18,20 @@ name = "hdp"
 path = "src/main.rs"
 
 [dependencies]
-hdp-preprocessor = { workspace = true }
-hdp-processor = { workspace = true }
+tracing-subscriber = { version = "0.3.0", features = ["env-filter"] }
+hdp = { path = "../crates/hdp", default-features = false, features = [
+    "preprocessor",
+    "processor",
+    "provider",
+    "primitives",
+] }
 anyhow = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
-hdp-provider = { workspace = true }
-hdp-primitives = { workspace = true }
 serde_json = { workspace = true }
+serde = { workspace = true }
 starknet = { workspace = true }
 clap = { version = "4.4.4", features = ["derive"] }
 dotenv = "0.15.0"
-tracing-subscriber = "0.3.0"
 inquire = "0.7.4"
 alloy = { workspace = true }