Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(host): extract worker message handling #307

Merged
merged 27 commits into from
Jul 7, 2024
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
49b5798
feat(host): extract worker message handling
petarvujovic98 Jul 2, 2024
e872f2a
[WIP](host): add initial cancel API for proof tasks
petarvujovic98 Jul 2, 2024
3365a6b
fix(host): add return expression
petarvujovic98 Jul 2, 2024
0156b8e
fix(host): wrap prover into a option string
petarvujovic98 Jul 2, 2024
446b754
fix(host): cleanup imports
petarvujovic98 Jul 2, 2024
0191f01
refactor(host): move proof handlers into separate module
petarvujovic98 Jul 3, 2024
931d97b
feat(host): use actor pattern to spawn tasks
petarvujovic98 Jul 3, 2024
1d74385
refactor(host): clean up task spawning
petarvujovic98 Jul 3, 2024
8ec60a1
Merge branch 'main' into v2-api-cleanup
petarvujovic98 Jul 3, 2024
a255876
chore(deps): remove hyper
petarvujovic98 Jul 3, 2024
8f03ef0
chore(deps): update dependencies
petarvujovic98 Jul 3, 2024
b768d2a
feat(host): add concurrency limit to proof tasks
petarvujovic98 Jul 3, 2024
8b5c051
fix(pipeline): convert buffer types
petarvujovic98 Jul 3, 2024
7275c10
fix(harness): pass second argument
petarvujovic98 Jul 3, 2024
c045f3c
refactor(host): move allocator next to memory module
petarvujovic98 Jul 3, 2024
144e4bd
feat(host): handle task cancell status and refactor handler
petarvujovic98 Jul 3, 2024
5658f75
feat(host,task_manager): add documentation to v2 API
petarvujovic98 Jul 4, 2024
9cc3d20
fix(host): fix typo in docs
petarvujovic98 Jul 4, 2024
06bd34c
chore(lib,core,host): use tracing instead of println
petarvujovic98 Jul 4, 2024
82671ff
feat(host): update cancellation route
petarvujovic98 Jul 4, 2024
94b94d4
refactor(host): move proof file into module
petarvujovic98 Jul 4, 2024
5c64134
[WIP](core,host,task_manager): stub prune and report endpoints
petarvujovic98 Jul 4, 2024
e5a98f0
refactor(task_manager): use `TaskDescriptor` inside `TaskReport`
petarvujovic98 Jul 5, 2024
55f91aa
Merge branch 'main' into v2-api-cleanup
petarvujovic98 Jul 5, 2024
935bcb4
feat(host): don't start work unless the latest status is `Registered`
petarvujovic98 Jul 5, 2024
4d56a2c
feat(host): update prune docs with `PruneStatus`
petarvujovic98 Jul 5, 2024
f94e8a7
feat(task_manager): refactor in memory implementation
petarvujovic98 Jul 5, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
218 changes: 127 additions & 91 deletions Cargo.lock

Large diffs are not rendered by default.

11 changes: 5 additions & 6 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ rlp = "0.5.2"
hex = { version = "0.4.3" }
hex-literal = "0.4"
base64-serde = "0.7.0"
base64 = "0.21.7"
base64 = "0.22.1"
libflate = { version = "2.0.0" }
typetag = { version = "0.2.15" }
num_enum = "0.7.2"
Expand All @@ -129,13 +129,12 @@ cap = { git = "https://github.com/brechtpd/cap", branch = "more-stats", features
axum = { version = "0.7.4", features = ["macros"] }
tower-http = { version = "0.5.2", features = ["full"] }
tower = { version = "0.4.13", features = ["full"] }
utoipa-swagger-ui = { version = "6.0.0", features = ["axum"] }
utoipa-swagger-ui = { version = "7.1.0", features = ["axum"] }
utoipa-scalar = { version = "0.1.0", features = ["axum"] }
utoipa = { version = "4.2.0", features = ["axum_extras"] }
structopt = "0.3.24"
prometheus = { version = "0.13.3", features = ["process"] }
tokio = { version = "^1.23", features = ["full"] }
hyper = { version = "0.14.27", features = ["server"] }
reqwest = { version = "0.11.22", features = ["json"] }
url = "2.5.0"
async-trait = "0.1.80"
Expand All @@ -151,7 +150,7 @@ secp256k1 = { version = "0.29", default-features = false, features = [
] }

# macro
syn = { version = "1.0", features = ["full"] }
syn = { version = "2.0.68", features = ["full"] }
quote = "1.0"
proc-macro2 = "1.0"

Expand All @@ -166,7 +165,7 @@ rusqlite = { version = "0.31.0", features = ["bundled"] }
# misc
hashbrown = { version = "0.14", features = ["inline-more"] }
tempfile = "3.8"
cargo_metadata = "0.12.1"
cargo_metadata = "0.18.1"
clap = { version = "4.4.6", features = ["derive", "string", "env"] }
lru_time_cache = "0.11.11"
lazy_static = "1.4.0"
Expand All @@ -176,7 +175,7 @@ cfg-if = "1.0.0"
chrono = { version = "0.4", default-features = false }
assert_cmd = "2.0"
regex = "1.5.4"
rstest = "0.18"
rstest = "0.21"
rand = "0.8.5"
rand_core = "0.6.4"
dirs = "5.0.1"
Expand Down
14 changes: 14 additions & 0 deletions core/src/interfaces.rs
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,20 @@ impl FromStr for ProofType {
}
}

impl TryFrom<u8> for ProofType {
type Error = RaikoError;

fn try_from(value: u8) -> Result<Self, Self::Error> {
match value {
0 => Ok(Self::Native),
1 => Ok(Self::Sp1),
2 => Ok(Self::Sgx),
3 => Ok(Self::Risc0),
_ => Err(RaikoError::Conversion("Invalid u8".to_owned())),
}
}
}

impl ProofType {
/// Run the prover driver depending on the proof type.
pub async fn run_prover(
Expand Down
6 changes: 3 additions & 3 deletions core/src/preflight.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ use reth_evm_ethereum::taiko::decode_anchor;
use reth_primitives::Block;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use tracing::{debug, info, warn};
use tracing::{debug, error, info, warn};

pub async fn preflight<BDP: BlockDataProvider>(
provider: BDP,
Expand Down Expand Up @@ -137,7 +137,7 @@ pub async fn preflight<BDP: BlockDataProvider>(
num_iterations += 1;
}
clear_line();
println!("State data fetched in {num_iterations} iterations");
info!("State data fetched in {num_iterations} iterations");

let provider_db = builder.db.as_mut().unwrap();

Expand Down Expand Up @@ -406,7 +406,7 @@ async fn get_blob_data_blobscan(
let blob: BlobScanData = response.json().await?;
Ok(blob_to_bytes(&blob.data))
} else {
println!(
error!(
"Request {url} failed with status code: {}",
response.status()
);
Expand Down
2 changes: 1 addition & 1 deletion harness/macro/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ impl syn::parse::Parse for EntryArgs {
let test_modules: Option<Punctuated<Path, Token![,]>> = if input.peek(Token![,]) {
input.parse::<Token![,]>()?; // Parse and consume the comma
// Now parse a list of module paths if they are present
Some(input.parse_terminated(Path::parse)?)
Some(input.parse_terminated(Path::parse, Token![,])?)
} else {
None
};
Expand Down
9 changes: 4 additions & 5 deletions host/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ serde = { workspace = true }
serde_with = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true }
hyper = { workspace = true }
env_logger = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
Expand All @@ -70,10 +69,10 @@ cfg-if = { workspace = true }
cap = { workspace = true }

# reth
reth-primitives.workspace = true
reth-evm.workspace = true
reth-evm-ethereum.workspace = true
reth-provider.workspace = true
reth-primitives = { workspace = true }
reth-evm = { workspace = true }
reth-evm-ethereum = { workspace = true }
reth-provider = { workspace = true }

[dev-dependencies]
assert_cmd = { workspace = true }
Expand Down
Loading
Loading