From 0529493359271b769e60aebd86ed3fe897369db4 Mon Sep 17 00:00:00 2001 From: Jason McCampbell Date: Thu, 4 Feb 2021 04:40:02 +0000 Subject: [PATCH] Added connection retry for cases where too many clients attempt to connect at once Improved error message for 'spraycc run' Bumped version to 0.8 --- Cargo.lock | 62 +++++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 3 ++- src/client.rs | 34 ++++++++++++++++++++++------ src/main.rs | 28 +++++++++++++---------- 4 files changed, 105 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2989b0..ac8d1f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,6 +223,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.23.0" @@ -412,6 +423,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + [[package]] name = "proc-macro2" version = "1.0.24" @@ -458,6 +475,28 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha", + "rand_core 0.6.1", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.1", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -473,6 +512,24 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +[[package]] +name = "rand_core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.1", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -571,7 +628,7 @@ dependencies = [ [[package]] name = "spraycc" -version = "0.1.0" +version = "0.8.0" dependencies = [ "bincode", "byteorder", @@ -581,6 +638,7 @@ dependencies = [ "gethostname", "home", "lazy_static", + "rand 0.8.3", "serde", "simple-process-stats", "tempdir", @@ -613,7 +671,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" dependencies = [ - "rand", + "rand 0.4.6", "remove_dir_all", ] diff --git a/Cargo.toml b/Cargo.toml index e38afeb..3876217 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spraycc" -version = "0.1.0" +version = "0.8.0" authors = ["Jason McCampbell "] edition = "2018" @@ -13,6 +13,7 @@ get_if_addrs = "0.5" gethostname = "0" home = "0.5" lazy_static = "1" +rand = "^0.8" serde = { version = "1", features = ["derive"] } simple-process-stats = "^1" tempdir = "0.3" diff --git a/src/client.rs b/src/client.rs index 9f3a2a1..8672ec5 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,12 +4,13 @@ /// results are available. /// extern crate lazy_static; +extern crate rand; extern crate tokio; extern crate which; use lazy_static::lazy_static; -use std::error::Error; use std::ffi::OsString; +use std::{error::Error, time::Duration}; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; use tokio::process; @@ -76,7 +77,7 @@ pub async fn run(args: Vec) -> Result> if run_local { status = run_local_task(task).await?; } else if let Some(callme) = config::read_server_contact_info() { - match TcpStream::connect(callme.addr).await { + match connect_w_retry(&callme).await { Ok(stream) => { let mut conn = ipc::Connection::new(stream); @@ -105,11 +106,7 @@ pub async fn run(args: Vec) -> Result> output_files.pop().unwrap().shutdown().await? } } - Err(err) => { - println!( - "Unable to connect to server at {}, please make sure it was started with 'spraycc server': {}", - callme.addr, err - ); + Err(_) => { status = Some(-1); } } @@ -122,6 +119,29 @@ pub async fn run(args: Vec) -> Result> Ok(status.expect("No task status as set")) } +async fn connect_w_retry(callme: &ipc::CallMe) -> Result> { + let mut count = 0; + loop { + match TcpStream::connect(callme.addr).await { + Ok(stream) => return Ok(stream), + Err(err) => { + if count < 5 { + count += 1; + println!("Retrying connection to {} ({}): {}", callme.addr, count, &err); + let duration = Duration::from_millis(rand::random::() % 2048 + 512); // 2-2.5s backoff period + tokio::time::sleep(duration).await; + } else { + println!( + "Unable to connect to server at {}, please make sure it was started with 'spraycc server': {}", + callme.addr, &err + ); + return Err(Box::new(err)); + } + } + } + } +} + /// Handles messages coming from the client via the server. These will be stderr, stdout and generated files from /// the execution of the task, and then finally the task status. /// diff --git a/src/main.rs b/src/main.rs index ae89b14..18a13b4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,7 +30,7 @@ pub mod task; #[tokio::main] async fn main() { let matches = App::new("SprayCC") - .version("0.1.0") + .version("0.8.0") .about("SprayCC - distributed compiler wrapper") .setting(AppSettings::SubcommandRequiredElseHelp) .subcommand( @@ -114,18 +114,22 @@ async fn main() { panic!("Error: access code must be a numeric value, got: {}", code); } } else if let Some(run) = matches.subcommand_matches("run") { - let args: Vec = run.values_of("compiler_options").unwrap().map(String::from).collect(); - match client::run(args).await { - Ok(ec) if ec > 0 => { - // Remote task failed so exit with the same exit code - std::process::exit(ec); + if let Some(compiler_opts) = run.values_of("compiler_options") { + let args: Vec = compiler_opts.map(String::from).collect(); + match client::run(args).await { + Ok(ec) if ec > 0 => { + // Remote task failed so exit with the same exit code + std::process::exit(ec); + } + Ok(ec) if ec < 0 => { + // Remote task failed with signal + std::process::exit(-1); + } + Ok(_) => Ok(()), + Err(e) => Err(e), } - Ok(ec) if ec < 0 => { - // Remote task failed with signal - std::process::exit(-1); - } - Ok(_) => Ok(()), - Err(e) => Err(e), + } else { + panic!("'spraycc run' must be followed by the command line to be run"); } } else if let Some(fakecc) = matches.subcommand_matches("fakecc") { for output in fakecc.values_of("output_file").unwrap() {