From 44aac5b0a08c036cb0f5c4f2715b54f580ae53e0 Mon Sep 17 00:00:00 2001 From: Hamza Khalid <36852564+hmzakhalid@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:42:11 +0500 Subject: [PATCH] Automated CipherNode deployment (#195) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Dockerfiles * Expose Udp port * Update Entry point for CN * Dev compose * delete old compose * push image to ecr * Update ECS Deployment Workflow * Update ECS Deployment Workflow * Update ECS Deployment Workflow * Update node version to 22 * Update Entrypoint * Use Docker Swarm to manage Ciphernodes * Mount Secrets * Switch to ghcr.io * Update image tag * Deploy to EC2 * Testing EC2 Deployment * Optional mdns * Update Configs * Update config * Update Deployment workflow * Bypass overlay and use host network * Env substitution (#203) * Env substitution works * Move to dev-dependencies * Formatting * Ensure that there are no quotes or anything around substitution * use non normal env vars to avoid future issues * merge main and update workflow deployment * Update Dockerfile and add deployments artifacts --------- Co-authored-by: гλ --- .github/workflows/ec2-deployment.yml | 86 +++++++++++++++ docker-compose.dev.yml | 16 +++ docker-compose.yml | 102 ++++++++++++++++++ packages/ciphernode/Cargo.lock | 55 ++++++---- packages/ciphernode/Cargo.toml | 2 + packages/ciphernode/Dockerfile | 48 +++++++++ packages/ciphernode/ciphernode-entrypoint.sh | 44 ++++++++ packages/ciphernode/config/Cargo.toml | 4 + packages/ciphernode/config/src/app_config.rs | 78 +++++++++++++- packages/ciphernode/config/src/lib.rs | 1 + packages/ciphernode/config/src/yaml.rs | 51 +++++++++ .../ciphernode/enclave_node/src/aggregator.rs | 2 + .../ciphernode/enclave_node/src/ciphernode.rs | 2 + .../evm/src/ciphernode_registry_sol.rs | 2 +- packages/ciphernode/net/src/bin/p2p_test.rs | 7 +- .../ciphernode/net/src/network_manager.rs | 13 ++- packages/ciphernode/net/src/network_peer.rs | 98 +++++++++-------- .../ciphernode/net/tests/docker-compose.yaml | 15 +-- packages/ciphernode/net/tests/entrypoint.sh | 7 -- tests/basic_integration/lib/ag/config.yaml | 2 + tests/basic_integration/lib/cn1/config.yaml | 2 + tests/basic_integration/lib/cn2/config.yaml | 2 + tests/basic_integration/lib/cn3/config.yaml | 2 + tests/basic_integration/lib/cn4/config.yaml | 2 + 24 files changed, 549 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/ec2-deployment.yml create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.yml create mode 100644 packages/ciphernode/Dockerfile create mode 100644 packages/ciphernode/ciphernode-entrypoint.sh create mode 100644 packages/ciphernode/config/src/yaml.rs diff --git a/.github/workflows/ec2-deployment.yml b/.github/workflows/ec2-deployment.yml new file mode 100644 index 00000000..793b5c01 --- /dev/null +++ b/.github/workflows/ec2-deployment.yml @@ -0,0 +1,86 @@ +name: Build and Deploy Ciphernode + +on: + push: + branches: + - release + - main + paths: + - 'packages/ciphernode/**' + - 'packages/evm/contracts/**' + pull_request: + branches: + - release + - main + paths: + - 'packages/ciphernode/**' + - 'packages/evm/contracts/**' + +env: + DOCKERFILE_PATH: packages/ciphernode/Dockerfile + IMAGE_NAME: ghcr.io/gnosisguild/ciphernode + +permissions: + contents: read + packages: write + +jobs: + build: + name: Build Image + runs-on: ubuntu-latest + outputs: + image_tag: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v3 + + - name: Generate version tag + id: version + run: echo "version=$(date +'%Y%m%d')-${GITHUB_SHA::8}" >> $GITHUB_OUTPUT + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build image + env: + IMAGE_TAG: ${{ steps.version.outputs.version }} + run: | + docker build -t $IMAGE_NAME:${{ steps.version.outputs.version }} -f $DOCKERFILE_PATH . + docker push $IMAGE_NAME:$IMAGE_TAG + + - name: Push to GHCR + if: github.ref == 'refs/heads/release' + env: + IMAGE_TAG: ${{ steps.version.outputs.version }} + run: | + docker tag $IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:latest + docker push $IMAGE_NAME:latest + + deploy: + name: Deploy to Production + needs: build + runs-on: ubuntu-latest + environment: + name: production + if: github.ref == 'refs/heads/release' + + steps: + - name: Deploy to EC2 + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_KEY }} + script: | + IMAGE_TAG="${{ needs.build.outputs.image_tag }}" + echo "Deploying version: $IMAGE_TAG" + docker pull $IMAGE_NAME:$IMAGE_TAG + + cd /home/ec2-user/enclave + git pull + + docker stack deploy -c docker-compose.yml ciphernode-stack + diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..c43f4d29 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,16 @@ +services: + cn1: + networks: + - cn-network + cn2: + networks: + - cn-network + cn3: + networks: + - cn-network + aggregator: + networks: + - cn-network + +networks: + cn-network: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..5573cce5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,102 @@ +services: + cn1: + image: ghcr.io/gnosisguild/ciphernode:latest + volumes: + - ./configs/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - cn1-data:/home/ciphernode/.local/share/enclave + secrets: + - secrets.json + environment: + RUST_LOG: "info" + AGGREGATOR: "false" + ports: + - target: 9091 + published: 9091 + protocol: udp + mode: host + deploy: + replicas: 1 + networks: + - global-network + + + cn2: + image: ghcr.io/gnosisguild/ciphernode:latest + depends_on: + - cn1 + volumes: + - ./configs/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - cn2-data:/home/ciphernode/.local/share/enclave + secrets: + - secrets.json + environment: + RUST_LOG: "info" + AGGREGATOR: "false" + ports: + - target: 9092 + published: 9092 + protocol: udp + mode: host + deploy: + replicas: 1 + networks: + - global-network + + cn3: + image: ghcr.io/gnosisguild/ciphernode:latest + depends_on: + - cn1 + volumes: + - ./configs/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - cn3-data:/home/ciphernode/.local/share/enclave + secrets: + - secrets.json + environment: + RUST_LOG: "info" + AGGREGATOR: "false" + ports: + - target: 9093 + published: 9093 + protocol: udp + mode: host + deploy: + replicas: 1 + networks: + - global-network + + + aggregator: + image: ghcr.io/gnosisguild/ciphernode:latest + depends_on: + - cn1 + volumes: + - ./configs/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - agg-data:/home/ciphernode/.local/share/enclave + secrets: + - secrets.json + environment: + RUST_LOG: "info" + AGGREGATOR: "true" + ports: + - target: 9094 + published: 9094 + protocol: udp + mode: host + deploy: + replicas: 1 + networks: + - global-network + +secrets: + secrets.json: + file: ./configs/secrets.json + +volumes: + cn1-data: + cn2-data: + cn3-data: + agg-data: + +networks: + global-network: + driver: overlay diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index fde6c003..0684a617 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -1104,7 +1104,7 @@ checksum = "d7ebdfa2ebdab6b1760375fa7d6f382b9f486eac35fc994625a00e89280bdbb7" dependencies = [ "async-task", "concurrent-queue", - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-lite 2.3.0", "slab", ] @@ -1169,7 +1169,7 @@ dependencies = [ "futures-lite 2.3.0", "parking", "polling 3.7.2", - "rustix 0.38.34", + "rustix 0.38.42", "slab", "tracing", "windows-sys 0.52.0", @@ -1219,7 +1219,7 @@ dependencies = [ "cfg-if", "event-listener 3.1.0", "futures-lite 1.13.0", - "rustix 0.38.34", + "rustix 0.38.42", "windows-sys 0.48.0", ] @@ -1235,7 +1235,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 0.38.34", + "rustix 0.38.42", "signal-hook-registry", "slab", "windows-sys 0.59.0", @@ -1694,6 +1694,8 @@ dependencies = [ "dirs", "figment", "serde", + "shellexpand", + "tempfile", "url", ] @@ -2254,12 +2256,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2341,9 +2343,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fastrlp" @@ -2618,7 +2620,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-core", "futures-io", "parking", @@ -3495,9 +3497,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libm" @@ -4672,7 +4674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391" dependencies = [ "atomic-waker", - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-io", ] @@ -4718,7 +4720,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.4.0", "pin-project-lite", - "rustix 0.38.34", + "rustix 0.38.42", "tracing", "windows-sys 0.52.0", ] @@ -5388,15 +5390,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys 0.4.14", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5692,6 +5694,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shellexpand" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -5970,15 +5981,15 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.11.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", - "fastrand 2.1.0", + "fastrand 2.3.0", "once_cell", - "rustix 0.38.34", - "windows-sys 0.52.0", + "rustix 0.38.42", + "windows-sys 0.59.0", ] [[package]] diff --git a/packages/ciphernode/Cargo.toml b/packages/ciphernode/Cargo.toml index 382ac6b7..8ce8f55a 100644 --- a/packages/ciphernode/Cargo.toml +++ b/packages/ciphernode/Cargo.toml @@ -39,6 +39,7 @@ clap = { version = "4.5.17", features = ["derive"] } cipher = { path = "./cipher" } dirs = "5.0.1" data = { path = "./data" } +shellexpand = "3.1.0" figment = { version = "0.10.19", features = ["yaml", "test"] } fhe_rs = { package = "fhe", git = "https://github.com/gnosisguild/fhe.rs", version = "0.1.0-beta.7" } fhe-traits = { git = "https://github.com/gnosisguild/fhe.rs", version = "0.1.0-beta.7" } @@ -54,6 +55,7 @@ serde = { version = "1.0.208", features = ["derive"] } serde_json = { version = "1.0.133" } sled = "0.34.7" sha2 = "0.10.8" +tempfile = "3.14.0" tokio = { version = "1.38", features = ["full"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/packages/ciphernode/Dockerfile b/packages/ciphernode/Dockerfile new file mode 100644 index 00000000..3f559d95 --- /dev/null +++ b/packages/ciphernode/Dockerfile @@ -0,0 +1,48 @@ +FROM node:22 AS evm-builder + +WORKDIR /build/packages/evm +COPY ./packages/evm ./ +RUN yarn install && yarn compile + +# Build stage +FROM rust:1.81 AS ciphernode-builder + +# Create build directory +WORKDIR /build/packages/ciphernode +COPY ./packages/ciphernode ./ +COPY --from=evm-builder /build/packages/evm/artifacts ../evm/artifacts +COPY --from=evm-builder /build/packages/evm/deployments ../evm/deployments +RUN cargo build --release + +# Runtime stage +FROM debian:stable-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends iptables ca-certificates jq && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Create non-root user +RUN useradd -m -u 1000 -s /bin/bash ciphernode + +# Create necessary directories with proper permissions +RUN mkdir -p /home/ciphernode/.config/enclave \ + /home/ciphernode/.local/share/enclave \ + /run/secrets && \ + chown -R ciphernode:ciphernode /home/ciphernode /run/secrets + +# Switch to non-root user +USER ciphernode +WORKDIR /home/ciphernode + +# Copy binary from builder +COPY --from=ciphernode-builder --chown=ciphernode:ciphernode /build/packages/ciphernode/target/release/enclave /usr/local/bin/ +COPY --from=ciphernode-builder --chmod=755 --chown=ciphernode:ciphernode /build/packages/ciphernode/ciphernode-entrypoint.sh /usr/local/bin/ + +# Environment variables for configuration +ENV CONFIG_DIR=/home/ciphernode/.config/enclave +ENV DATA_DIR=/home/ciphernode/.local/share/enclave +ENV RUST_LOG=info + +# Add entrypoint script + +ENTRYPOINT ["ciphernode-entrypoint.sh"] \ No newline at end of file diff --git a/packages/ciphernode/ciphernode-entrypoint.sh b/packages/ciphernode/ciphernode-entrypoint.sh new file mode 100644 index 00000000..3fa42fdd --- /dev/null +++ b/packages/ciphernode/ciphernode-entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -e + +# Paths to config and secrets +CONFIG_FILE="$CONFIG_DIR/config.yaml" +SECRETS_FILE="/run/secrets/secrets.json" +AGGREGATOR="$AGGREGATOR" + +# Ensure required files exist +if [ ! -f "$CONFIG_FILE" ]; then + echo "Error: Config file $CONFIG_FILE not found!" + exit 1 +fi + +if [ ! -f "$SECRETS_FILE" ]; then + echo "Error: Secrets file $SECRETS_FILE not found!" + exit 1 +fi + +# Read secrets from the JSON file +PRIVATE_KEY=$(jq -r '.private_key' "$SECRETS_FILE") +PASSWORD=$(jq -r '.password' "$SECRETS_FILE") + +if [ -z "$PRIVATE_KEY" ] || [ -z "$PASSWORD" ]; then + echo "Error: Missing 'private_key' or 'password' in secrets file!" + exit 1 +fi + +# Set password and private key +echo "Setting password" +enclave password create --config "$CONFIG_FILE" --password "$PASSWORD" + +if [ "$AGGREGATOR" = "true" ]; then + echo "Setting private key" + enclave wallet set --config "$CONFIG_FILE" --private-key "$PRIVATE_KEY" + + echo "Starting aggregator" + exec enclave aggregator start --config "$CONFIG_FILE" +else + echo "Starting Ciphernode" + exec enclave start --config "$CONFIG_FILE" +fi + + diff --git a/packages/ciphernode/config/Cargo.toml b/packages/ciphernode/config/Cargo.toml index a336f7fe..e58c695f 100644 --- a/packages/ciphernode/config/Cargo.toml +++ b/packages/ciphernode/config/Cargo.toml @@ -9,5 +9,9 @@ anyhow = { workspace = true } serde = { workspace = true } figment = { workspace = true } alloy = { workspace = true } +shellexpand = { workspace = true } url = { workspace = true } +[dev-dependencies] +tempfile = { workspace = true } + diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index 133c23be..cc6eac80 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -14,6 +14,8 @@ use std::{ }; use url::Url; +use crate::yaml::load_yaml_with_env; + #[derive(Debug, Deserialize, Serialize, PartialEq)] #[serde(untagged)] pub enum Contract { @@ -157,6 +159,10 @@ pub struct AppConfig { address: Option
, /// A list of libp2p multiaddrs to dial to as peers when joining the network peers: Vec, + /// The port to use for the quic listener + quic_port: u16, + /// Whether to enable mDNS discovery + enable_mdns: bool, } impl Default for AppConfig { @@ -172,6 +178,8 @@ impl Default for AppConfig { peers: vec![], // NOTE: This should remain empty and we should look at config // generation via ipns fetch for the latest nodes address: None, + quic_port: 9091, + enable_mdns: false, } } } @@ -247,6 +255,14 @@ impl AppConfig { pub fn peers(&self) -> Vec { self.peers.clone() } + + pub fn quic_port(&self) -> u16 { + self.quic_port + } + + pub fn enable_mdns(&self) -> bool { + self.enable_mdns + } } /// Load the config at the config_file or the default location if not provided @@ -256,8 +272,10 @@ pub fn load_config(config_file: Option<&str>) -> Result { defaults.config_file = file.into(); } + let with_envs = load_yaml_with_env(&defaults.config_file())?; + let config = Figment::from(Serialized::defaults(&defaults)) - .merge(Yaml::file(defaults.config_file())) + .merge(Yaml::string(&with_envs)) .extract()?; Ok(config) @@ -501,4 +519,62 @@ chains: Ok(()) }); } + + #[test] + fn test_config_env_vars() { + Jail::expect_with(|jail| { + let home = format!("{}", jail.directory().to_string_lossy()); + jail.set_env("HOME", &home); + jail.set_env("XDG_CONFIG_HOME", &format!("{}/.config", home)); + jail.set_env("TEST_RPC_URL_PORT", "8545"); + jail.set_env("TEST_USERNAME", "envUser"); + jail.set_env("TEST_PASSWORD", "envPassword"); + jail.set_env( + "TEST_CONTRACT_ADDRESS", + "0x1234567890123456789012345678901234567890", + ); + + let filename = format!("{}/.config/enclave/config.yaml", home); + let filedir = format!("{}/.config/enclave", home); + jail.create_dir(filedir)?; + jail.create_file( + filename, + r#" +chains: + - name: "hardhat" + rpc_url: "ws://test-endpoint:${TEST_RPC_URL_PORT}" + rpc_auth: + type: "Basic" + credentials: + username: "${TEST_USERNAME}" + password: "${TEST_PASSWORD}" + contracts: + enclave: "${TEST_CONTRACT_ADDRESS}" + ciphernode_registry: + address: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" + deploy_block: 1764352873645 + filter_registry: "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" +"#, + )?; + + let config: AppConfig = load_config(None).map_err(|err| err.to_string())?; + let chain = config.chains().first().unwrap(); + + // Test that environment variables are properly substituted + assert_eq!(chain.rpc_url, "ws://test-endpoint:8545"); + assert_eq!( + chain.rpc_auth, + RpcAuth::Basic { + username: "envUser".to_string(), + password: "envPassword".to_string(), + } + ); + assert_eq!( + chain.contracts.enclave.address(), + "0x1234567890123456789012345678901234567890" + ); + + Ok(()) + }); + } } diff --git a/packages/ciphernode/config/src/lib.rs b/packages/ciphernode/config/src/lib.rs index ba182a1b..83e06ce9 100644 --- a/packages/ciphernode/config/src/lib.rs +++ b/packages/ciphernode/config/src/lib.rs @@ -1,2 +1,3 @@ mod app_config; +mod yaml; pub use app_config::*; diff --git a/packages/ciphernode/config/src/yaml.rs b/packages/ciphernode/config/src/yaml.rs new file mode 100644 index 00000000..92da78e1 --- /dev/null +++ b/packages/ciphernode/config/src/yaml.rs @@ -0,0 +1,51 @@ +use anyhow::Result; +use std::{fs, path::PathBuf}; + +pub fn load_yaml_with_env(file_path: &PathBuf) -> Result { + // Read the file content to string + let content = match fs::read_to_string(file_path) { + Ok(val) => val, + Err(_) => "".to_string(), + }; + + // Collect environment variables and perform substitution + Ok(shellexpand::env(&content)?.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + use std::fs::File; + use std::io::Write; + use tempfile::tempdir; + + #[test] + fn test_yaml_env_substitution() -> Result<()> { + // Create a temporary directory and file + let dir = tempdir()?; + let file_path = dir.path().join("test.yaml"); + let mut file = File::create(&file_path)?; + + // Write test YAML content + writeln!( + file, + "database:\n url: $MY_DATABASE_URL\n password: ${{MY_DB_PASSWORD}}" + )?; + + // Set environment variables + env::set_var("MY_DATABASE_URL", "postgres://localhost:5432"); + env::set_var("MY_DB_PASSWORD", "secret123"); + + // Test the function + let processed = load_yaml_with_env(&file_path)?; + + env::remove_var("MY_DATABASE_URL"); + env::remove_var("MY_DB_PASSWORD"); + + assert!(processed.contains("postgres://localhost:5432")); + assert!(processed.contains("secret123")); + + Ok(()) + } +} diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index b5dfba4e..b215ce8f 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -81,6 +81,8 @@ pub async fn setup_aggregator( bus.clone(), config.peers(), &cipher, + config.quic_port(), + config.enable_mdns(), repositories.libp2pid(), ) .await?; diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 5b1e60d9..5205d9be 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -70,6 +70,8 @@ pub async fn setup_ciphernode( bus.clone(), config.peers(), &cipher, + config.quic_port(), + config.enable_mdns(), repositories.libp2pid(), ) .await?; diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index a616411b..d4846a50 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -3,7 +3,7 @@ use crate::{ helpers::{ReadonlyProvider, WithChainId}, EvmEventReader, }; -use actix::{Actor, Addr}; +use actix::Addr; use alloy::{ primitives::{LogData, B256}, sol, diff --git a/packages/ciphernode/net/src/bin/p2p_test.rs b/packages/ciphernode/net/src/bin/p2p_test.rs index 15435f2a..e44f43b8 100644 --- a/packages/ciphernode/net/src/bin/p2p_test.rs +++ b/packages/ciphernode/net/src/bin/p2p_test.rs @@ -30,10 +30,15 @@ async fn main() -> Result<()> { .ok() .and_then(|p| p.parse::().ok()); + let enable_mdns = env::var("ENABLE_MDNS") + .unwrap_or("false".to_string()) + .parse::() + .unwrap(); + let peers: Vec = dial_to.iter().cloned().collect(); let id = libp2p::identity::Keypair::generate_ed25519(); - let mut peer = NetworkPeer::new(&id, peers, udp_port, "test-topic")?; + let mut peer = NetworkPeer::new(&id, peers, udp_port, "test-topic", enable_mdns)?; // Extract input and outputs let tx = peer.tx(); diff --git a/packages/ciphernode/net/src/network_manager.rs b/packages/ciphernode/net/src/network_manager.rs index 2c1b3f02..8969c908 100644 --- a/packages/ciphernode/net/src/network_manager.rs +++ b/packages/ciphernode/net/src/network_manager.rs @@ -8,11 +8,10 @@ use cipher::Cipher; use data::Repository; use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; use libp2p::identity::ed25519; +use std::collections::HashSet; use std::sync::Arc; -use std::{collections::HashSet, error::Error}; use tokio::sync::mpsc::{Receiver, Sender}; use tracing::{error, info, instrument, trace}; -use zeroize::Zeroize; /// NetworkManager Actor converts between EventBus events and Libp2p events forwarding them to a /// NetworkPeer for propagation over the p2p network @@ -72,6 +71,8 @@ impl NetworkManager { bus: Addr, peers: Vec, cipher: &Arc, + quic_port: u16, + enable_mdns: bool, repository: Repository>, ) -> Result<(Addr, tokio::task::JoinHandle>, String)> { info!("Reading from repository"); @@ -93,7 +94,13 @@ impl NetworkManager { let ed25519_keypair = ed25519::Keypair::try_from_bytes(&mut bytes)?; let keypair: libp2p::identity::Keypair = ed25519_keypair.try_into()?; - let mut peer = NetworkPeer::new(&keypair, peers, None, "tmp-enclave-gossip-topic")?; + let mut peer = NetworkPeer::new( + &keypair, + peers, + Some(quic_port), + "tmp-enclave-gossip-topic", + enable_mdns, + )?; let rx = peer.rx().ok_or(anyhow!("Peer rx already taken"))?; let p2p_addr = NetworkManager::setup(bus, peer.tx(), rx); let handle = tokio::spawn(async move { Ok(peer.start().await?) }); diff --git a/packages/ciphernode/net/src/network_peer.rs b/packages/ciphernode/net/src/network_peer.rs index 8e1d80d6..17151076 100644 --- a/packages/ciphernode/net/src/network_peer.rs +++ b/packages/ciphernode/net/src/network_peer.rs @@ -7,14 +7,14 @@ use libp2p::{ identity::Keypair, kad::{store::MemoryStore, Behaviour as KademliaBehaviour}, mdns, - swarm::{NetworkBehaviour, SwarmEvent}, + swarm::{behaviour::toggle::Toggle, NetworkBehaviour, SwarmEvent}, Multiaddr, Swarm, }; use std::hash::{Hash, Hasher}; use std::{hash::DefaultHasher, io::Error, time::Duration}; use tokio::{ select, - sync::mpsc::{self, channel, Receiver, Sender}, + sync::mpsc::{channel, Receiver, Sender}, }; use tracing::{debug, error, info, trace, warn}; @@ -23,7 +23,7 @@ pub struct NodeBehaviour { gossipsub: gossipsub::Behaviour, kademlia: KademliaBehaviour, connection_limits: connection_limits::Behaviour, - mdns: mdns::tokio::Behaviour, + mdns: Toggle, identify: IdentifyBehaviour, } @@ -44,6 +44,7 @@ impl NetworkPeer { peers: Vec, udp_port: Option, topic: &str, + enable_mdns: bool, ) -> Result { let (to_bus_tx, from_net_rx) = channel(100); // TODO : tune this param let (to_net_tx, from_bus_rx) = channel(100); // TODO : tune this param @@ -51,7 +52,7 @@ impl NetworkPeer { let swarm = libp2p::SwarmBuilder::with_existing_identity(id.clone()) .with_tokio() .with_quic() - .with_behaviour(create_mdns_kad_behaviour())? + .with_behaviour(|key| create_mdns_kad_behaviour(enable_mdns, key))? .with_swarm_config(|c| c.with_idle_connection_timeout(Duration::from_secs(60))) .build(); @@ -115,50 +116,53 @@ impl NetworkPeer { } } -fn create_mdns_kad_behaviour() -> impl FnOnce( - &Keypair, -) -> std::result::Result< - NodeBehaviour, - Box, -> { - |key| { - let connection_limits = connection_limits::Behaviour::new(ConnectionLimits::default()); - let identify_config = IdentifyBehaviour::new( - identify::Config::new("/kad/0.1.0".into(), key.public()) - .with_interval(Duration::from_secs(60)), // do this so we can get timeouts for dropped WebRTC connections - ); - let message_id_fn = |message: &gossipsub::Message| { - let mut s = DefaultHasher::new(); - message.data.hash(&mut s); - gossipsub::MessageId::from(s.finish().to_string()) - }; +fn create_mdns_kad_behaviour( + enable_mdns: bool, + key: &Keypair, +) -> std::result::Result> { + let connection_limits = connection_limits::Behaviour::new(ConnectionLimits::default()); + let identify_config = IdentifyBehaviour::new( + identify::Config::new("/kad/0.1.0".into(), key.public()) + .with_interval(Duration::from_secs(60)), + ); + + let message_id_fn = |message: &gossipsub::Message| { + let mut s = DefaultHasher::new(); + message.data.hash(&mut s); + gossipsub::MessageId::from(s.finish().to_string()) + }; - // TODO: Allow for config inputs to new() - let gossipsub_config = gossipsub::ConfigBuilder::default() - .heartbeat_interval(Duration::from_secs(10)) - .validation_mode(gossipsub::ValidationMode::Strict) - .message_id_fn(message_id_fn) - .build() - .map_err(|msg| Error::new(std::io::ErrorKind::Other, msg))?; - - let gossipsub = gossipsub::Behaviour::new( - gossipsub::MessageAuthenticity::Signed(key.clone()), - gossipsub_config, - )?; - - let mdns = mdns::tokio::Behaviour::new(mdns::Config::default(), key.public().to_peer_id())?; - - Ok(NodeBehaviour { - gossipsub, - kademlia: KademliaBehaviour::new( - key.public().to_peer_id(), - MemoryStore::new(key.public().to_peer_id()), - ), - mdns, - connection_limits, - identify: identify_config, - }) - } + let gossipsub_config = gossipsub::ConfigBuilder::default() + .heartbeat_interval(Duration::from_secs(10)) + .validation_mode(gossipsub::ValidationMode::Strict) + .message_id_fn(message_id_fn) + .build() + .map_err(|msg| Error::new(std::io::ErrorKind::Other, msg))?; + + let gossipsub = gossipsub::Behaviour::new( + gossipsub::MessageAuthenticity::Signed(key.clone()), + gossipsub_config, + )?; + + let mdns = if enable_mdns { + Toggle::from(Some(mdns::tokio::Behaviour::new( + mdns::Config::default(), + key.public().to_peer_id(), + )?)) + } else { + Toggle::from(None) + }; + + Ok(NodeBehaviour { + gossipsub, + kademlia: KademliaBehaviour::new( + key.public().to_peer_id(), + MemoryStore::new(key.public().to_peer_id()), + ), + mdns, + connection_limits, + identify: identify_config, + }) } async fn process_swarm_event( diff --git a/packages/ciphernode/net/tests/docker-compose.yaml b/packages/ciphernode/net/tests/docker-compose.yaml index f55d3698..d54a3046 100644 --- a/packages/ciphernode/net/tests/docker-compose.yaml +++ b/packages/ciphernode/net/tests/docker-compose.yaml @@ -11,11 +11,8 @@ services: environment: QUIC_PORT: 9091 DIAL_TO: "/ip4/172.16.238.12/udp/9091/quic-v1" - BLOCK_MDNS: "${BLOCK_MDNS:-false}" + ENABLE_MDNS: "${ENABLE_MDNS:-true}" entrypoint: ["/app/entrypoint.sh"] - cap_add: - - NET_ADMIN - - NET_RAW bob: image: p2p-test-image @@ -26,11 +23,8 @@ services: environment: QUIC_PORT: 9091 DIAL_TO: "/ip4/172.16.238.12/udp/9091/quic-v1" - BLOCK_MDNS: "${BLOCK_MDNS:-false}" + ENABLE_MDNS: "${ENABLE_MDNS:-true}" entrypoint: ["/app/entrypoint.sh"] - cap_add: - - NET_ADMIN - - NET_RAW charlie: image: p2p-test-image @@ -40,11 +34,8 @@ services: command: ["/app/p2p_test", "charlie"] environment: QUIC_PORT: 9091 - BLOCK_MDNS: "${BLOCK_MDNS:-false}" + ENABLE_MDNS: "${ENABLE_MDNS:-true}" entrypoint: ["/app/entrypoint.sh"] - cap_add: - - NET_ADMIN - - NET_RAW networks: app_net: diff --git a/packages/ciphernode/net/tests/entrypoint.sh b/packages/ciphernode/net/tests/entrypoint.sh index dfc54f53..a6453106 100755 --- a/packages/ciphernode/net/tests/entrypoint.sh +++ b/packages/ciphernode/net/tests/entrypoint.sh @@ -1,11 +1,4 @@ #!/bin/bash set -e -if [ "${BLOCK_MDNS:-false}" = "true" ]; then - iptables -A INPUT -p udp --dport 5353 -j DROP - iptables -A OUTPUT -p udp --dport 5353 -j DROP - iptables -L | grep DROP -fi - -# Execute the original command exec "$@" diff --git a/tests/basic_integration/lib/ag/config.yaml b/tests/basic_integration/lib/ag/config.yaml index 68b919f1..21e84dea 100644 --- a/tests/basic_integration/lib/ag/config.yaml +++ b/tests/basic_integration/lib/ag/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199" +quic_port: 9095 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545" diff --git a/tests/basic_integration/lib/cn1/config.yaml b/tests/basic_integration/lib/cn1/config.yaml index 508112cd..3cd8d55c 100644 --- a/tests/basic_integration/lib/cn1/config.yaml +++ b/tests/basic_integration/lib/cn1/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0x2546BcD3c84621e976D8185a91A922aE77ECEc30" +quic_port: 9091 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545" diff --git a/tests/basic_integration/lib/cn2/config.yaml b/tests/basic_integration/lib/cn2/config.yaml index 5cbce324..f3e8e48e 100644 --- a/tests/basic_integration/lib/cn2/config.yaml +++ b/tests/basic_integration/lib/cn2/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" +quic_port: 9092 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545" diff --git a/tests/basic_integration/lib/cn3/config.yaml b/tests/basic_integration/lib/cn3/config.yaml index 258b8488..9c04ab47 100644 --- a/tests/basic_integration/lib/cn3/config.yaml +++ b/tests/basic_integration/lib/cn3/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0xdD2FD4581271e230360230F9337D5c0430Bf44C0" +quic_port: 9093 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545" diff --git a/tests/basic_integration/lib/cn4/config.yaml b/tests/basic_integration/lib/cn4/config.yaml index 55bdb526..ff73bf71 100644 --- a/tests/basic_integration/lib/cn4/config.yaml +++ b/tests/basic_integration/lib/cn4/config.yaml @@ -1,6 +1,8 @@ config_dir: . data_dir: . address: "0x8626f6940E2eb28930eFb4CeF49B2d1F2C9C1199" +quic_port: 9094 +enable_mdns: true chains: - name: "hardhat" rpc_url: "ws://localhost:8545"