From 97824d42745aa2e9dcf8325a824495bc1787ba68 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 15:45:28 +1100 Subject: [PATCH 01/36] Dont need dev compose --- docker-compose.dev.yml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 docker-compose.dev.yml diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml deleted file mode 100644 index c43f4d29..00000000 --- a/docker-compose.dev.yml +++ /dev/null @@ -1,16 +0,0 @@ -services: - cn1: - networks: - - cn-network - cn2: - networks: - - cn-network - cn3: - networks: - - cn-network - aggregator: - networks: - - cn-network - -networks: - cn-network: From 9ac30323cbed5b36f6aada4f51da57a6f9c28d69 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 18:36:12 +1100 Subject: [PATCH 02/36] Create secrets script --- .deploy/.gitignore | 2 + .deploy/copy-secrets.sh | 54 ++++++++++++++++ .../docker-compose.yml | 64 ++++++++++++++----- .deploy/example.secrets.json | 4 ++ .deploy/swarm_deployment.md | 31 +++++++++ 5 files changed, 140 insertions(+), 15 deletions(-) create mode 100644 .deploy/.gitignore create mode 100755 .deploy/copy-secrets.sh rename docker-compose.yml => .deploy/docker-compose.yml (55%) create mode 100644 .deploy/example.secrets.json create mode 100644 .deploy/swarm_deployment.md diff --git a/.deploy/.gitignore b/.deploy/.gitignore new file mode 100644 index 00000000..c60bb1cf --- /dev/null +++ b/.deploy/.gitignore @@ -0,0 +1,2 @@ +*.secrets.json +!example.secrets.json diff --git a/.deploy/copy-secrets.sh b/.deploy/copy-secrets.sh new file mode 100755 index 00000000..dadbb38b --- /dev/null +++ b/.deploy/copy-secrets.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Set working directory to script location +cd "$(dirname "$0")" || exit 1 + +# Source file path (in current directory) +SOURCE="example.secrets.json" + +# Color codes +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# List of target files +TARGETS=("cn1" "cn2" "cn3" "agg") + +# Check if source file exists +if [ ! -f "$SOURCE" ]; then + echo "Error: Source file $SOURCE not found!" + exit 1 +fi + +# Copy file to each target, skipping if exists +for target in "${TARGETS[@]}"; do + if [ -f "${target}.secrets.json" ]; then + echo "Skipping ${target}.secrets.json - file already exists" + else + cp "$SOURCE" "${target}.secrets.json" + echo "Created ${target}.secrets.json" + fi +done + +echo "Copy operation completed!" + +# Check for unchanged files +echo -e "\nChecking for unchanged secret files..." +UNCHANGED_FILES=() + +for target in "${TARGETS[@]}"; do + if [ -f "${target}.secrets.json" ]; then + if cmp -s "$SOURCE" "${target}.secrets.json"; then + UNCHANGED_FILES+=("${target}.secrets.json") + fi + fi +done + +# Display warning if unchanged files found +if [ ${#UNCHANGED_FILES[@]} -gt 0 ]; then + echo -e "${RED}WARNING: The following files are identical to example.secrets.json:${NC}" + for file in "${UNCHANGED_FILES[@]}"; do + echo -e "${YELLOW}==> ${NC}${file}${YELLOW} <==${NC}" + done + echo -e "${RED}These files should be modified before use in production!${NC}" +fi diff --git a/docker-compose.yml b/.deploy/docker-compose.yml similarity index 55% rename from docker-compose.yml rename to .deploy/docker-compose.yml index 5573cce5..9de87760 100644 --- a/docker-compose.yml +++ b/.deploy/docker-compose.yml @@ -2,10 +2,11 @@ services: cn1: image: ghcr.io/gnosisguild/ciphernode:latest volumes: - - ./configs/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - ./.deploy/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/enclave secrets: - - secrets.json + - source: secrets_cn1 + target: secrets.json environment: RUST_LOG: "info" AGGREGATOR: "false" @@ -14,8 +15,14 @@ services: published: 9091 protocol: udp mode: host - deploy: + deploy: replicas: 1 + update_config: + parallelism: 1 + order: stop-first + failure_action: rollback + restart_policy: + condition: any networks: - global-network @@ -25,10 +32,11 @@ services: depends_on: - cn1 volumes: - - ./configs/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - ./.deploy/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn2-data:/home/ciphernode/.local/share/enclave secrets: - - secrets.json + - source: secrets_cn2 + target: secrets.json environment: RUST_LOG: "info" AGGREGATOR: "false" @@ -37,8 +45,14 @@ services: published: 9092 protocol: udp mode: host - deploy: + deploy: replicas: 1 + update_config: + parallelism: 1 + order: stop-first + failure_action: rollback + restart_policy: + condition: any networks: - global-network @@ -47,10 +61,11 @@ services: depends_on: - cn1 volumes: - - ./configs/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - ./.deploy/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn3-data:/home/ciphernode/.local/share/enclave secrets: - - secrets.json + - source: secrets_cn3 + target: secrets.json environment: RUST_LOG: "info" AGGREGATOR: "false" @@ -59,8 +74,14 @@ services: published: 9093 protocol: udp mode: host - deploy: + deploy: replicas: 1 + update_config: + parallelism: 1 + order: stop-first + failure_action: rollback + restart_policy: + condition: any networks: - global-network @@ -70,10 +91,11 @@ services: depends_on: - cn1 volumes: - - ./configs/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - ./.deploy/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - agg-data:/home/ciphernode/.local/share/enclave secrets: - - secrets.json + - source: secrets_agg + target: secrets.json environment: RUST_LOG: "info" AGGREGATOR: "true" @@ -82,15 +104,27 @@ services: published: 9094 protocol: udp mode: host - deploy: + deploy: replicas: 1 + update_config: + parallelism: 1 + order: stop-first + failure_action: rollback + restart_policy: + condition: any networks: - global-network secrets: - secrets.json: - file: ./configs/secrets.json - + secrets_cn1: + file: .deploy/cn1.secrets.json + secrets_cn2: + file: .deploy/cn2.secrets.json + secrets_cn3: + file: .deploy/cn3.secrets.json + secrets_agg: + file: .deploy/agg.secrets.json + volumes: cn1-data: cn2-data: diff --git a/.deploy/example.secrets.json b/.deploy/example.secrets.json new file mode 100644 index 00000000..f01ec15a --- /dev/null +++ b/.deploy/example.secrets.json @@ -0,0 +1,4 @@ +{ + "password": "changeme", + "private_key": "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +} diff --git a/.deploy/swarm_deployment.md b/.deploy/swarm_deployment.md new file mode 100644 index 00000000..e252ab75 --- /dev/null +++ b/.deploy/swarm_deployment.md @@ -0,0 +1,31 @@ + +# Secrets Setup Script + +To deploy with swarm we need to set up the secrets file for our cluster. + +## Run +```bash +./.deploy/copy-secrets.sh +``` + +## What it does +- Copies `example.secrets.json` to create `cn1/2/3` and `agg.secrets.json` files +- Skips existing files +- Warns with yellow arrows (==>) if any files are identical to the example + +## Example output +```bash +Created cn1.secrets.json +Skipping cn2.secrets.json - file already exists + +==> cn1.secrets.json <== # Yellow arrows indicate files that need customization +``` + +Remember to modify any highlighted files before use. + +# Run docker swarm + +``` +docker stack deploy -c .deploy/docker-compose.yml p2p-stack +``` + From 680d9b14cc8fd74366283d162c36df6feee0db66 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 18:49:10 +1100 Subject: [PATCH 03/36] Fix compose file --- .deploy/docker-compose.yml | 38 ++++++++++++++----------------------- .deploy/swarm_deployment.md | 2 +- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/.deploy/docker-compose.yml b/.deploy/docker-compose.yml index 9de87760..84fa21b8 100644 --- a/.deploy/docker-compose.yml +++ b/.deploy/docker-compose.yml @@ -1,6 +1,6 @@ services: cn1: - image: ghcr.io/gnosisguild/ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode:${TAG} volumes: - ./.deploy/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/enclave @@ -11,11 +11,8 @@ services: RUST_LOG: "info" AGGREGATOR: "false" ports: - - target: 9091 - published: 9091 - protocol: udp - mode: host - deploy: + - 9091:9091 + deploy: replicas: 1 update_config: parallelism: 1 @@ -28,7 +25,7 @@ services: cn2: - image: ghcr.io/gnosisguild/ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode:${TAG} depends_on: - cn1 volumes: @@ -41,11 +38,9 @@ services: RUST_LOG: "info" AGGREGATOR: "false" ports: - - target: 9092 - published: 9092 - protocol: udp - mode: host - deploy: + - 9092:9092 + + deploy: replicas: 1 update_config: parallelism: 1 @@ -57,7 +52,7 @@ services: - global-network cn3: - image: ghcr.io/gnosisguild/ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode:${TAG} depends_on: - cn1 volumes: @@ -70,11 +65,8 @@ services: RUST_LOG: "info" AGGREGATOR: "false" ports: - - target: 9093 - published: 9093 - protocol: udp - mode: host - deploy: + - 9093:9093 + deploy: replicas: 1 update_config: parallelism: 1 @@ -87,7 +79,7 @@ services: aggregator: - image: ghcr.io/gnosisguild/ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode:${TAG} depends_on: - cn1 volumes: @@ -100,11 +92,9 @@ services: RUST_LOG: "info" AGGREGATOR: "true" ports: - - target: 9094 - published: 9094 - protocol: udp - mode: host - deploy: + - 9094:9094 + + deploy: replicas: 1 update_config: parallelism: 1 diff --git a/.deploy/swarm_deployment.md b/.deploy/swarm_deployment.md index e252ab75..9dec959d 100644 --- a/.deploy/swarm_deployment.md +++ b/.deploy/swarm_deployment.md @@ -26,6 +26,6 @@ Remember to modify any highlighted files before use. # Run docker swarm ``` -docker stack deploy -c .deploy/docker-compose.yml p2p-stack +docker stack deploy -c .deploy/docker-compose.yml enclave-stack ``` From 963e872bd55cb0bc20ab85813c0277e7cfdfcc5c Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 19:02:39 +1100 Subject: [PATCH 04/36] Fix paths --- .deploy/docker-compose.yml | 16 ++++++++-------- .deploy/swarm_deployment.md | 14 +++++++++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.deploy/docker-compose.yml b/.deploy/docker-compose.yml index 84fa21b8..40f9c368 100644 --- a/.deploy/docker-compose.yml +++ b/.deploy/docker-compose.yml @@ -2,7 +2,7 @@ services: cn1: image: ghcr.io/gnosisguild/ciphernode:${TAG} volumes: - - ./.deploy/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - ./cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/enclave secrets: - source: secrets_cn1 @@ -29,7 +29,7 @@ services: depends_on: - cn1 volumes: - - ./.deploy/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - ./cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn2-data:/home/ciphernode/.local/share/enclave secrets: - source: secrets_cn2 @@ -56,7 +56,7 @@ services: depends_on: - cn1 volumes: - - ./.deploy/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - ./cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn3-data:/home/ciphernode/.local/share/enclave secrets: - source: secrets_cn3 @@ -83,7 +83,7 @@ services: depends_on: - cn1 volumes: - - ./.deploy/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro + - ./agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - agg-data:/home/ciphernode/.local/share/enclave secrets: - source: secrets_agg @@ -107,13 +107,13 @@ services: secrets: secrets_cn1: - file: .deploy/cn1.secrets.json + file: cn1.secrets.json secrets_cn2: - file: .deploy/cn2.secrets.json + file: cn2.secrets.json secrets_cn3: - file: .deploy/cn3.secrets.json + file: cn3.secrets.json secrets_agg: - file: .deploy/agg.secrets.json + file: agg.secrets.json volumes: cn1-data: diff --git a/.deploy/swarm_deployment.md b/.deploy/swarm_deployment.md index 9dec959d..ee084749 100644 --- a/.deploy/swarm_deployment.md +++ b/.deploy/swarm_deployment.md @@ -25,7 +25,19 @@ Remember to modify any highlighted files before use. # Run docker swarm +First we need to initialize swarm. + +``` +docker swarm init +``` + +If you get an error about not being able to choose between IP addresses choose the more private IP address. + +``` +docker swarm init --advertise-addr 10.49.x.x +``` + ``` -docker stack deploy -c .deploy/docker-compose.yml enclave-stack +TAG=latest docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false ``` From 5212911086fb0d8ffc54f2e3a7f9bd41d581b6cc Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 19:51:00 +1100 Subject: [PATCH 05/36] Supply yam files --- .deploy/.env.example | 1 + .deploy/.gitignore | 1 + .deploy/agg.yaml | 10 ++++++++++ .deploy/cn1.yaml | 10 ++++++++++ .deploy/cn2.yaml | 10 ++++++++++ .deploy/cn3.yaml | 10 ++++++++++ .deploy/run.sh | 5 +++++ 7 files changed, 47 insertions(+) create mode 100644 .deploy/.env.example create mode 100644 .deploy/agg.yaml create mode 100644 .deploy/cn1.yaml create mode 100644 .deploy/cn2.yaml create mode 100644 .deploy/cn3.yaml create mode 100755 .deploy/run.sh diff --git a/.deploy/.env.example b/.deploy/.env.example new file mode 100644 index 00000000..266fb5c8 --- /dev/null +++ b/.deploy/.env.example @@ -0,0 +1 @@ +export TAG=latest diff --git a/.deploy/.gitignore b/.deploy/.gitignore index c60bb1cf..0aef23bb 100644 --- a/.deploy/.gitignore +++ b/.deploy/.gitignore @@ -1,2 +1,3 @@ *.secrets.json +.env !example.secrets.json diff --git a/.deploy/agg.yaml b/.deploy/agg.yaml new file mode 100644 index 00000000..e41b6d79 --- /dev/null +++ b/.deploy/agg.yaml @@ -0,0 +1,10 @@ +address: "${AGG_ADDRESS}" +quic_port: ${AGG_QUIC_PORT} +enable_mdns: false +chains: + - name: "sepolia" + rpc_url: "${RPC_URL}" + contracts: + enclave: "${SEPOLIA_ENCLAVE_ADDRESS}" + ciphernode_registry: "${SEPOLIA_CIPHERNODE_REGISTRY_ADDRESS}" + filter_registry: "${SEPOLIA_FILTER_REGISTRY}" diff --git a/.deploy/cn1.yaml b/.deploy/cn1.yaml new file mode 100644 index 00000000..18b730e9 --- /dev/null +++ b/.deploy/cn1.yaml @@ -0,0 +1,10 @@ +address: "${CN1_ADDRESS}" +quic_port: ${CN1_QUIC_PORT} +enable_mdns: false +chains: + - name: "sepolia" + rpc_url: "${RPC_URL}" + contracts: + enclave: "${SEPOLIA_ENCLAVE_ADDRESS}" + ciphernode_registry: "${SEPOLIA_CIPHERNODE_REGISTRY_ADDRESS}" + filter_registry: "${SEPOLIA_FILTER_REGISTRY}" diff --git a/.deploy/cn2.yaml b/.deploy/cn2.yaml new file mode 100644 index 00000000..84fba006 --- /dev/null +++ b/.deploy/cn2.yaml @@ -0,0 +1,10 @@ +address: "${CN2_ADDRESS}" +quic_port: ${CN2_QUIC_PORT} +enable_mdns: false +chains: + - name: "sepolia" + rpc_url: "${RPC_URL}" + contracts: + enclave: "${SEPOLIA_ENCLAVE_ADDRESS}" + ciphernode_registry: "${SEPOLIA_CIPHERNODE_REGISTRY_ADDRESS}" + filter_registry: "${SEPOLIA_FILTER_REGISTRY}" diff --git a/.deploy/cn3.yaml b/.deploy/cn3.yaml new file mode 100644 index 00000000..8d167ddc --- /dev/null +++ b/.deploy/cn3.yaml @@ -0,0 +1,10 @@ +address: "${CN3_ADDRESS}" +quic_port: ${CN3_QUIC_PORT} +enable_mdns: false +chains: + - name: "sepolia" + rpc_url: "${RPC_URL}" + contracts: + enclave: "${SEPOLIA_ENCLAVE_ADDRESS}" + ciphernode_registry: "${SEPOLIA_CIPHERNODE_REGISTRY_ADDRESS}" + filter_registry: "${SEPOLIA_FILTER_REGISTRY}" diff --git a/.deploy/run.sh b/.deploy/run.sh new file mode 100755 index 00000000..807ce5a9 --- /dev/null +++ b/.deploy/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source .deploy/.env + +docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false From a7ce08244f3f1cc25b0bd172ee53852209ebf0c0 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 20:03:17 +1100 Subject: [PATCH 06/36] Add env example --- .deploy/.env.example | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.deploy/.env.example b/.deploy/.env.example index 266fb5c8..4d2b4940 100644 --- a/.deploy/.env.example +++ b/.deploy/.env.example @@ -1 +1,11 @@ -export TAG=latest +export TAG=20241210-44aac5b0 +export AGG_ADDRESS="0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199" +export CN1_ADDRESS="0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" +export CN2_ADDRESS="0xdD2FD4581271e230360230F9337D5c0430Bf44C0" +export CN3_ADDRESS="0x2546BcD3c84621e976D8185a91A922aE77ECEc30" + +export CN1_QUIC_PORT=9091 +export CN2_QUIC_PORT=9092 +export CN3_QUIC_PORT=9093 +export AGG_QUIC_PORT=9094 +export RPC_URL=wss://eth-sepolia.g.alchemy.com/v2/API_KEY From ec02226eb055c5725391a08c3bacf6bd5b793382 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 20:40:00 +1100 Subject: [PATCH 07/36] Update env vars --- .deploy/.env.example | 20 +++++++++---------- .deploy/agg.yaml | 4 ++-- .deploy/cn2.yaml | 4 ++-- .deploy/cn3.yaml | 4 ++-- .deploy/deploy.sh | 13 ++++++++++++ .deploy/docker-compose.yml | 28 ++++++++++++++++++-------- .deploy/run.sh | 5 ----- .deploy/swarm_deployment.md | 40 +++++++++++++++++++++++++++++++++++-- 8 files changed, 87 insertions(+), 31 deletions(-) create mode 100755 .deploy/deploy.sh delete mode 100755 .deploy/run.sh diff --git a/.deploy/.env.example b/.deploy/.env.example index 4d2b4940..95786950 100644 --- a/.deploy/.env.example +++ b/.deploy/.env.example @@ -1,11 +1,11 @@ -export TAG=20241210-44aac5b0 -export AGG_ADDRESS="0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199" -export CN1_ADDRESS="0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" -export CN2_ADDRESS="0xdD2FD4581271e230360230F9337D5c0430Bf44C0" -export CN3_ADDRESS="0x2546BcD3c84621e976D8185a91A922aE77ECEc30" +TAG=20241210-44aac5b0 +AGG_ADDRESS=0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199 +CN1_ADDRESS=0xbDA5747bFD65F08deb54cb465eB87D40e51B197E +CN2_ADDRESS=0xdD2FD4581271e230360230F9337D5c0430Bf44C0 +CN3_ADDRESS=0x2546BcD3c84621e976D8185a91A922aE77ECEc30 -export CN1_QUIC_PORT=9091 -export CN2_QUIC_PORT=9092 -export CN3_QUIC_PORT=9093 -export AGG_QUIC_PORT=9094 -export RPC_URL=wss://eth-sepolia.g.alchemy.com/v2/API_KEY +RPC_URL=wss://eth-sepolia.g.alchemy.com/v2/API_KEY + +SEPOLIA_ENCLAVE_ADDRESS=0xCe087F31e20E2F76b6544A2E4A74D4557C8fDf77 +SEPOLIA_CIPHERNODE_REGISTRY_ADDRESS=0x0952388f6028a9Eda93a5041a3B216Ea331d97Ab +SEPOLIA_FILTER_REGISTRY=0xcBaCE7C360b606bb554345b20884A28e41436934 diff --git a/.deploy/agg.yaml b/.deploy/agg.yaml index e41b6d79..064cf8c7 100644 --- a/.deploy/agg.yaml +++ b/.deploy/agg.yaml @@ -1,5 +1,5 @@ -address: "${AGG_ADDRESS}" -quic_port: ${AGG_QUIC_PORT} +address: "${ADDRESS}" +quic_port: ${QUIC_PORT} enable_mdns: false chains: - name: "sepolia" diff --git a/.deploy/cn2.yaml b/.deploy/cn2.yaml index 84fba006..064cf8c7 100644 --- a/.deploy/cn2.yaml +++ b/.deploy/cn2.yaml @@ -1,5 +1,5 @@ -address: "${CN2_ADDRESS}" -quic_port: ${CN2_QUIC_PORT} +address: "${ADDRESS}" +quic_port: ${QUIC_PORT} enable_mdns: false chains: - name: "sepolia" diff --git a/.deploy/cn3.yaml b/.deploy/cn3.yaml index 8d167ddc..064cf8c7 100644 --- a/.deploy/cn3.yaml +++ b/.deploy/cn3.yaml @@ -1,5 +1,5 @@ -address: "${CN3_ADDRESS}" -quic_port: ${CN3_QUIC_PORT} +address: "${ADDRESS}" +quic_port: ${QUIC_PORT} enable_mdns: false chains: - name: "sepolia" diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh new file mode 100755 index 00000000..f881472e --- /dev/null +++ b/.deploy/deploy.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +if [ ! -f ".env" ]; then + echo "Environment file .env not found!" + exit 1 +fi + +# Source the environment file +set -a # automatically export all variables +source .deploy/.env +set +a # turn off auto-export + +docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false diff --git a/.deploy/docker-compose.yml b/.deploy/docker-compose.yml index 40f9c368..b1838375 100644 --- a/.deploy/docker-compose.yml +++ b/.deploy/docker-compose.yml @@ -1,17 +1,20 @@ services: cn1: - image: ghcr.io/gnosisguild/ciphernode:${TAG} + image: ghcr.io/gnosisguild/ciphernode:${TAG:-latest} volumes: - ./cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/enclave secrets: - source: secrets_cn1 target: secrets.json + env_file: .env environment: RUST_LOG: "info" AGGREGATOR: "false" + ADDRESS: ${CN1_ADDRESS} + QUIC_PORT: 9091 ports: - - 9091:9091 + - "9091:9091" deploy: replicas: 1 update_config: @@ -25,7 +28,7 @@ services: cn2: - image: ghcr.io/gnosisguild/ciphernode:${TAG} + image: ghcr.io/gnosisguild/ciphernode:${TAG:-latest} depends_on: - cn1 volumes: @@ -34,11 +37,14 @@ services: secrets: - source: secrets_cn2 target: secrets.json + env_file: .env environment: RUST_LOG: "info" AGGREGATOR: "false" + ADDRESS: ${CN2_ADDRESS} + QUIC_PORT: 9092 ports: - - 9092:9092 + - "9092:9092" deploy: replicas: 1 @@ -52,7 +58,7 @@ services: - global-network cn3: - image: ghcr.io/gnosisguild/ciphernode:${TAG} + image: ghcr.io/gnosisguild/ciphernode:${TAG:-latest} depends_on: - cn1 volumes: @@ -61,11 +67,14 @@ services: secrets: - source: secrets_cn3 target: secrets.json + env_file: .env environment: RUST_LOG: "info" AGGREGATOR: "false" + ADDRESS: ${CN3_ADDRESS} + QUIC_PORT: 9093 ports: - - 9093:9093 + - "9093:9093" deploy: replicas: 1 update_config: @@ -79,7 +88,7 @@ services: aggregator: - image: ghcr.io/gnosisguild/ciphernode:${TAG} + image: ghcr.io/gnosisguild/ciphernode:${TAG:-latest} depends_on: - cn1 volumes: @@ -88,11 +97,14 @@ services: secrets: - source: secrets_agg target: secrets.json + env_file: .env environment: RUST_LOG: "info" AGGREGATOR: "true" + ADDRESS: ${AGG_ADDRESS} + QUIC_PORT: 9094 ports: - - 9094:9094 + - "9094:9094" deploy: replicas: 1 diff --git a/.deploy/run.sh b/.deploy/run.sh deleted file mode 100755 index 807ce5a9..00000000 --- a/.deploy/run.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -source .deploy/.env - -docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false diff --git a/.deploy/swarm_deployment.md b/.deploy/swarm_deployment.md index ee084749..78299248 100644 --- a/.deploy/swarm_deployment.md +++ b/.deploy/swarm_deployment.md @@ -1,3 +1,34 @@ +# Setup `.env` vars + +Copy the `.env.example` file to `.env` + +``` +cp .env.example .env +``` + +Alter the variables to reflect the correct values required for the stack: + +``` +export TAG=latest +export AGG_ADDRESS=0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199 +export CN1_ADDRESS=0xbDA5747bFD65F08deb54cb465eB87D40e51B197E +export CN2_ADDRESS=0xdD2FD4581271e230360230F9337D5c0430Bf44C0 +export CN3_ADDRESS=0x2546BcD3c84621e976D8185a91A922aE77ECEc30 + +export CN1_QUIC_PORT=9091 +export CN2_QUIC_PORT=9092 +export CN3_QUIC_PORT=9093 +export AGG_QUIC_PORT=9094 +export RPC_URL=wss://eth-sepolia.g.alchemy.com/v2/ + +export SEPOLIA_ENCLAVE_ADDRESS=0xCe087F31e20E2F76b6544A2E4A74D4557C8fDf77 +export SEPOLIA_CIPHERNODE_REGISTRY_ADDRESS=0x0952388f6028a9Eda93a5041a3B216Ea331d97Ab +export SEPOLIA_FILTER_REGISTRY=0xcBaCE7C360b606bb554345b20884A28e41436934 +``` + +Pay special attention to the `TAG` and `RPC_URL` vars. + +You can peruse the yaml config files for the nodes to see how the vars are used within the config. # Secrets Setup Script @@ -23,7 +54,7 @@ Skipping cn2.secrets.json - file already exists Remember to modify any highlighted files before use. -# Run docker swarm +# Initialize docker swarm First we need to initialize swarm. @@ -37,7 +68,12 @@ If you get an error about not being able to choose between IP addresses choose t docker swarm init --advertise-addr 10.49.x.x ``` + +# Deploy a version to the stack + +To deploy + ``` -TAG=latest docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false +.deploy/deploy.sh ``` From 4d92cd36cce624287658ac4030775f2320163bc5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 20:43:52 +1100 Subject: [PATCH 08/36] Update env location and envs --- .deploy/.env.example | 2 +- .deploy/deploy.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.deploy/.env.example b/.deploy/.env.example index 95786950..56904289 100644 --- a/.deploy/.env.example +++ b/.deploy/.env.example @@ -1,4 +1,4 @@ -TAG=20241210-44aac5b0 +TAG=latest AGG_ADDRESS=0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199 CN1_ADDRESS=0xbDA5747bFD65F08deb54cb465eB87D40e51B197E CN2_ADDRESS=0xdD2FD4581271e230360230F9337D5c0430Bf44C0 diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh index f881472e..cf7eff7c 100755 --- a/.deploy/deploy.sh +++ b/.deploy/deploy.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -if [ ! -f ".env" ]; then - echo "Environment file .env not found!" +if [ ! -f "./.deploy/.env" ]; then + echo "Environment file ./.deploy/.env not found!" exit 1 fi From 79d8d64e3ce5259ef0aad00477d0246d163fb8d9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 21:03:43 +1100 Subject: [PATCH 09/36] Try using the --env-file arg --- .deploy/deploy.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh index cf7eff7c..31d216a2 100755 --- a/.deploy/deploy.sh +++ b/.deploy/deploy.sh @@ -5,9 +5,4 @@ if [ ! -f "./.deploy/.env" ]; then exit 1 fi -# Source the environment file -set -a # automatically export all variables -source .deploy/.env -set +a # turn off auto-export - -docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false +docker stack deploy -c .deploy/docker-compose.yml --env-file .deploy/.env enclave-stack --detach=false From 2a7722f207a88c5a20fbd8ac2ce6e69da6b25823 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 21:05:07 +1100 Subject: [PATCH 10/36] Manually export vars --- .deploy/deploy.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh index 31d216a2..2a05c99f 100755 --- a/.deploy/deploy.sh +++ b/.deploy/deploy.sh @@ -5,4 +5,6 @@ if [ ! -f "./.deploy/.env" ]; then exit 1 fi -docker stack deploy -c .deploy/docker-compose.yml --env-file .deploy/.env enclave-stack --detach=false +export $(cat .deploy/.env | xargs) + +docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false From da9d805edba25fda11e48e733a362ce18f59392e Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 21:08:46 +1100 Subject: [PATCH 11/36] source the env vars --- .deploy/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh index 2a05c99f..e2f10a76 100755 --- a/.deploy/deploy.sh +++ b/.deploy/deploy.sh @@ -5,6 +5,6 @@ if [ ! -f "./.deploy/.env" ]; then exit 1 fi -export $(cat .deploy/.env | xargs) +source .deploy/.env docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false From abdc9b983a3f83affc966e5c7ffe0aee0a5870ff Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 21:21:12 +1100 Subject: [PATCH 12/36] use env file --- .deploy/deploy.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh index e2f10a76..a4822ba0 100755 --- a/.deploy/deploy.sh +++ b/.deploy/deploy.sh @@ -5,6 +5,4 @@ if [ ! -f "./.deploy/.env" ]; then exit 1 fi -source .deploy/.env - -docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false +docker stack deploy --env-file ./.deploy/.env -c .deploy/docker-compose.yml enclave-stack --detach=false From a199f7f2885fb2a4e88d45add9069ba7129b7116 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 11 Dec 2024 11:26:05 +0000 Subject: [PATCH 13/36] temporarily print the config as it is loaded --- .deploy/build.sh | 3 +++ .deploy/cn1.yaml | 4 ++-- .deploy/deploy.sh | 2 +- .deploy/docker-compose.yml | 19 ++++++++----------- packages/ciphernode/config/src/app_config.rs | 1 + 5 files changed, 15 insertions(+), 14 deletions(-) create mode 100755 .deploy/build.sh diff --git a/.deploy/build.sh b/.deploy/build.sh new file mode 100755 index 00000000..dbde3504 --- /dev/null +++ b/.deploy/build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker build -t ghcr.io/gnosisguild/ciphernode:mytest -f ./packages/ciphernode/Dockerfile . diff --git a/.deploy/cn1.yaml b/.deploy/cn1.yaml index 18b730e9..064cf8c7 100644 --- a/.deploy/cn1.yaml +++ b/.deploy/cn1.yaml @@ -1,5 +1,5 @@ -address: "${CN1_ADDRESS}" -quic_port: ${CN1_QUIC_PORT} +address: "${ADDRESS}" +quic_port: ${QUIC_PORT} enable_mdns: false chains: - name: "sepolia" diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh index a4822ba0..929d9a6c 100755 --- a/.deploy/deploy.sh +++ b/.deploy/deploy.sh @@ -5,4 +5,4 @@ if [ ! -f "./.deploy/.env" ]; then exit 1 fi -docker stack deploy --env-file ./.deploy/.env -c .deploy/docker-compose.yml enclave-stack --detach=false +docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false diff --git a/.deploy/docker-compose.yml b/.deploy/docker-compose.yml index b1838375..7c363abf 100644 --- a/.deploy/docker-compose.yml +++ b/.deploy/docker-compose.yml @@ -1,6 +1,6 @@ services: cn1: - image: ghcr.io/gnosisguild/ciphernode:${TAG:-latest} + image: ghcr.io/gnosisguild/ciphernode:20241210-44aac5b0 volumes: - ./cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/enclave @@ -11,7 +11,7 @@ services: environment: RUST_LOG: "info" AGGREGATOR: "false" - ADDRESS: ${CN1_ADDRESS} + ADDRESS: "0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" QUIC_PORT: 9091 ports: - "9091:9091" @@ -26,9 +26,8 @@ services: networks: - global-network - cn2: - image: ghcr.io/gnosisguild/ciphernode:${TAG:-latest} + image: ghcr.io/gnosisguild/ciphernode:20241210-44aac5b0 depends_on: - cn1 volumes: @@ -41,11 +40,10 @@ services: environment: RUST_LOG: "info" AGGREGATOR: "false" - ADDRESS: ${CN2_ADDRESS} + ADDRESS: "0xdD2FD4581271e230360230F9337D5c0430Bf44C0" QUIC_PORT: 9092 ports: - "9092:9092" - deploy: replicas: 1 update_config: @@ -58,7 +56,7 @@ services: - global-network cn3: - image: ghcr.io/gnosisguild/ciphernode:${TAG:-latest} + image: ghcr.io/gnosisguild/ciphernode:20241210-44aac5b0 depends_on: - cn1 volumes: @@ -71,7 +69,7 @@ services: environment: RUST_LOG: "info" AGGREGATOR: "false" - ADDRESS: ${CN3_ADDRESS} + ADDRESS: "0x2546BcD3c84621e976D8185a91A922aE77ECEc30" QUIC_PORT: 9093 ports: - "9093:9093" @@ -86,9 +84,8 @@ services: networks: - global-network - aggregator: - image: ghcr.io/gnosisguild/ciphernode:${TAG:-latest} + image: ghcr.io/gnosisguild/ciphernode:20241210-44aac5b0 depends_on: - cn1 volumes: @@ -101,7 +98,7 @@ services: environment: RUST_LOG: "info" AGGREGATOR: "true" - ADDRESS: ${AGG_ADDRESS} + ADDRESS: "0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199" QUIC_PORT: 9094 ports: - "9094:9094" diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index cc6eac80..d821b29e 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -273,6 +273,7 @@ pub fn load_config(config_file: Option<&str>) -> Result { } let with_envs = load_yaml_with_env(&defaults.config_file())?; + println!("{}", with_envs); let config = Figment::from(Serialized::defaults(&defaults)) .merge(Yaml::string(&with_envs)) From c92e338f5dc3bf3d3c18846279a981e1084c8ed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Wed, 11 Dec 2024 22:49:52 +1100 Subject: [PATCH 14/36] Update swarm_deployment.md --- .deploy/swarm_deployment.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.deploy/swarm_deployment.md b/.deploy/swarm_deployment.md index 78299248..93a922fc 100644 --- a/.deploy/swarm_deployment.md +++ b/.deploy/swarm_deployment.md @@ -9,18 +9,7 @@ cp .env.example .env Alter the variables to reflect the correct values required for the stack: ``` -export TAG=latest -export AGG_ADDRESS=0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199 -export CN1_ADDRESS=0xbDA5747bFD65F08deb54cb465eB87D40e51B197E -export CN2_ADDRESS=0xdD2FD4581271e230360230F9337D5c0430Bf44C0 -export CN3_ADDRESS=0x2546BcD3c84621e976D8185a91A922aE77ECEc30 - -export CN1_QUIC_PORT=9091 -export CN2_QUIC_PORT=9092 -export CN3_QUIC_PORT=9093 -export AGG_QUIC_PORT=9094 export RPC_URL=wss://eth-sepolia.g.alchemy.com/v2/ - export SEPOLIA_ENCLAVE_ADDRESS=0xCe087F31e20E2F76b6544A2E4A74D4557C8fDf77 export SEPOLIA_CIPHERNODE_REGISTRY_ADDRESS=0x0952388f6028a9Eda93a5041a3B216Ea331d97Ab export SEPOLIA_FILTER_REGISTRY=0xcBaCE7C360b606bb554345b20884A28e41436934 From 196bb16aa30c0d8a5af947362db25d5190be6eb8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 19 Dec 2024 18:01:08 +1100 Subject: [PATCH 15/36] Compiling --- .deploy/build.sh | 11 +- .deploy/deploy.sh | 29 ++- .deploy/docker-compose.yml | 22 ++- packages/ciphernode/Cargo.lock | 31 +++ packages/ciphernode/Cargo.toml | 1 + packages/ciphernode/enclave/Cargo.toml | 3 + packages/ciphernode/enclave/src/compile_id.rs | 13 ++ packages/ciphernode/enclave/src/main.rs | 7 +- packages/ciphernode/net/src/bin/p2p_test.rs | 20 +- packages/ciphernode/net/src/correlation_id.rs | 24 +++ packages/ciphernode/net/src/dialer.rs | 182 ++++++++++++++++++ packages/ciphernode/net/src/events.rs | 44 +++++ packages/ciphernode/net/src/lib.rs | 5 + .../ciphernode/net/src/network_manager.rs | 60 ++++-- packages/ciphernode/net/src/network_peer.rs | 140 +++++++++----- packages/ciphernode/net/src/retry.rs | 71 +++++++ 16 files changed, 577 insertions(+), 86 deletions(-) create mode 100644 packages/ciphernode/enclave/src/compile_id.rs create mode 100644 packages/ciphernode/net/src/correlation_id.rs create mode 100644 packages/ciphernode/net/src/dialer.rs create mode 100644 packages/ciphernode/net/src/events.rs create mode 100644 packages/ciphernode/net/src/retry.rs diff --git a/.deploy/build.sh b/.deploy/build.sh index dbde3504..f3bdfbdc 100755 --- a/.deploy/build.sh +++ b/.deploy/build.sh @@ -1,3 +1,12 @@ #!/usr/bin/env bash -docker build -t ghcr.io/gnosisguild/ciphernode:mytest -f ./packages/ciphernode/Dockerfile . +# Enable BuildKit +export DOCKER_BUILDKIT=1 + +mkdir -p /tmp/docker-cache + +time docker buildx build \ + --cache-from=type=local,src=/tmp/docker-cache \ + --cache-to=type=local,dest=/tmp/docker-cache \ + --load \ + -t ghcr.io/gnosisguild/ciphernode -f ./packages/ciphernode/Dockerfile . diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh index 929d9a6c..397c8ec8 100755 --- a/.deploy/deploy.sh +++ b/.deploy/deploy.sh @@ -1,8 +1,27 @@ #!/usr/bin/env bash -if [ ! -f "./.deploy/.env" ]; then - echo "Environment file ./.deploy/.env not found!" - exit 1 -fi +wait_ready() { + local stack_name="$1" + until [ "$(docker stack services $stack_name --format '{{.Replicas}}' | awk -F'/' '$1 != $2')" = "" ]; do + printf "." + sleep 1 + done + echo -ne "\r\033[K" + echo "Stack $stack_name is ready!" +} -docker stack deploy -c .deploy/docker-compose.yml enclave-stack --detach=false +wait_removed() { + local stack_name="$1" + while docker stack ps $stack_name >/dev/null 2>&1; do + printf "." + sleep 1 + done + echo -ne "\r\033[K" + echo "Stack $stack_name is removed" +} + +stack_name=${1:-enclave} +docker stack rm $stack_name +wait_removed $stack_name +docker stack deploy -c docker-compose.yml --prune $stack_name +wait_ready $stack_name diff --git a/.deploy/docker-compose.yml b/.deploy/docker-compose.yml index 7c363abf..3a3d45e2 100644 --- a/.deploy/docker-compose.yml +++ b/.deploy/docker-compose.yml @@ -1,6 +1,6 @@ services: cn1: - image: ghcr.io/gnosisguild/ciphernode:20241210-44aac5b0 + image: ghcr.io/gnosisguild/ciphernode:latest volumes: - ./cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/enclave @@ -14,9 +14,11 @@ services: ADDRESS: "0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" QUIC_PORT: 9091 ports: - - "9091:9091" + - "9091:9091/udp" + - "9091:9091/tcp" deploy: replicas: 1 + endpoint_mode: dnsrr update_config: parallelism: 1 order: stop-first @@ -27,7 +29,7 @@ services: - global-network cn2: - image: ghcr.io/gnosisguild/ciphernode:20241210-44aac5b0 + image: ghcr.io/gnosisguild/ciphernode:latest depends_on: - cn1 volumes: @@ -43,7 +45,8 @@ services: ADDRESS: "0xdD2FD4581271e230360230F9337D5c0430Bf44C0" QUIC_PORT: 9092 ports: - - "9092:9092" + - "9092:9092/udp" + - "9092:9092/tcp" deploy: replicas: 1 update_config: @@ -56,7 +59,7 @@ services: - global-network cn3: - image: ghcr.io/gnosisguild/ciphernode:20241210-44aac5b0 + image: ghcr.io/gnosisguild/ciphernode:latest depends_on: - cn1 volumes: @@ -72,7 +75,8 @@ services: ADDRESS: "0x2546BcD3c84621e976D8185a91A922aE77ECEc30" QUIC_PORT: 9093 ports: - - "9093:9093" + - "9093:9093/udp" + - "9093:9093/tcp" deploy: replicas: 1 update_config: @@ -85,7 +89,7 @@ services: - global-network aggregator: - image: ghcr.io/gnosisguild/ciphernode:20241210-44aac5b0 + image: ghcr.io/gnosisguild/ciphernode:latest depends_on: - cn1 volumes: @@ -101,8 +105,8 @@ services: ADDRESS: "0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199" QUIC_PORT: 9094 ports: - - "9094:9094" - + - "9094:9094/udp" + - "9094:9094/tcp" deploy: replicas: 1 update_config: diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 0684a617..ffdfb98e 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -1676,6 +1676,20 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +[[package]] +name = "compile-time" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e55ede5279d4d7c528906853743abeb26353ae1e6c440fcd6d18316c2c2dd903" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "rustc_version 0.4.0", + "semver 1.0.23", + "time", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2164,6 +2178,7 @@ dependencies = [ "anyhow", "cipher 0.1.0", "clap", + "compile-time", "config", "data", "dialoguer", @@ -2172,7 +2187,9 @@ dependencies = [ "enclave_node", "hex", "once_cell", + "petname", "phf", + "rand", "router", "serde", "serde_json", @@ -4583,6 +4600,20 @@ dependencies = [ "indexmap", ] +[[package]] +name = "petname" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cd31dcfdbbd7431a807ef4df6edd6473228e94d5c805e8cf671227a21bad068" +dependencies = [ + "anyhow", + "clap", + "itertools 0.13.0", + "proc-macro2", + "quote", + "rand", +] + [[package]] name = "pharos" version = "0.5.3" diff --git a/packages/ciphernode/Cargo.toml b/packages/ciphernode/Cargo.toml index 8ce8f55a..ae75b191 100644 --- a/packages/ciphernode/Cargo.toml +++ b/packages/ciphernode/Cargo.toml @@ -37,6 +37,7 @@ bs58 = "0.5.1" base64 = "0.22.1" clap = { version = "4.5.17", features = ["derive"] } cipher = { path = "./cipher" } +compile-time = "0.2.0" dirs = "5.0.1" data = { path = "./data" } shellexpand = "3.1.0" diff --git a/packages/ciphernode/enclave/Cargo.toml b/packages/ciphernode/enclave/Cargo.toml index d38cf579..cbec2fca 100644 --- a/packages/ciphernode/enclave/Cargo.toml +++ b/packages/ciphernode/enclave/Cargo.toml @@ -28,6 +28,9 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } zeroize = { workspace = true } phf = { version = "0.11", features = ["macros"] } +compile-time = { workspace = true } +rand = { workspace = true } +petname = "2.0.2" [build-dependencies] serde_json = { workspace = true } diff --git a/packages/ciphernode/enclave/src/compile_id.rs b/packages/ciphernode/enclave/src/compile_id.rs new file mode 100644 index 00000000..38b4b9d4 --- /dev/null +++ b/packages/ciphernode/enclave/src/compile_id.rs @@ -0,0 +1,13 @@ +use petname::{Generator, Petnames}; +use rand::rngs::StdRng; +use rand::SeedableRng; + +static COMPILE_ID: u64 = compile_time::unix!(); + +/// Generate a unique compilation ID for the build based on the time of compilation +pub fn generate_id() -> String { + let mut rng = StdRng::seed_from_u64(COMPILE_ID); + Petnames::small() + .generate(&mut rng, 3, "_") + .unwrap_or("default-name".to_owned()) +} diff --git a/packages/ciphernode/enclave/src/main.rs b/packages/ciphernode/enclave/src/main.rs index 808b7629..3b2c4d5d 100644 --- a/packages/ciphernode/enclave/src/main.rs +++ b/packages/ciphernode/enclave/src/main.rs @@ -3,9 +3,11 @@ use clap::Parser; use commands::{aggregator, init, net, password, start, wallet, Commands}; use config::load_config; use enclave_core::{get_tag, set_tag}; -use tracing::instrument; +use tracing::{info, instrument}; use tracing_subscriber::EnvFilter; + pub mod commands; +mod compile_id; const OWO: &str = r#" ___ ___ ___ ___ ___ @@ -85,6 +87,9 @@ pub async fn main() { // .with_env_filter("[app{id=cn4}]=info") // .with_env_filter("[app{id=ag}]=info") .init(); + + info!("COMPILATION ID: '{}'", compile_id::generate_id()); + let cli = Cli::parse(); // Set the tag for all future traces diff --git a/packages/ciphernode/net/src/bin/p2p_test.rs b/packages/ciphernode/net/src/bin/p2p_test.rs index e44f43b8..bea5b098 100644 --- a/packages/ciphernode/net/src/bin/p2p_test.rs +++ b/packages/ciphernode/net/src/bin/p2p_test.rs @@ -1,4 +1,6 @@ use anyhow::Result; +use net::correlation_id::CorrelationId; +use net::events::{NetworkPeerCommand, NetworkPeerEvent}; use net::NetworkPeer; use std::time::Duration; use std::{collections::HashSet, env, process}; @@ -19,7 +21,7 @@ async fn main() -> Result<()> { .with(tracing_subscriber::fmt::layer()) .init(); let name = env::args().nth(1).expect("need name"); - + let topic = "test-topic"; println!("{} starting up", name); let udp_port = env::var("QUIC_PORT") @@ -42,7 +44,7 @@ async fn main() -> Result<()> { // Extract input and outputs let tx = peer.tx(); - let mut rx = peer.rx().unwrap(); + let mut rx = peer.rx(); let router_task = tokio::spawn({ let name = name.clone(); @@ -60,7 +62,12 @@ async fn main() -> Result<()> { // Send our message first println!("{} sending message", name); - tx.send(name.as_bytes().to_vec()).await?; + tx.send(NetworkPeerCommand::GossipPublish { + correlation_id: CorrelationId::new(), + topic: topic.to_string(), + data: name.as_bytes().to_vec(), + }) + .await?; println!("{} message sent", name); let expected: HashSet = vec![ @@ -79,8 +86,8 @@ async fn main() -> Result<()> { // Wrap the message receiving loop in a timeout let receive_result = timeout(Duration::from_secs(10), async { while received != expected { - if let Some(msg) = rx.recv().await { - match String::from_utf8(msg) { + match rx.recv().await? { + NetworkPeerEvent::GossipData(msg) => match String::from_utf8(msg) { Ok(msg) => { if !received.contains(&msg) { println!("{} received '{}'", name, msg); @@ -88,7 +95,8 @@ async fn main() -> Result<()> { } } Err(e) => println!("{} received invalid UTF8: {}", name, e), - } + }, + _ => (), } } Ok::<(), anyhow::Error>(()) diff --git a/packages/ciphernode/net/src/correlation_id.rs b/packages/ciphernode/net/src/correlation_id.rs new file mode 100644 index 00000000..3d14bcab --- /dev/null +++ b/packages/ciphernode/net/src/correlation_id.rs @@ -0,0 +1,24 @@ +use std::{ + fmt::Display, + sync::atomic::{AtomicUsize, Ordering}, +}; + +static NEXT_CORRELATION_ID: AtomicUsize = AtomicUsize::new(1); + +#[derive(Debug,Clone)] +pub struct CorrelationId { + id: usize, +} + +impl CorrelationId { + pub fn new() -> Self { + let id = NEXT_CORRELATION_ID.fetch_add(1, Ordering::SeqCst); + Self { id } + } +} + +impl Display for CorrelationId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.id) + } +} diff --git a/packages/ciphernode/net/src/dialer.rs b/packages/ciphernode/net/src/dialer.rs new file mode 100644 index 00000000..7447fe88 --- /dev/null +++ b/packages/ciphernode/net/src/dialer.rs @@ -0,0 +1,182 @@ +use anyhow::Context; +use anyhow::Result; +use futures::future::join_all; +use libp2p::{ + multiaddr::Protocol, + swarm::{dial_opts::DialOpts, ConnectionId, DialError}, + Multiaddr, +}; +use std::net::ToSocketAddrs; +use tokio::sync::{broadcast, mpsc}; +use tracing::error; + +use crate::{ + events::{NetworkPeerCommand, NetworkPeerEvent}, + retry::{retry_with_backoff, to_retry, RetryError, BACKOFF_DELAY, BACKOFF_MAX_RETRIES}, +}; + +async fn dial_multiaddr( + cmd_tx: &mpsc::Sender, + event_tx: &broadcast::Sender, + multiaddr_str: &str, +) -> Result<()> { + let multiaddr = &multiaddr_str.parse()?; + println!("Now dialing in to {}", multiaddr); + retry_with_backoff( + || attempt_connection(cmd_tx, event_tx, multiaddr), + BACKOFF_MAX_RETRIES, + BACKOFF_DELAY, + ) + .await?; + Ok(()) +} + +fn trace_error(r: Result<()>) { + if let Err(err) = r { + error!("{}", err); + } +} + +pub async fn dial_peers( + cmd_tx: &mpsc::Sender, + event_tx: &broadcast::Sender, + peers: &Vec, +) -> Result<()> { + let futures: Vec<_> = peers + .iter() + .map(|addr| dial_multiaddr(cmd_tx, event_tx, addr)) + .collect(); + let results = join_all(futures).await; + results.into_iter().for_each(trace_error); + Ok(()) +} + +async fn attempt_connection( + cmd_tx: &mpsc::Sender, + event_tx: &broadcast::Sender, + multiaddr: &Multiaddr, +) -> Result<(), RetryError> { + let mut event_rx = event_tx.subscribe(); + let multi = get_resolved_multiaddr(multiaddr).map_err(to_retry)?; + let opts: DialOpts = multi.clone().into(); + let dial_connection = opts.connection_id(); + println!("Dialing: '{}' with connection '{}'", multi, dial_connection); + cmd_tx + .send(NetworkPeerCommand::Dial(opts)) + .await + .map_err(to_retry)?; + wait_for_connection(&mut event_rx, dial_connection).await +} + +async fn wait_for_connection( + event_rx: &mut broadcast::Receiver, + dial_connection: ConnectionId, +) -> Result<(), RetryError> { + loop { + match event_rx.recv().await.map_err(to_retry)? { + NetworkPeerEvent::ConnectionEstablished { connection_id } => { + if connection_id == dial_connection { + println!("Connection Established"); + return Ok(()); + } + } + NetworkPeerEvent::DialError { error } => { + println!("DialError!"); + return match error.as_ref() { + // If we are dialing ourself then we should just fail + DialError::NoAddresses { .. } => { + println!("DialError received. Returning RetryError::Failure"); + Err(RetryError::Failure(error.clone().into())) + } + // Try again otherwise + _ => Err(RetryError::Retry(error.clone().into())), + }; + } + NetworkPeerEvent::OutgoingConnectionError { + connection_id, + error, + } => { + println!("OutgoingConnectionError!"); + if connection_id == dial_connection { + println!( + "Connection {} failed because of error {}. Retrying...", + connection_id, error + ); + return match error.as_ref() { + // If we are dialing ourself then we should just fail + DialError::NoAddresses { .. } => { + Err(RetryError::Failure(error.clone().into())) + } + // Try again otherwise + _ => Err(RetryError::Retry(error.clone().into())), + }; + } + } + _ => (), + } + } +} + +fn dns_to_ip_addr(original: &Multiaddr, ip_str: &str) -> Result { + let ip = ip_str.parse()?; + let mut new_addr = Multiaddr::empty(); + let mut skip_next = false; + + for proto in original.iter() { + if skip_next { + skip_next = false; + continue; + } + + match proto { + Protocol::Dns4(_) | Protocol::Dns6(_) => { + new_addr.push(Protocol::Ip4(ip)); + skip_next = false; + } + _ => new_addr.push(proto), + } + } + + Ok(new_addr) +} + +fn extract_dns_host(addr: &Multiaddr) -> Option { + // Iterate through the protocols in the multiaddr + for proto in addr.iter() { + match proto { + // Match on DNS4 or DNS6 protocols + Protocol::Dns4(hostname) | Protocol::Dns6(hostname) => { + return Some(hostname.to_string()) + } + _ => continue, + } + } + None +} + +fn get_resolved_multiaddr(value: &Multiaddr) -> Result { + let maybe_domain = extract_dns_host(value); + if let Some(domain) = maybe_domain { + let ip = resolve_ipv4(&domain)?; + let multi = dns_to_ip_addr(value, &ip)?; + return Ok(multi); + } else { + Ok(value.clone()) + } +} + +fn resolve_ipv4(domain: &str) -> Result { + let addr = format!("{}:0", domain) + .to_socket_addrs()? + .find(|addr| addr.ip().is_ipv4()) + .context("no IPv4 addresses found")?; + Ok(addr.ip().to_string()) +} + +fn resolve_ipv6(domain: &str) -> Result { + let addr = format!("{}:0", domain) + .to_socket_addrs()? + .find(|addr| addr.ip().is_ipv6()) + .context("no IPv6 addresses found")?; + Ok(addr.ip().to_string()) +} diff --git a/packages/ciphernode/net/src/events.rs b/packages/ciphernode/net/src/events.rs new file mode 100644 index 00000000..7719279e --- /dev/null +++ b/packages/ciphernode/net/src/events.rs @@ -0,0 +1,44 @@ +use std::sync::Arc; + +use actix::Message; +use libp2p::{ + gossipsub::{MessageId, PublishError}, + swarm::{dial_opts::DialOpts, ConnectionId, DialError}, +}; + +use crate::correlation_id::CorrelationId; + +pub enum NetworkPeerCommand { + GossipPublish { + topic: String, + data: Vec, + correlation_id: CorrelationId, + }, + Dial(DialOpts), +} + +#[derive(Message, Clone, Debug)] +#[rtype(result = "anyhow::Result<()>")] +pub enum NetworkPeerEvent { + GossipData(Vec), + GossipPublishError { + // TODO: return an error here? DialError is not Clonable so we have + // avoided passing it on + correlation_id: CorrelationId, + error: Arc, + }, + GossipPublished { + correlation_id: CorrelationId, + message_id: MessageId, + }, + DialError { + error: Arc, + }, + ConnectionEstablished { + connection_id: ConnectionId, + }, + OutgoingConnectionError { + connection_id: ConnectionId, + error: Arc, + }, +} diff --git a/packages/ciphernode/net/src/lib.rs b/packages/ciphernode/net/src/lib.rs index 42f3da11..394055bf 100644 --- a/packages/ciphernode/net/src/lib.rs +++ b/packages/ciphernode/net/src/lib.rs @@ -3,6 +3,11 @@ mod network_manager; mod network_peer; +mod dialer; +pub mod events; +mod retry; +pub mod correlation_id; pub use network_manager::*; pub use network_peer::*; + diff --git a/packages/ciphernode/net/src/network_manager.rs b/packages/ciphernode/net/src/network_manager.rs index 8969c908..bd2b4f87 100644 --- a/packages/ciphernode/net/src/network_manager.rs +++ b/packages/ciphernode/net/src/network_manager.rs @@ -1,8 +1,10 @@ +use crate::correlation_id::CorrelationId; +use crate::events::NetworkPeerCommand; +use crate::events::NetworkPeerEvent; use crate::NetworkPeer; /// Actor for connecting to an libp2p client via it's mpsc channel interface /// This Actor should be responsible for use actix::prelude::*; -use anyhow::anyhow; use anyhow::Result; use cipher::Cipher; use data::Repository; @@ -10,15 +12,18 @@ use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; use libp2p::identity::ed25519; use std::collections::HashSet; use std::sync::Arc; -use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::select; +use tokio::sync::broadcast; +use tokio::sync::mpsc; use tracing::{error, info, instrument, trace}; /// NetworkManager Actor converts between EventBus events and Libp2p events forwarding them to a /// NetworkPeer for propagation over the p2p network pub struct NetworkManager { bus: Addr, - tx: Sender>, + tx: mpsc::Sender, sent_events: HashSet, + topic: String, } impl Actor for NetworkManager { @@ -31,20 +36,22 @@ struct LibP2pEvent(pub Vec); impl NetworkManager { /// Create a new NetworkManager actor - pub fn new(bus: Addr, tx: Sender>) -> Self { + pub fn new(bus: Addr, tx: mpsc::Sender, topic: &str) -> Self { Self { bus, tx, sent_events: HashSet::new(), + topic: topic.to_string(), } } pub fn setup( bus: Addr, - tx: Sender>, - mut rx: Receiver>, + tx: mpsc::Sender, + mut rx: broadcast::Receiver, + topic: &str, ) -> Addr { - let addr = NetworkManager::new(bus.clone(), tx).start(); + let addr = NetworkManager::new(bus.clone(), tx, topic).start(); // Listen on all events bus.do_send(Subscribe { @@ -54,10 +61,18 @@ impl NetworkManager { tokio::spawn({ let addr = addr.clone(); - async move { - while let Some(msg) = rx.recv().await { - addr.do_send(LibP2pEvent(msg)) + loop { + select! { + Ok(event) = rx.recv() => { + match event { + NetworkPeerEvent::GossipData(data) => { + addr.do_send(LibP2pEvent(data)) + }, + _ => () + } + } + } } } }); @@ -75,6 +90,7 @@ impl NetworkManager { enable_mdns: bool, repository: Repository>, ) -> Result<(Addr, tokio::task::JoinHandle>, String)> { + let topic = "tmp-enclave-gossip-topic"; info!("Reading from repository"); let mut bytes = if let Some(bytes) = repository.read().await? { let decrypted = cipher.decrypt_data(&bytes)?; @@ -94,15 +110,9 @@ impl NetworkManager { let ed25519_keypair = ed25519::Keypair::try_from_bytes(&mut bytes)?; let keypair: libp2p::identity::Keypair = ed25519_keypair.try_into()?; - let mut peer = NetworkPeer::new( - &keypair, - peers, - Some(quic_port), - "tmp-enclave-gossip-topic", - enable_mdns, - )?; - let rx = peer.rx().ok_or(anyhow!("Peer rx already taken"))?; - let p2p_addr = NetworkManager::setup(bus, peer.tx(), rx); + let mut peer = NetworkPeer::new(&keypair, peers, Some(quic_port), topic, enable_mdns)?; + let rx = peer.rx(); + let p2p_addr = NetworkManager::setup(bus, peer.tx(), rx, topic); let handle = tokio::spawn(async move { Ok(peer.start().await?) }); Ok((p2p_addr, handle, keypair.public().to_peer_id().to_string())) } @@ -129,6 +139,7 @@ impl Handler for NetworkManager { let sent_events = self.sent_events.clone(); let tx = self.tx.clone(); let evt = event.clone(); + let topic = self.topic.clone(); Box::pin(async move { let id: EventId = evt.clone().into(); @@ -145,8 +156,15 @@ impl Handler for NetworkManager { } match evt.to_bytes() { - Ok(bytes) => { - if let Err(e) = tx.send(bytes).await { + Ok(data) => { + if let Err(e) = tx + .send(NetworkPeerCommand::GossipPublish { + topic, + data, + correlation_id: CorrelationId::new(), + }) + .await + { error!(error=?e, "Error sending bytes to libp2p"); }; } diff --git a/packages/ciphernode/net/src/network_peer.rs b/packages/ciphernode/net/src/network_peer.rs index 17151076..00366439 100644 --- a/packages/ciphernode/net/src/network_peer.rs +++ b/packages/ciphernode/net/src/network_peer.rs @@ -8,15 +8,17 @@ use libp2p::{ kad::{store::MemoryStore, Behaviour as KademliaBehaviour}, mdns, swarm::{behaviour::toggle::Toggle, NetworkBehaviour, SwarmEvent}, - Multiaddr, Swarm, + Swarm, }; use std::hash::{Hash, Hasher}; +use std::sync::Arc; use std::{hash::DefaultHasher, io::Error, time::Duration}; -use tokio::{ - select, - sync::mpsc::{channel, Receiver, Sender}, -}; -use tracing::{debug, error, info, trace, warn}; +use tokio::{select, sync::broadcast, sync::mpsc}; +use tracing::{debug, info, trace, warn}; + +use crate::dialer::dial_peers; +use crate::events::NetworkPeerCommand; +use crate::events::NetworkPeerEvent; #[derive(NetworkBehaviour)] pub struct NodeBehaviour { @@ -32,10 +34,9 @@ pub struct NetworkPeer { peers: Vec, udp_port: Option, topic: gossipsub::IdentTopic, - to_bus_tx: Sender>, // to event bus - from_net_rx: Option>>, // from network - to_net_tx: Sender>, // to network - from_bus_rx: Receiver>, // from event bus + event_tx: broadcast::Sender, // to event bus + cmd_tx: mpsc::Sender, // to network + cmd_rx: mpsc::Receiver, // from event bus } impl NetworkPeer { @@ -46,14 +47,13 @@ impl NetworkPeer { topic: &str, enable_mdns: bool, ) -> Result { - let (to_bus_tx, from_net_rx) = channel(100); // TODO : tune this param - let (to_net_tx, from_bus_rx) = channel(100); // TODO : tune this param + let (event_tx, _) = broadcast::channel(100); // TODO : tune this param + let (cmd_tx, cmd_rx) = mpsc::channel(100); // TODO : tune this param let swarm = libp2p::SwarmBuilder::with_existing_identity(id.clone()) .with_tokio() .with_quic() .with_behaviour(|key| create_mdns_kad_behaviour(enable_mdns, key))? - .with_swarm_config(|c| c.with_idle_connection_timeout(Duration::from_secs(60))) .build(); // TODO: Use topics to manage network traffic instead of just using a single topic @@ -64,52 +64,84 @@ impl NetworkPeer { peers, udp_port, topic, - to_bus_tx, - from_net_rx: Some(from_net_rx), - to_net_tx, - from_bus_rx, + event_tx, + cmd_tx, + cmd_rx, }) } - pub fn rx(&mut self) -> Option>> { - self.from_net_rx.take() + pub fn rx(&mut self) -> broadcast::Receiver { + self.event_tx.subscribe() } - pub fn tx(&self) -> Sender> { - self.to_net_tx.clone() + pub fn tx(&self) -> mpsc::Sender { + self.cmd_tx.clone() } pub async fn start(&mut self) -> Result<()> { - let addr = match self.udp_port { - Some(port) => format!("/ip4/0.0.0.0/udp/{}/quic-v1", port), - None => "/ip4/0.0.0.0/udp/0/quic-v1".to_string(), - }; - info!("Requesting node.listen_on('{}')", addr); + let event_tx = self.event_tx.clone(); + let cmd_tx = self.cmd_tx.clone(); + let cmd_rx = &mut self.cmd_rx; + // Subscribe to topic self.swarm .behaviour_mut() .gossipsub .subscribe(&self.topic)?; + + // Listen on the quic port + let addr = match self.udp_port { + Some(port) => format!("/ip4/0.0.0.0/udp/{}/quic-v1", port), + None => "/ip4/0.0.0.0/udp/0/quic-v1".to_string(), + }; + + info!("Requesting node.listen_on('{}')", addr); self.swarm.listen_on(addr.parse()?)?; info!("Peers to dial: {:?}", self.peers); - for addr in self.peers.clone() { - let multiaddr: Multiaddr = addr.parse()?; - self.swarm.dial(multiaddr)?; - } + tokio::spawn({ + let event_tx = event_tx.clone(); + let peers = self.peers.clone(); + async move { + dial_peers(&cmd_tx, &event_tx, &peers).await?; + + return anyhow::Ok(()); + } + }); loop { select! { - Some(line) = self.from_bus_rx.recv() => { - if let Err(e) = self.swarm - .behaviour_mut().gossipsub - .publish(self.topic.clone(), line) { - error!(error=?e, "Error publishing line to swarm"); + // Process commands + Some(command) = cmd_rx.recv() => { + match command { + NetworkPeerCommand::GossipPublish { data, topic, correlation_id } => { + let gossipsub_behaviour = &mut self.swarm.behaviour_mut().gossipsub; + match gossipsub_behaviour + .publish(gossipsub::IdentTopic::new(topic), data) { + Ok(message_id) => { + event_tx.send(NetworkPeerEvent::GossipPublished { correlation_id, message_id })?; + }, + Err(e) => { + warn!(error=?e, "Could not publish to swarm. Retrying..."); + event_tx.send(NetworkPeerEvent::GossipPublishError { correlation_id, error: Arc::new(e) })?; + } + } + }, + NetworkPeerCommand::Dial(multi) => { + println!("DIAL: {:?}", multi); + match self.swarm.dial(multi) { + Ok(v) => println!("Dial returned {:?}", v), + Err(error) => { + println!("Dialing error! {}", error); + event_tx.send(NetworkPeerEvent::DialError { error: error.into() })?; + } + } + } } } - + // Process events event = self.swarm.select_next_some() => { - process_swarm_event(&mut self.swarm, &mut self.to_bus_tx, event).await? + process_swarm_event(&mut self.swarm, &event_tx, event).await? } } } @@ -167,16 +199,39 @@ fn create_mdns_kad_behaviour( async fn process_swarm_event( swarm: &mut Swarm, - to_bus_tx: &mut Sender>, + event_tx: &broadcast::Sender, event: SwarmEvent, ) -> Result<()> { match event { - SwarmEvent::ConnectionEstablished { peer_id, .. } => { + SwarmEvent::ConnectionEstablished { + peer_id, + endpoint, + connection_id, + .. + } => { info!("Connected to {peer_id}"); + let remote_addr = endpoint.get_remote_address().clone(); + swarm + .behaviour_mut() + .kademlia + .add_address(&peer_id, remote_addr.clone()); + + info!("Added address to kademlia {}", remote_addr); + swarm.behaviour_mut().gossipsub.add_explicit_peer(&peer_id); + info!("Added peer to gossipsub {}", remote_addr); + event_tx.send(NetworkPeerEvent::ConnectionEstablished { connection_id })?; } - SwarmEvent::OutgoingConnectionError { peer_id, error, .. } => { - warn!("Failed to dial {peer_id:?}: {error}"); + SwarmEvent::OutgoingConnectionError { + peer_id, + error, + connection_id, + } => { + info!("Failed to dial {peer_id:?}: {error}"); + event_tx.send(NetworkPeerEvent::OutgoingConnectionError { + connection_id, + error: Arc::new(error), + })?; } SwarmEvent::IncomingConnectionError { error, .. } => { @@ -210,8 +265,7 @@ async fn process_swarm_event( message, })) => { trace!("Got message with id: {id} from peer: {peer_id}",); - trace!("{:?}", message); - to_bus_tx.send(message.data).await?; + event_tx.send(NetworkPeerEvent::GossipData(message.data))?; } SwarmEvent::NewListenAddr { address, .. } => { warn!("Local node is listening on {address}"); diff --git a/packages/ciphernode/net/src/retry.rs b/packages/ciphernode/net/src/retry.rs new file mode 100644 index 00000000..55ab151e --- /dev/null +++ b/packages/ciphernode/net/src/retry.rs @@ -0,0 +1,71 @@ +use std::{future::Future, time::Duration}; +use anyhow::Result; +use tokio::time::sleep; + +pub enum RetryError { + Failure(anyhow::Error), + Retry(anyhow::Error), +} + +pub fn to_retry(e: impl Into) -> RetryError { + RetryError::Retry(e.into()) +} + +pub const BACKOFF_DELAY: u64 = 500; +pub const BACKOFF_MAX_RETRIES: u32 = 10; + +/// Retries an async operation with exponential backoff +/// +/// # Arguments +/// * `operation` - Async function to retry +/// * `max_attempts` - Maximum number of retry attempts +/// * `initial_delay_ms` - Initial delay between retries in milliseconds +/// +/// # Returns +/// * `Result<()>` - Ok if the operation succeeded, Err if all retries failed +pub async fn retry_with_backoff( + operation: F, + max_attempts: u32, + initial_delay_ms: u64, +) -> Result<()> +where + F: Fn() -> Fut, + Fut: Future>, +{ + let mut current_attempt = 1; + let mut delay_ms = initial_delay_ms; + + loop { + match operation().await { + Ok(_) => return Ok(()), + Err(re) => { + match re { + RetryError::Retry(e) => { + if current_attempt >= max_attempts { + return Err(anyhow::anyhow!( + "Operation failed after {} attempts. Last error: {}", + max_attempts, + e + )); + } + + println!( + "Attempt {}/{} failed, retrying in {}ms: {}", + current_attempt, max_attempts, delay_ms, e + ); + + sleep(Duration::from_millis(delay_ms)).await; + current_attempt += 1; + delay_ms *= 2; // Exponential backoff + } + RetryError::Failure(e) => { + println!("FAILURE!: returning to caller."); + return Err(e); + } + } + } + } + } +} + + From 1c0162c5536156f0b94cba316deb4b2b4f451031 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 19 Dec 2024 18:02:28 +1100 Subject: [PATCH 16/36] Redundant comment --- packages/ciphernode/net/src/events.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/ciphernode/net/src/events.rs b/packages/ciphernode/net/src/events.rs index 7719279e..06f641df 100644 --- a/packages/ciphernode/net/src/events.rs +++ b/packages/ciphernode/net/src/events.rs @@ -22,8 +22,6 @@ pub enum NetworkPeerCommand { pub enum NetworkPeerEvent { GossipData(Vec), GossipPublishError { - // TODO: return an error here? DialError is not Clonable so we have - // avoided passing it on correlation_id: CorrelationId, error: Arc, }, From e364201e2ecdec711ec868a9c914f88842cf41ea Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 19 Dec 2024 21:57:34 +1100 Subject: [PATCH 17/36] Setup deploy folder --- .deploy/.env.example | 7 ----- .deploy/cn1.yaml | 5 +++ .deploy/cn2.yaml | 5 +++ .deploy/cn3.yaml | 5 +++ .deploy/deploy.sh | 2 +- .deploy/docker-compose.yml | 51 +++++-------------------------- .deploy/inspect.sh | 48 +++++++++++++++++++++++++++++ .deploy/swarm_deployment.md | 7 ++++- packages/ciphernode/.dockerignore | 13 +------- packages/ciphernode/Dockerfile | 29 ++++++++++++++++-- 10 files changed, 104 insertions(+), 68 deletions(-) create mode 100755 .deploy/inspect.sh diff --git a/.deploy/.env.example b/.deploy/.env.example index 56904289..14723a66 100644 --- a/.deploy/.env.example +++ b/.deploy/.env.example @@ -1,11 +1,4 @@ -TAG=latest -AGG_ADDRESS=0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199 -CN1_ADDRESS=0xbDA5747bFD65F08deb54cb465eB87D40e51B197E -CN2_ADDRESS=0xdD2FD4581271e230360230F9337D5c0430Bf44C0 -CN3_ADDRESS=0x2546BcD3c84621e976D8185a91A922aE77ECEc30 - RPC_URL=wss://eth-sepolia.g.alchemy.com/v2/API_KEY - SEPOLIA_ENCLAVE_ADDRESS=0xCe087F31e20E2F76b6544A2E4A74D4557C8fDf77 SEPOLIA_CIPHERNODE_REGISTRY_ADDRESS=0x0952388f6028a9Eda93a5041a3B216Ea331d97Ab SEPOLIA_FILTER_REGISTRY=0xcBaCE7C360b606bb554345b20884A28e41436934 diff --git a/.deploy/cn1.yaml b/.deploy/cn1.yaml index 064cf8c7..aefdc431 100644 --- a/.deploy/cn1.yaml +++ b/.deploy/cn1.yaml @@ -1,6 +1,11 @@ address: "${ADDRESS}" quic_port: ${QUIC_PORT} enable_mdns: false +peers: + - "/dns4/cn1/udp/9091/quic-v1" + - "/dns4/cn1/udp/9092/quic-v1" + - "/dns4/cn1/udp/9093/quic-v1" + - "/dns4/cn1/udp/9094/quic-v1" chains: - name: "sepolia" rpc_url: "${RPC_URL}" diff --git a/.deploy/cn2.yaml b/.deploy/cn2.yaml index 064cf8c7..aefdc431 100644 --- a/.deploy/cn2.yaml +++ b/.deploy/cn2.yaml @@ -1,6 +1,11 @@ address: "${ADDRESS}" quic_port: ${QUIC_PORT} enable_mdns: false +peers: + - "/dns4/cn1/udp/9091/quic-v1" + - "/dns4/cn1/udp/9092/quic-v1" + - "/dns4/cn1/udp/9093/quic-v1" + - "/dns4/cn1/udp/9094/quic-v1" chains: - name: "sepolia" rpc_url: "${RPC_URL}" diff --git a/.deploy/cn3.yaml b/.deploy/cn3.yaml index 064cf8c7..aefdc431 100644 --- a/.deploy/cn3.yaml +++ b/.deploy/cn3.yaml @@ -1,6 +1,11 @@ address: "${ADDRESS}" quic_port: ${QUIC_PORT} enable_mdns: false +peers: + - "/dns4/cn1/udp/9091/quic-v1" + - "/dns4/cn1/udp/9092/quic-v1" + - "/dns4/cn1/udp/9093/quic-v1" + - "/dns4/cn1/udp/9094/quic-v1" chains: - name: "sepolia" rpc_url: "${RPC_URL}" diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh index 397c8ec8..75ff0f21 100755 --- a/.deploy/deploy.sh +++ b/.deploy/deploy.sh @@ -23,5 +23,5 @@ wait_removed() { stack_name=${1:-enclave} docker stack rm $stack_name wait_removed $stack_name -docker stack deploy -c docker-compose.yml --prune $stack_name +docker stack deploy -c ./.deploy/docker-compose.yml --prune $stack_name wait_ready $stack_name diff --git a/.deploy/docker-compose.yml b/.deploy/docker-compose.yml index 3a3d45e2..1085ce1d 100644 --- a/.deploy/docker-compose.yml +++ b/.deploy/docker-compose.yml @@ -1,6 +1,6 @@ services: cn1: - image: ghcr.io/gnosisguild/ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode volumes: - ./cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/enclave @@ -13,25 +13,14 @@ services: AGGREGATOR: "false" ADDRESS: "0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" QUIC_PORT: 9091 - ports: - - "9091:9091/udp" - - "9091:9091/tcp" deploy: replicas: 1 endpoint_mode: dnsrr - update_config: - parallelism: 1 - order: stop-first - failure_action: rollback - restart_policy: - condition: any networks: - global-network cn2: - image: ghcr.io/gnosisguild/ciphernode:latest - depends_on: - - cn1 + image: ghcr.io/gnosisguild/ciphernode volumes: - ./cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn2-data:/home/ciphernode/.local/share/enclave @@ -44,24 +33,14 @@ services: AGGREGATOR: "false" ADDRESS: "0xdD2FD4581271e230360230F9337D5c0430Bf44C0" QUIC_PORT: 9092 - ports: - - "9092:9092/udp" - - "9092:9092/tcp" deploy: replicas: 1 - update_config: - parallelism: 1 - order: stop-first - failure_action: rollback - restart_policy: - condition: any + endpoint_mode: dnsrr networks: - global-network cn3: - image: ghcr.io/gnosisguild/ciphernode:latest - depends_on: - - cn1 + image: ghcr.io/gnosisguild/ciphernode volumes: - ./cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn3-data:/home/ciphernode/.local/share/enclave @@ -74,22 +53,14 @@ services: AGGREGATOR: "false" ADDRESS: "0x2546BcD3c84621e976D8185a91A922aE77ECEc30" QUIC_PORT: 9093 - ports: - - "9093:9093/udp" - - "9093:9093/tcp" deploy: replicas: 1 - update_config: - parallelism: 1 - order: stop-first - failure_action: rollback - restart_policy: - condition: any + endpoint_mode: dnsrr networks: - global-network aggregator: - image: ghcr.io/gnosisguild/ciphernode:latest + image: ghcr.io/gnosisguild/ciphernode depends_on: - cn1 volumes: @@ -104,17 +75,9 @@ services: AGGREGATOR: "true" ADDRESS: "0x8626a6940E2eb28930eFb4CeF49B2d1F2C9C1199" QUIC_PORT: 9094 - ports: - - "9094:9094/udp" - - "9094:9094/tcp" deploy: replicas: 1 - update_config: - parallelism: 1 - order: stop-first - failure_action: rollback - restart_policy: - condition: any + endpoint_mode: dnsrr networks: - global-network diff --git a/.deploy/inspect.sh b/.deploy/inspect.sh new file mode 100755 index 00000000..20959b5a --- /dev/null +++ b/.deploy/inspect.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +get_logs_by_version() { + local SERVICE_NAME=$1 + + # Get current version number + CURRENT_VERSION=$(docker service inspect --format '{{.Version.Index}}' $SERVICE_NAME) + + # Get all tasks with this version + TASK_IDS=$(docker service ps --filter "desired-state=running" \ + --format '{{.ID}}' $SERVICE_NAME) + + # Get logs from these specific tasks + for TASK_ID in $TASK_IDS; do + docker service logs --raw "$TASK_ID" + done +} + +echo "" +echo "=================================" +echo " CN1 " +echo "=================================" + +get_logs_by_version enclave_cn1 + + +echo "" +echo "=================================" +echo " CN2 " +echo "=================================" + +get_logs_by_version enclave_cn2 + + +echo "" +echo "=================================" +echo " CN3 " +echo "=================================" + +get_logs_by_version enclave_cn3 + + +echo "" +echo "=================================" +echo " AGG " +echo "=================================" + +get_logs_by_version enclave_aggregator diff --git a/.deploy/swarm_deployment.md b/.deploy/swarm_deployment.md index 93a922fc..2bf09371 100644 --- a/.deploy/swarm_deployment.md +++ b/.deploy/swarm_deployment.md @@ -57,12 +57,17 @@ If you get an error about not being able to choose between IP addresses choose t docker swarm init --advertise-addr 10.49.x.x ``` +# Build the dockerfile image + +``` +./.deploy/build.sh +``` # Deploy a version to the stack To deploy ``` -.deploy/deploy.sh +./.deploy/deploy.sh ``` diff --git a/packages/ciphernode/.dockerignore b/packages/ciphernode/.dockerignore index 7d50aa6c..521131cf 100644 --- a/packages/ciphernode/.dockerignore +++ b/packages/ciphernode/.dockerignore @@ -6,18 +6,7 @@ !Cargo.toml !Cargo.lock -# Allow core crate -!core/ -!core/Cargo.toml -!core/src/**/*.rs - -# net -!net/ -!net/Cargo.toml -!net/src/**/*.rs - - -# Allow all other workspace members (adjust paths as needed) +# Allow all other workspace members !*/Cargo.toml !*/src/**/*.rs diff --git a/packages/ciphernode/Dockerfile b/packages/ciphernode/Dockerfile index 3f559d95..330b28bc 100644 --- a/packages/ciphernode/Dockerfile +++ b/packages/ciphernode/Dockerfile @@ -7,11 +7,35 @@ RUN yarn install && yarn compile # Build stage FROM rust:1.81 AS ciphernode-builder +# Force incremental +ENV CARGO_INCREMENTAL=1 +ENV RUSTC_FORCE_INCREMENTAL=1 +ENV CARGO_BUILD_JOBS=8 + # Create build directory WORKDIR /build/packages/ciphernode -COPY ./packages/ciphernode ./ COPY --from=evm-builder /build/packages/evm/artifacts ../evm/artifacts COPY --from=evm-builder /build/packages/evm/deployments ../evm/deployments + +# Copy workpace Cargo.toml +COPY ./packages/ciphernode/Cargo.toml ./Cargo.toml +COPY ./packages/ciphernode/Cargo.lock ./Cargo.lock +COPY ./packages/ciphernode/*/Cargo.toml ./ + +# Build all dependencies and add them to the build cache +RUN mkdir -p src && \ + echo "fn main() {}" > src/main.rs && \ + for d in ./*/ ; do \ + if [ -f "$d/Cargo.toml" ]; then \ + mkdir -p "$d/src" && \ + echo "fn main() {}" > "$d/src/lib.rs"; \ + fi \ + done + +RUN cargo build --release + +COPY ./packages/ciphernode . + RUN cargo build --release # Runtime stage @@ -44,5 +68,4 @@ ENV DATA_DIR=/home/ciphernode/.local/share/enclave ENV RUST_LOG=info # Add entrypoint script - -ENTRYPOINT ["ciphernode-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["ciphernode-entrypoint.sh"] From f68c9c5ebf69a6d0061b8fb222c7fd76a60ba455 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 19 Dec 2024 21:59:08 +1100 Subject: [PATCH 18/36] Setup correct peers --- .deploy/agg.yaml | 5 +++++ .deploy/cn2.yaml | 6 +++--- .deploy/cn3.yaml | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.deploy/agg.yaml b/.deploy/agg.yaml index 064cf8c7..8321ce76 100644 --- a/.deploy/agg.yaml +++ b/.deploy/agg.yaml @@ -1,6 +1,11 @@ address: "${ADDRESS}" quic_port: ${QUIC_PORT} enable_mdns: false +peers: + - "/dns4/cn1/udp/9091/quic-v1" + - "/dns4/cn2/udp/9092/quic-v1" + - "/dns4/cn3/udp/9093/quic-v1" + - "/dns4/aggregator/udp/9094/quic-v1" chains: - name: "sepolia" rpc_url: "${RPC_URL}" diff --git a/.deploy/cn2.yaml b/.deploy/cn2.yaml index aefdc431..8321ce76 100644 --- a/.deploy/cn2.yaml +++ b/.deploy/cn2.yaml @@ -3,9 +3,9 @@ quic_port: ${QUIC_PORT} enable_mdns: false peers: - "/dns4/cn1/udp/9091/quic-v1" - - "/dns4/cn1/udp/9092/quic-v1" - - "/dns4/cn1/udp/9093/quic-v1" - - "/dns4/cn1/udp/9094/quic-v1" + - "/dns4/cn2/udp/9092/quic-v1" + - "/dns4/cn3/udp/9093/quic-v1" + - "/dns4/aggregator/udp/9094/quic-v1" chains: - name: "sepolia" rpc_url: "${RPC_URL}" diff --git a/.deploy/cn3.yaml b/.deploy/cn3.yaml index aefdc431..8321ce76 100644 --- a/.deploy/cn3.yaml +++ b/.deploy/cn3.yaml @@ -3,9 +3,9 @@ quic_port: ${QUIC_PORT} enable_mdns: false peers: - "/dns4/cn1/udp/9091/quic-v1" - - "/dns4/cn1/udp/9092/quic-v1" - - "/dns4/cn1/udp/9093/quic-v1" - - "/dns4/cn1/udp/9094/quic-v1" + - "/dns4/cn2/udp/9092/quic-v1" + - "/dns4/cn3/udp/9093/quic-v1" + - "/dns4/aggregator/udp/9094/quic-v1" chains: - name: "sepolia" rpc_url: "${RPC_URL}" From 5491353f8a55c429e2e61bf1cd5046c86faae23e Mon Sep 17 00:00:00 2001 From: Hamza Khalid <36852564+hmzakhalid@users.noreply.github.com> Date: Fri, 20 Dec 2024 02:35:53 +0500 Subject: [PATCH 19/36] Add Network keypair to enclave init. (#209) * update cargo.toml * add net-pk to cli * update validation * generate and purge keypair * set network key * update repositories * update ci tests * formatting * Zeroize bytes --- packages/ciphernode/Cargo.lock | 1 + packages/ciphernode/enclave/Cargo.toml | 1 + .../ciphernode/enclave/src/commands/init.rs | 43 ++++++++------ .../ciphernode/enclave/src/commands/mod.rs | 8 +++ .../enclave/src/commands/net/generate.rs | 37 ++++++++++++ .../enclave/src/commands/net/mod.rs | 15 ++++- .../enclave/src/commands/net/purge.rs | 2 +- .../enclave/src/commands/net/set.rs | 59 +++++++++++++++++++ .../enclave/src/commands/password/mod.rs | 2 +- .../enclave/src/commands/wallet/set.rs | 31 ++-------- packages/ciphernode/enclave/src/main.rs | 18 +++++- .../ciphernode/enclave_node/src/aggregator.rs | 2 +- .../ciphernode/enclave_node/src/ciphernode.rs | 2 +- .../ciphernode/net/src/network_manager.rs | 33 +++++------ .../ciphernode/router/src/repositories.rs | 4 +- tests/basic_integration/base.sh | 7 +++ tests/basic_integration/fns.sh | 16 +++++ tests/basic_integration/persist.sh | 7 +++ 18 files changed, 215 insertions(+), 73 deletions(-) create mode 100644 packages/ciphernode/enclave/src/commands/net/generate.rs create mode 100644 packages/ciphernode/enclave/src/commands/net/set.rs diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 0684a617..9e9fc345 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -2171,6 +2171,7 @@ dependencies = [ "enclave-core", "enclave_node", "hex", + "libp2p", "once_cell", "phf", "router", diff --git a/packages/ciphernode/enclave/Cargo.toml b/packages/ciphernode/enclave/Cargo.toml index d38cf579..284a4d12 100644 --- a/packages/ciphernode/enclave/Cargo.toml +++ b/packages/ciphernode/enclave/Cargo.toml @@ -19,6 +19,7 @@ dialoguer = "0.11.0" enclave-core = { path = "../core" } enclave_node = { path = "../enclave_node" } hex = { workspace = true } +libp2p = { workspace = true } once_cell = "1.20.2" router = { path = "../router" } tokio = { workspace = true } diff --git a/packages/ciphernode/enclave/src/commands/init.rs b/packages/ciphernode/enclave/src/commands/init.rs index 65b16658..dbfe9952 100644 --- a/packages/ciphernode/enclave/src/commands/init.rs +++ b/packages/ciphernode/enclave/src/commands/init.rs @@ -1,7 +1,9 @@ -use crate::commands::password::{self, PasswordCommands}; -use anyhow::anyhow; -use anyhow::bail; -use anyhow::Result; +use crate::commands::{ + net, + password::{self, PasswordCommands}, +}; +use alloy::primitives::Address; +use anyhow::{anyhow, bail, Result}; use config::load_config; use config::RPC; use dialoguer::{theme::ColorfulTheme, Input}; @@ -27,21 +29,10 @@ fn validate_rpc_url(url: &String) -> Result<()> { } fn validate_eth_address(address: &String) -> Result<()> { - if address.is_empty() { - return Ok(()); + match Address::parse_checksummed(address, None) { + Ok(_) => Ok(()), + Err(e) => bail!("Invalid Ethereum address: {}", e), } - if !address.starts_with("0x") { - bail!("Address must start with '0x'") - } - if address.len() != 42 { - bail!("Address must be 42 characters long (including '0x')") - } - for c in address[2..].chars() { - if !c.is_ascii_hexdigit() { - bail!("Address must contain only hexadecimal characters") - } - } - Ok(()) } #[instrument(name = "app", skip_all, fields(id = get_tag()))] @@ -50,6 +41,8 @@ pub async fn execute( eth_address: Option, password: Option, skip_eth: bool, + net_keypair: Option, + generate_net_keypair: bool, ) -> Result<()> { let rpc_url = match rpc_url { Some(url) => { @@ -133,10 +126,22 @@ chains: password, overwrite: true, }, - config, + &config, ) .await?; + if generate_net_keypair { + net::execute(net::NetCommands::GenerateKey, &config).await?; + } else { + net::execute( + net::NetCommands::SetKey { + net_keypair: net_keypair, + }, + &config, + ) + .await?; + } + println!("Enclave configuration successfully created!"); println!("You can start your node using `enclave start`"); diff --git a/packages/ciphernode/enclave/src/commands/mod.rs b/packages/ciphernode/enclave/src/commands/mod.rs index ec26de49..92cc422b 100644 --- a/packages/ciphernode/enclave/src/commands/mod.rs +++ b/packages/ciphernode/enclave/src/commands/mod.rs @@ -56,5 +56,13 @@ pub enum Commands { /// Skip asking for eth #[arg(long = "skip-eth")] skip_eth: bool, + + /// The network private key (ed25519) + #[arg(long = "net-keypair")] + net_keypair: Option, + + /// Generate a new network keypair + #[arg(long = "generate-net-keypair")] + generate_net_keypair: bool, }, } diff --git a/packages/ciphernode/enclave/src/commands/net/generate.rs b/packages/ciphernode/enclave/src/commands/net/generate.rs new file mode 100644 index 00000000..1e747379 --- /dev/null +++ b/packages/ciphernode/enclave/src/commands/net/generate.rs @@ -0,0 +1,37 @@ +use actix::Actor; +use anyhow::{bail, Result}; +use cipher::Cipher; +use config::AppConfig; +use enclave_core::{EventBus, GetErrors}; +use enclave_node::get_repositories; +use libp2p::identity::Keypair; +use zeroize::Zeroize; + +pub async fn execute(config: &AppConfig) -> Result<()> { + let kp = Keypair::generate_ed25519(); + println!( + "Generated new keypair with peer ID: {}", + kp.public().to_peer_id() + ); + let mut bytes = kp.try_into_ed25519()?.to_bytes().to_vec(); + let cipher = Cipher::from_config(config).await?; + let encrypted = cipher.encrypt_data(&mut bytes.clone())?; + let bus = EventBus::new(true).start(); + let repositories = get_repositories(&config, &bus)?; + bytes.zeroize(); + + // NOTE: We are writing an encrypted string here + repositories.libp2p_keypair().write(&encrypted); + + let errors = bus.send(GetErrors).await?; + if errors.len() > 0 { + for error in errors.iter() { + println!("{error}"); + } + bail!("There were errors generating the network keypair") + } + + println!("Network keypair has been successfully generated and encrypted."); + + Ok(()) +} diff --git a/packages/ciphernode/enclave/src/commands/net/mod.rs b/packages/ciphernode/enclave/src/commands/net/mod.rs index 913cf69e..37b6b11e 100644 --- a/packages/ciphernode/enclave/src/commands/net/mod.rs +++ b/packages/ciphernode/enclave/src/commands/net/mod.rs @@ -1,4 +1,6 @@ +mod generate; mod purge; +mod set; use anyhow::*; use clap::Subcommand; use config::AppConfig; @@ -7,11 +9,22 @@ use config::AppConfig; pub enum NetCommands { /// Purge the current peer ID from the database. PurgeId, + + /// Generate a new network keypair + GenerateKey, + + /// Set the network private key + SetKey { + #[arg(long = "net-keypair")] + net_keypair: Option, + }, } -pub async fn execute(command: NetCommands, config: AppConfig) -> Result<()> { +pub async fn execute(command: NetCommands, config: &AppConfig) -> Result<()> { match command { NetCommands::PurgeId => purge::execute(&config).await?, + NetCommands::GenerateKey => generate::execute(&config).await?, + NetCommands::SetKey { net_keypair } => set::execute(&config, net_keypair).await?, }; Ok(()) diff --git a/packages/ciphernode/enclave/src/commands/net/purge.rs b/packages/ciphernode/enclave/src/commands/net/purge.rs index 3c2b3aae..520a75d5 100644 --- a/packages/ciphernode/enclave/src/commands/net/purge.rs +++ b/packages/ciphernode/enclave/src/commands/net/purge.rs @@ -7,7 +7,7 @@ use enclave_node::get_repositories; pub async fn execute(config: &AppConfig) -> Result<()> { let bus = EventBus::new(true).start(); let repositories = get_repositories(&config, &bus)?; - repositories.libp2pid().clear(); + repositories.libp2p_keypair().clear(); println!("Peer ID has been purged. A new Peer ID will be generated upon restart."); Ok(()) } diff --git a/packages/ciphernode/enclave/src/commands/net/set.rs b/packages/ciphernode/enclave/src/commands/net/set.rs new file mode 100644 index 00000000..853ecde0 --- /dev/null +++ b/packages/ciphernode/enclave/src/commands/net/set.rs @@ -0,0 +1,59 @@ +use actix::Actor; +use alloy::primitives::hex; +use anyhow::{bail, Result}; +use cipher::Cipher; +use config::AppConfig; +use dialoguer::{theme::ColorfulTheme, Password}; +use enclave_core::{EventBus, GetErrors}; +use enclave_node::get_repositories; +use libp2p::identity::Keypair; + +pub fn create_keypair(input: &String) -> Result { + match hex::check(input) { + Ok(()) => match Keypair::ed25519_from_bytes(hex::decode(input)?) { + Ok(kp) => Ok(kp), + Err(e) => bail!("Invalid network keypair: {}", e), + }, + Err(e) => bail!("Error decoding network keypair: {}", e), + } +} + +fn validate_keypair_input(input: &String) -> Result<()> { + create_keypair(input).map(|_| ()) +} + +pub async fn execute(config: &AppConfig, net_keypair: Option) -> Result<()> { + let input = if let Some(net_keypair) = net_keypair { + let kp = create_keypair(&net_keypair)?; + kp.try_into_ed25519()?.to_bytes().to_vec() + } else { + let kp = Password::with_theme(&ColorfulTheme::default()) + .with_prompt("Enter your network private key") + .validate_with(validate_keypair_input) + .interact()? + .trim() + .to_string(); + let kp = create_keypair(&kp)?; + kp.try_into_ed25519()?.to_bytes().to_vec() + }; + + let cipher = Cipher::from_config(config).await?; + let encrypted = cipher.encrypt_data(&mut input.clone())?; + let bus = EventBus::new(true).start(); + let repositories = get_repositories(&config, &bus)?; + + // NOTE: We are writing an encrypted string here + repositories.libp2p_keypair().write(&encrypted); + + let errors = bus.send(GetErrors).await?; + if errors.len() > 0 { + for error in errors.iter() { + println!("{error}"); + } + bail!("There were errors setting the network keypair") + } + + println!("Network keypair has been successfully encrypted."); + + Ok(()) +} diff --git a/packages/ciphernode/enclave/src/commands/password/mod.rs b/packages/ciphernode/enclave/src/commands/password/mod.rs index 5ebaf984..fe6259d1 100644 --- a/packages/ciphernode/enclave/src/commands/password/mod.rs +++ b/packages/ciphernode/enclave/src/commands/password/mod.rs @@ -30,7 +30,7 @@ pub enum PasswordCommands { }, } -pub async fn execute(command: PasswordCommands, config: AppConfig) -> Result<()> { +pub async fn execute(command: PasswordCommands, config: &AppConfig) -> Result<()> { match command { PasswordCommands::Create { password, diff --git a/packages/ciphernode/enclave/src/commands/wallet/set.rs b/packages/ciphernode/enclave/src/commands/wallet/set.rs index 0e1faac5..fcafd38e 100644 --- a/packages/ciphernode/enclave/src/commands/wallet/set.rs +++ b/packages/ciphernode/enclave/src/commands/wallet/set.rs @@ -1,4 +1,5 @@ use actix::Actor; +use alloy::{hex::FromHex, primitives::FixedBytes, signers::local::PrivateKeySigner}; use anyhow::{anyhow, bail, Result}; use cipher::Cipher; use config::AppConfig; @@ -7,33 +8,13 @@ use enclave_core::{EventBus, GetErrors}; use enclave_node::get_repositories; pub fn validate_private_key(input: &String) -> Result<()> { - // Require 0x prefix - if !input.starts_with("0x") { - return Err(anyhow!( - "Invalid private key format: must start with '0x' prefix" - )); - } - - // Remove 0x prefix - let key = &input[2..]; - - // Check length - if key.len() != 64 { - return Err(anyhow!( - "Invalid private key length: {}. Expected 64 characters after '0x' prefix", - key.len() - )); - } - - // Validate hex characters and convert to bytes - let _ = (0..key.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&key[i..i + 2], 16)) - .collect::, _>>() - .map_err(|e| anyhow!("Invalid hex character: {}", e))?; - + let bytes = + FixedBytes::<32>::from_hex(input).map_err(|e| anyhow!("Invalid private key: {}", e))?; + let _ = + PrivateKeySigner::from_bytes(&bytes).map_err(|e| anyhow!("Invalid private key: {}", e))?; Ok(()) } + pub async fn execute(config: &AppConfig, private_key: Option) -> Result<()> { let input = if let Some(private_key) = private_key { validate_private_key(&private_key)?; diff --git a/packages/ciphernode/enclave/src/main.rs b/packages/ciphernode/enclave/src/main.rs index 808b7629..d344ecf5 100644 --- a/packages/ciphernode/enclave/src/main.rs +++ b/packages/ciphernode/enclave/src/main.rs @@ -55,11 +55,23 @@ impl Cli { eth_address, password, skip_eth, - } => init::execute(rpc_url, eth_address, password, skip_eth).await?, - Commands::Password { command } => password::execute(command, config).await?, + net_keypair, + generate_net_keypair, + } => { + init::execute( + rpc_url, + eth_address, + password, + skip_eth, + net_keypair, + generate_net_keypair, + ) + .await? + } + Commands::Password { command } => password::execute(command, &config).await?, Commands::Aggregator { command } => aggregator::execute(command, config).await?, Commands::Wallet { command } => wallet::execute(command, config).await?, - Commands::Net { command } => net::execute(command, config).await?, + Commands::Net { command } => net::execute(command, &config).await?, } Ok(()) diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index b215ce8f..e9fbbf6a 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -83,7 +83,7 @@ pub async fn setup_aggregator( &cipher, config.quic_port(), config.enable_mdns(), - repositories.libp2pid(), + repositories.libp2p_keypair(), ) .await?; diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 5205d9be..bf71c73c 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -72,7 +72,7 @@ pub async fn setup_ciphernode( &cipher, config.quic_port(), config.enable_mdns(), - repositories.libp2pid(), + repositories.libp2p_keypair(), ) .await?; diff --git a/packages/ciphernode/net/src/network_manager.rs b/packages/ciphernode/net/src/network_manager.rs index 8969c908..777909f3 100644 --- a/packages/ciphernode/net/src/network_manager.rs +++ b/packages/ciphernode/net/src/network_manager.rs @@ -2,8 +2,7 @@ use crate::NetworkPeer; /// Actor for connecting to an libp2p client via it's mpsc channel interface /// This Actor should be responsible for use actix::prelude::*; -use anyhow::anyhow; -use anyhow::Result; +use anyhow::{anyhow, bail, Result}; use cipher::Cipher; use data::Repository; use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; @@ -75,25 +74,18 @@ impl NetworkManager { enable_mdns: bool, repository: Repository>, ) -> Result<(Addr, tokio::task::JoinHandle>, String)> { - info!("Reading from repository"); - let mut bytes = if let Some(bytes) = repository.read().await? { - let decrypted = cipher.decrypt_data(&bytes)?; - info!("Found keypair in repository"); - decrypted - } else { - let kp = libp2p::identity::Keypair::generate_ed25519(); - info!("Generated new keypair {}", kp.public().to_peer_id()); - let innerkp = kp.try_into_ed25519()?; - let bytes = innerkp.to_bytes().to_vec(); - - // We need to clone here so that returned bytes are not zeroized - repository.write(&cipher.encrypt_data(&mut bytes.clone())?); - info!("Saved new keypair to repository"); - bytes + // Get existing keypair or generate a new one + let mut bytes = match repository.read().await? { + Some(bytes) => { + info!("Found keypair in repository"); + cipher.decrypt_data(&bytes)? + } + None => bail!("No network keypair found in repository, please generate a new one using `enclave net generate-key`"), }; - let ed25519_keypair = ed25519::Keypair::try_from_bytes(&mut bytes)?; - let keypair: libp2p::identity::Keypair = ed25519_keypair.try_into()?; + // Create peer from keypair + let keypair: libp2p::identity::Keypair = + ed25519::Keypair::try_from_bytes(&mut bytes)?.try_into()?; let mut peer = NetworkPeer::new( &keypair, peers, @@ -101,9 +93,12 @@ impl NetworkManager { "tmp-enclave-gossip-topic", enable_mdns, )?; + + // Setup and start network manager let rx = peer.rx().ok_or(anyhow!("Peer rx already taken"))?; let p2p_addr = NetworkManager::setup(bus, peer.tx(), rx); let handle = tokio::spawn(async move { Ok(peer.start().await?) }); + Ok((p2p_addr, handle, keypair.public().to_peer_id().to_string())) } } diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs index bf785d21..d9ab879c 100644 --- a/packages/ciphernode/router/src/repositories.rs +++ b/packages/ciphernode/router/src/repositories.rs @@ -74,8 +74,8 @@ impl Repositories { Repository::new(self.store.scope(format!("//eth_private_key"))) } - pub fn libp2pid(&self) -> Repository> { - Repository::new(self.store.scope(format!("//libp2pid"))) + pub fn libp2p_keypair(&self) -> Repository> { + Repository::new(self.store.scope(format!("//libp2p/keypair"))) } pub fn enclave_sol_reader(&self, chain_id: u64) -> Repository { diff --git a/tests/basic_integration/base.sh b/tests/basic_integration/base.sh index 8765b7c5..43485526 100755 --- a/tests/basic_integration/base.sh +++ b/tests/basic_integration/base.sh @@ -24,6 +24,13 @@ set_password cn4 "$CIPHERNODE_SECRET" set_password ag "$CIPHERNODE_SECRET" set_private_key ag "$PRIVATE_KEY" +# Set the network private key for all ciphernodes +set_network_private_key cn1 "$NETWORK_PRIVATE_KEY_1" +set_network_private_key cn2 "$NETWORK_PRIVATE_KEY_2" +set_network_private_key cn3 "$NETWORK_PRIVATE_KEY_3" +set_network_private_key cn4 "$NETWORK_PRIVATE_KEY_4" +set_network_private_key ag "$NETWORK_PRIVATE_KEY_AG" + # Launch 4 ciphernodes launch_ciphernode cn1 launch_ciphernode cn2 diff --git a/tests/basic_integration/fns.sh b/tests/basic_integration/fns.sh index 720fcd80..be24e90a 100644 --- a/tests/basic_integration/fns.sh +++ b/tests/basic_integration/fns.sh @@ -16,6 +16,7 @@ fi RPC_URL="ws://localhost:8545" PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +NETWORK_PRIVATE_KEY_AG="0x51a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" CIPHERNODE_SECRET="We are the music makers and we are the dreamers of the dreams." # These contracts are based on the deterministic order of hardhat deploy @@ -31,6 +32,12 @@ CIPHERNODE_ADDRESS_2="0xbDA5747bFD65F08deb54cb465eB87D40e51B197E" CIPHERNODE_ADDRESS_3="0xdD2FD4581271e230360230F9337D5c0430Bf44C0" CIPHERNODE_ADDRESS_4="0x8626f6940E2eb28930eFb4CeF49B2d1F2C9C1199" +# These are the network private keys for the ciphernodes +NETWORK_PRIVATE_KEY_1="0x11a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" +NETWORK_PRIVATE_KEY_2="0x21a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" +NETWORK_PRIVATE_KEY_3="0x31a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" +NETWORK_PRIVATE_KEY_4="0x41a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" + # Function to clean up background processes cleanup() { @@ -102,6 +109,15 @@ set_private_key() { --private-key "$private_key" } +set_network_private_key() { + local name="$1" + local private_key="$2" + + yarn enclave net set-key \ + --config "$SCRIPT_DIR/lib/$name/config.yaml" \ + --net-keypair "$private_key" +} + launch_aggregator() { local name="$1" heading "Launch aggregator $name" diff --git a/tests/basic_integration/persist.sh b/tests/basic_integration/persist.sh index d4f4f489..6372e6ee 100755 --- a/tests/basic_integration/persist.sh +++ b/tests/basic_integration/persist.sh @@ -24,6 +24,13 @@ set_password cn4 "$CIPHERNODE_SECRET" set_password ag "$CIPHERNODE_SECRET" set_private_key ag "$PRIVATE_KEY" +# Set the network private key for all ciphernodes +set_network_private_key cn1 "$NETWORK_PRIVATE_KEY_1" +set_network_private_key cn2 "$NETWORK_PRIVATE_KEY_2" +set_network_private_key cn3 "$NETWORK_PRIVATE_KEY_3" +set_network_private_key cn4 "$NETWORK_PRIVATE_KEY_4" +set_network_private_key ag "$NETWORK_PRIVATE_KEY_AG" + # Launch 4 ciphernodes launch_ciphernode cn1 launch_ciphernode cn2 From 28b44b8b996ebc120bc9d0040e93307266ed7f80 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 20 Dec 2024 11:01:12 +1100 Subject: [PATCH 20/36] Appears to be working --- .deploy/build-dummy.sh | 3 ++ .deploy/build.sh | 2 +- .deploy/deploy.sh | 50 ++++++++++++++++++++++++++-------- .deploy/docker-compose.yml | 8 +++--- .deploy/dummy.Dockerfile | 33 ++++++++++++++++++++++ .deploy/swarm_deployment.md | 31 +++++++++++++++++++-- packages/ciphernode/Dockerfile | 2 +- 7 files changed, 110 insertions(+), 19 deletions(-) create mode 100755 .deploy/build-dummy.sh create mode 100644 .deploy/dummy.Dockerfile diff --git a/.deploy/build-dummy.sh b/.deploy/build-dummy.sh new file mode 100755 index 00000000..a3453ffa --- /dev/null +++ b/.deploy/build-dummy.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker build -t norepo/dummy:666 -f ./.deploy/dummy.Dockerfile . diff --git a/.deploy/build.sh b/.deploy/build.sh index f3bdfbdc..a68aa2b2 100755 --- a/.deploy/build.sh +++ b/.deploy/build.sh @@ -9,4 +9,4 @@ time docker buildx build \ --cache-from=type=local,src=/tmp/docker-cache \ --cache-to=type=local,dest=/tmp/docker-cache \ --load \ - -t ghcr.io/gnosisguild/ciphernode -f ./packages/ciphernode/Dockerfile . + -t ${1:-ghcr.io/gnosisguild/ciphernode} -f ./packages/ciphernode/Dockerfile . diff --git a/.deploy/deploy.sh b/.deploy/deploy.sh index 75ff0f21..feaaf918 100755 --- a/.deploy/deploy.sh +++ b/.deploy/deploy.sh @@ -1,27 +1,55 @@ #!/usr/bin/env bash +TIMESTAMP=$(date +%s) +RUN_FILE="./.deploy/tmp.docker-compose.${TIMESTAMP}.yml" +TEMPLATE_FILE="./.deploy/docker-compose.yml" + wait_ready() { - local stack_name="$1" - until [ "$(docker stack services $stack_name --format '{{.Replicas}}' | awk -F'/' '$1 != $2')" = "" ]; do + local STACK_NAME="$1" + until [ "$(docker stack services $STACK_NAME --format '{{.Replicas}}' | awk -F'/' '$1 != $2')" = "" ]; do printf "." sleep 1 done echo -ne "\r\033[K" - echo "Stack $stack_name is ready!" + echo "Stack $STACK_NAME is ready!" } wait_removed() { - local stack_name="$1" - while docker stack ps $stack_name >/dev/null 2>&1; do + local STACK_NAME="$1" + while docker stack ps $STACK_NAME >/dev/null 2>&1; do printf "." sleep 1 done echo -ne "\r\033[K" - echo "Stack $stack_name is removed" + echo "Stack $STACK_NAME is removed" } -stack_name=${1:-enclave} -docker stack rm $stack_name -wait_removed $stack_name -docker stack deploy -c ./.deploy/docker-compose.yml --prune $stack_name -wait_ready $stack_name + +if [ -z "$1" ]; then + echo "Error: Please provide a stack name as an argument" + echo "Usage: $0 " + exit 1 +fi + +if [ -z "$2" ]; then + echo "Error: Please provide an image name as an argument" + echo "Usage: $0 " + exit 1 +fi + +# Check if docker-compose.yml exists +if [ ! -f "$TEMPLATE_FILE" ]; then + echo "Error: $TEMPLATE_FILE not found" + exit 1 +fi + +sed "s|{{IMAGE}}|$2|g" $TEMPLATE_FILE > "${RUN_FILE}" + +cat $RUN_FILE + +STACK_NAME=$1 +docker stack rm $STACK_NAME +wait_removed $STACK_NAME +docker stack deploy -c $RUN_FILE $STACK_NAME +wait_ready $STACK_NAME +rm ./.deploy/tmp.*.* diff --git a/.deploy/docker-compose.yml b/.deploy/docker-compose.yml index 1085ce1d..ef076133 100644 --- a/.deploy/docker-compose.yml +++ b/.deploy/docker-compose.yml @@ -1,6 +1,6 @@ services: cn1: - image: ghcr.io/gnosisguild/ciphernode + image: {{IMAGE}} volumes: - ./cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/enclave @@ -20,7 +20,7 @@ services: - global-network cn2: - image: ghcr.io/gnosisguild/ciphernode + image: {{IMAGE}} volumes: - ./cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn2-data:/home/ciphernode/.local/share/enclave @@ -40,7 +40,7 @@ services: - global-network cn3: - image: ghcr.io/gnosisguild/ciphernode + image: {{IMAGE}} volumes: - ./cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro - cn3-data:/home/ciphernode/.local/share/enclave @@ -60,7 +60,7 @@ services: - global-network aggregator: - image: ghcr.io/gnosisguild/ciphernode + image: {{IMAGE}} depends_on: - cn1 volumes: diff --git a/.deploy/dummy.Dockerfile b/.deploy/dummy.Dockerfile new file mode 100644 index 00000000..3cd39847 --- /dev/null +++ b/.deploy/dummy.Dockerfile @@ -0,0 +1,33 @@ +# THIS IS A DUMMY IMAGE FOR TESTING +# Recreate a runtime image to test env vars without long build times +FROM debian:stable-slim +RUN apt-get update && apt-get install -y --no-install-recommends iptables ca-certificates jq dnsutils iputils-ping && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Create non-root user +RUN useradd -m -u 1000 -s /bin/bash ciphernode + +# Create necessary directories with proper permissions +RUN mkdir -p /home/ciphernode/.config/enclave \ + /home/ciphernode/.local/share/enclave \ + /run/secrets && \ + chown -R ciphernode:ciphernode /home/ciphernode /run/secrets + +RUN echo "echo \"HELLO\" && sleep infinity" > /home/ciphernode/ciphernode-entrypoint.sh && \ + chmod +x /home/ciphernode/ciphernode-entrypoint.sh && \ + chown ciphernode:ciphernode /home/ciphernode/ciphernode-entrypoint.sh + +# Switch to non-root user +USER ciphernode +WORKDIR /home/ciphernode + +# Create the entrypoint script as root +RUN chmod +x /home/ciphernode/ciphernode-entrypoint.sh + +# Environment variables for configuration +ENV CONFIG_DIR=/home/ciphernode/.config/enclave +ENV DATA_DIR=/home/ciphernode/.local/share/enclave +ENV RUST_LOG=info + +# Add entrypoint script +ENTRYPOINT ["/bin/bash","/home/ciphernode/ciphernode-entrypoint.sh"] diff --git a/.deploy/swarm_deployment.md b/.deploy/swarm_deployment.md index 2bf09371..ca0d4e30 100644 --- a/.deploy/swarm_deployment.md +++ b/.deploy/swarm_deployment.md @@ -60,7 +60,7 @@ docker swarm init --advertise-addr 10.49.x.x # Build the dockerfile image ``` -./.deploy/build.sh +./.deploy/build.sh ghcr.io/gnosisguild/ciphernode ``` # Deploy a version to the stack @@ -68,6 +68,33 @@ docker swarm init --advertise-addr 10.49.x.x To deploy ``` -./.deploy/deploy.sh +./.deploy/deploy.sh enclave ghcr.io/gnosisguild/ciphernode:latest ``` +This will deploy the following services: + +``` +❯ docker service ls +ID NAME MODE REPLICAS IMAGE PORTS +tr44go8vevh1 enclave_aggregator replicated 1/1 ghcr.io/gnosisguild/ciphernode:latest +kdqktv85xcuv enclave_cn1 replicated 1/1 ghcr.io/gnosisguild/ciphernode:latest +nguul381w6mu enclave_cn2 replicated 1/1 ghcr.io/gnosisguild/ciphernode:latest +zgmwmv7cd63j enclave_cn3 replicated 1/1 ghcr.io/gnosisguild/ciphernode:latest +``` + +# Get the logs + +You can get the logs: + +``` +docker service logs enclave_cn1 +``` + +Notice the line: + +``` +enclave_cn2.1.zom4r645ophf@nixos | 2024-12-19T23:47:08.582536Z INFO enclave: COMPILATION ID: 'painfully_fluent_crane' +``` + +This can help you identify which compilation you are looking at. This works by generating a unique ID based on the complication time. + diff --git a/packages/ciphernode/Dockerfile b/packages/ciphernode/Dockerfile index 330b28bc..cb100076 100644 --- a/packages/ciphernode/Dockerfile +++ b/packages/ciphernode/Dockerfile @@ -42,7 +42,7 @@ RUN cargo build --release FROM debian:stable-slim # Install runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends iptables ca-certificates jq && \ +RUN apt-get update && apt-get install -y --no-install-recommends iptables dnsutils iputils-ping ca-certificates jq && \ apt-get clean && rm -rf /var/lib/apt/lists/* # Create non-root user From 16e17be55aabb754f0616e924947958b93528e3e Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 21 Dec 2024 13:49:21 +1100 Subject: [PATCH 21/36] Formatting --- packages/ciphernode/net/src/correlation_id.rs | 2 +- packages/ciphernode/net/src/lib.rs | 7 +++---- packages/ciphernode/net/src/network_manager.rs | 8 +------- packages/ciphernode/net/src/retry.rs | 4 +--- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/ciphernode/net/src/correlation_id.rs b/packages/ciphernode/net/src/correlation_id.rs index 3d14bcab..2d82c37e 100644 --- a/packages/ciphernode/net/src/correlation_id.rs +++ b/packages/ciphernode/net/src/correlation_id.rs @@ -5,7 +5,7 @@ use std::{ static NEXT_CORRELATION_ID: AtomicUsize = AtomicUsize::new(1); -#[derive(Debug,Clone)] +#[derive(Debug, Clone)] pub struct CorrelationId { id: usize, } diff --git a/packages/ciphernode/net/src/lib.rs b/packages/ciphernode/net/src/lib.rs index 394055bf..35d4f37e 100644 --- a/packages/ciphernode/net/src/lib.rs +++ b/packages/ciphernode/net/src/lib.rs @@ -1,13 +1,12 @@ #![crate_name = "net"] #![crate_type = "lib"] -mod network_manager; -mod network_peer; +pub mod correlation_id; mod dialer; pub mod events; +mod network_manager; +mod network_peer; mod retry; -pub mod correlation_id; pub use network_manager::*; pub use network_peer::*; - diff --git a/packages/ciphernode/net/src/network_manager.rs b/packages/ciphernode/net/src/network_manager.rs index bea06e38..5032c36d 100644 --- a/packages/ciphernode/net/src/network_manager.rs +++ b/packages/ciphernode/net/src/network_manager.rs @@ -103,13 +103,7 @@ impl NetworkManager { // Create peer from keypair let keypair: libp2p::identity::Keypair = ed25519::Keypair::try_from_bytes(&mut bytes)?.try_into()?; - let mut peer = NetworkPeer::new( - &keypair, - peers, - Some(quic_port), - topic, - enable_mdns, - )?; + let mut peer = NetworkPeer::new(&keypair, peers, Some(quic_port), topic, enable_mdns)?; // Setup and start network manager let rx = peer.rx(); diff --git a/packages/ciphernode/net/src/retry.rs b/packages/ciphernode/net/src/retry.rs index 55ab151e..5065c467 100644 --- a/packages/ciphernode/net/src/retry.rs +++ b/packages/ciphernode/net/src/retry.rs @@ -1,5 +1,5 @@ -use std::{future::Future, time::Duration}; use anyhow::Result; +use std::{future::Future, time::Duration}; use tokio::time::sleep; pub enum RetryError { @@ -67,5 +67,3 @@ where } } } - - From 89a2f7f16dc8f3a705490e9eab388af157346975 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 21 Dec 2024 14:49:34 +1100 Subject: [PATCH 22/36] Ensure tests work with changes to NetworkManager --- .../tests/test_aggregation_and_decryption.rs | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 2464eb50..b2011ed5 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -8,7 +8,7 @@ use enclave_core::{ }; use fhe::{setup_crp_params, ParamsWithCrp, SharedRng}; use logger::SimpleLogger; -use net::NetworkManager; +use net::{correlation_id::CorrelationId, events::NetworkPeerEvent, NetworkManager}; use router::{ CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, PlaintextAggregatorFeature, PublicKeyAggregatorFeature, RepositoriesFactory, @@ -27,8 +27,8 @@ use rand::Rng; use rand::SeedableRng; use rand_chacha::ChaCha20Rng; use std::{env, path::Path, sync::Arc, time::Duration}; -use tokio::sync::Mutex; -use tokio::{sync::mpsc::channel, time::sleep}; +use tokio::sync::{broadcast, Mutex}; +use tokio::{sync::mpsc, time::sleep}; // Simulating a local node type LocalCiphernodeTuple = ( @@ -465,24 +465,35 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { #[actix::test] async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { // Setup elements in test - let (tx, mut output) = channel(100); // Transmit byte events to the network - let (input, rx) = channel(100); // Receive byte events from the network + let (cmd_tx, mut cmd_rx) = mpsc::channel(100); // Transmit byte events to the network + let (event_tx, _) = broadcast::channel(100); // Receive byte events from the network let bus = EventBus::new(true).start(); - NetworkManager::setup(bus.clone(), tx.clone(), rx); + let event_rx = event_tx.subscribe(); + // Pas cmd and event channels to NetworkManager + NetworkManager::setup(bus.clone(), cmd_tx.clone(), event_rx, "my-topic"); // Capture messages from output on msgs vec let msgs: Arc>>> = Arc::new(Mutex::new(Vec::new())); let msgs_loop = msgs.clone(); tokio::spawn(async move { - while let Some(msg) = output.recv().await { - msgs_loop.lock().await.push(msg.clone()); - let _ = input.send(msg).await; - // loopback to simulate a rebroadcast message + // Pull events from command channel + while let Some(cmd) = cmd_rx.recv().await { + // If the command is a GossipPublish then extract it and save it whilst sending it to + // the event bus as if it was gossiped from the network and ended up as an external + // message this simulates a rebroadcast message + if let Some(msg) = match cmd { + net::events::NetworkPeerCommand::GossipPublish { data, .. } => Some(data), + _ => None, + } { + msgs_loop.lock().await.push(msg.clone()); + event_tx.send(NetworkPeerEvent::GossipData(msg))?; + } // if this manages to broadcast an event to the // event bus we will expect to see an extra event on - // the bus + // the bus but we don't because we handle this } + anyhow::Ok(()) }); let evt_1 = EnclaveEvent::from(PlaintextAggregated { @@ -532,10 +543,10 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { let seed = Seed(ChaCha20Rng::seed_from_u64(123).get_seed()); // Setup elements in test - let (tx, _) = channel(100); // Transmit byte events to the network - let (input, rx) = channel(100); // Receive byte events from the network + let (cmd_tx, _) = mpsc::channel(100); // Transmit byte events to the network + let (event_tx, event_rx) = broadcast::channel(100); // Receive byte events from the network let bus = EventBus::new(true).start(); - NetworkManager::setup(bus.clone(), tx.clone(), rx); + NetworkManager::setup(bus.clone(), cmd_tx.clone(), event_rx, "mytopic"); // Capture messages from output on msgs vec let event = EnclaveEvent::from(E3Requested { @@ -547,7 +558,7 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { }); // lets send an event from the network - let _ = input.send(event.to_bytes()?).await; + let _ = event_tx.send(NetworkPeerEvent::GossipData(event.to_bytes()?)); sleep(Duration::from_millis(1)).await; // need to push to next tick From e3458a227e02b6efd3d46918c409e34085299bc5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 22 Dec 2024 12:10:02 +1100 Subject: [PATCH 23/36] Get networking test to pass --- packages/ciphernode/net/src/bin/p2p_test.rs | 2 +- packages/ciphernode/net/tests/Dockerfile | 6 +-- .../ciphernode/net/tests/docker-compose.yaml | 38 ++++++++----------- packages/ciphernode/net/tests/entrypoint.sh | 4 -- 4 files changed, 19 insertions(+), 31 deletions(-) delete mode 100755 packages/ciphernode/net/tests/entrypoint.sh diff --git a/packages/ciphernode/net/src/bin/p2p_test.rs b/packages/ciphernode/net/src/bin/p2p_test.rs index bea5b098..b34b6ce9 100644 --- a/packages/ciphernode/net/src/bin/p2p_test.rs +++ b/packages/ciphernode/net/src/bin/p2p_test.rs @@ -20,7 +20,7 @@ async fn main() -> Result<()> { .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) .with(tracing_subscriber::fmt::layer()) .init(); - let name = env::args().nth(1).expect("need name"); + let name = env::args().nth(2).expect("need name"); let topic = "test-topic"; println!("{} starting up", name); diff --git a/packages/ciphernode/net/tests/Dockerfile b/packages/ciphernode/net/tests/Dockerfile index d983a17d..4dd0812f 100644 --- a/packages/ciphernode/net/tests/Dockerfile +++ b/packages/ciphernode/net/tests/Dockerfile @@ -13,8 +13,6 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends iptables ca-certificates && \ apt-get clean && rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/target/release/p2p_test /app/ -COPY net/tests/entrypoint.sh /app/ -RUN chmod +x /app/entrypoint.sh +COPY --from=builder /app/target/release/p2p_test . -ENTRYPOINT ["/app/entrypoint.sh"] +ENTRYPOINT ["/app/p2p_test"] diff --git a/packages/ciphernode/net/tests/docker-compose.yaml b/packages/ciphernode/net/tests/docker-compose.yaml index d54a3046..71a2449c 100644 --- a/packages/ciphernode/net/tests/docker-compose.yaml +++ b/packages/ciphernode/net/tests/docker-compose.yaml @@ -3,44 +3,38 @@ services: build: dockerfile: net/tests/Dockerfile context: ../.. - image: p2p-test-image - networks: - app_net: - ipv4_address: 172.16.238.10 command: ["/app/p2p_test", "alice"] environment: QUIC_PORT: 9091 - DIAL_TO: "/ip4/172.16.238.12/udp/9091/quic-v1" + DIAL_TO: "/dns4/charlie/udp/9091/quic-v1" ENABLE_MDNS: "${ENABLE_MDNS:-true}" - entrypoint: ["/app/entrypoint.sh"] + networks: + - p2p_test_net bob: - image: p2p-test-image - networks: - app_net: - ipv4_address: 172.16.238.11 + build: + dockerfile: net/tests/Dockerfile + context: ../.. command: ["/app/p2p_test", "bob"] environment: QUIC_PORT: 9091 - DIAL_TO: "/ip4/172.16.238.12/udp/9091/quic-v1" + DIAL_TO: "/dns4/charlie/udp/9091/quic-v1" ENABLE_MDNS: "${ENABLE_MDNS:-true}" - entrypoint: ["/app/entrypoint.sh"] + networks: + - p2p_test_net charlie: - image: p2p-test-image - networks: - app_net: - ipv4_address: 172.16.238.12 + build: + dockerfile: net/tests/Dockerfile + context: ../.. command: ["/app/p2p_test", "charlie"] environment: QUIC_PORT: 9091 ENABLE_MDNS: "${ENABLE_MDNS:-true}" - entrypoint: ["/app/entrypoint.sh"] + networks: + - p2p_test_net networks: - app_net: + p2p_test_net: driver: bridge - ipam: - driver: default - config: - - subnet: 172.16.238.0/24 + diff --git a/packages/ciphernode/net/tests/entrypoint.sh b/packages/ciphernode/net/tests/entrypoint.sh deleted file mode 100755 index a6453106..00000000 --- a/packages/ciphernode/net/tests/entrypoint.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e - -exec "$@" From 178c8760f4c2cfef881b05db016c4915cb12acb0 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 22 Dec 2024 21:23:48 +1100 Subject: [PATCH 24/36] Rename .deploy deploy --- .deploy/build-dummy.sh | 3 --- {.deploy => deploy}/.env.example | 0 {.deploy => deploy}/.gitignore | 0 {.deploy => deploy}/agg.yaml | 0 deploy/build-dummy.sh | 3 +++ {.deploy => deploy}/build.sh | 0 {.deploy => deploy}/cn1.yaml | 0 {.deploy => deploy}/cn2.yaml | 0 {.deploy => deploy}/cn3.yaml | 0 {.deploy => deploy}/copy-secrets.sh | 0 {.deploy => deploy}/deploy.sh | 6 +++--- {.deploy => deploy}/docker-compose.yml | 0 {.deploy => deploy}/dummy.Dockerfile | 0 {.deploy => deploy}/example.secrets.json | 0 {.deploy => deploy}/inspect.sh | 0 {.deploy => deploy}/swarm_deployment.md | 0 16 files changed, 6 insertions(+), 6 deletions(-) delete mode 100755 .deploy/build-dummy.sh rename {.deploy => deploy}/.env.example (100%) rename {.deploy => deploy}/.gitignore (100%) rename {.deploy => deploy}/agg.yaml (100%) create mode 100755 deploy/build-dummy.sh rename {.deploy => deploy}/build.sh (100%) rename {.deploy => deploy}/cn1.yaml (100%) rename {.deploy => deploy}/cn2.yaml (100%) rename {.deploy => deploy}/cn3.yaml (100%) rename {.deploy => deploy}/copy-secrets.sh (100%) rename {.deploy => deploy}/deploy.sh (90%) rename {.deploy => deploy}/docker-compose.yml (100%) rename {.deploy => deploy}/dummy.Dockerfile (100%) rename {.deploy => deploy}/example.secrets.json (100%) rename {.deploy => deploy}/inspect.sh (100%) rename {.deploy => deploy}/swarm_deployment.md (100%) diff --git a/.deploy/build-dummy.sh b/.deploy/build-dummy.sh deleted file mode 100755 index a3453ffa..00000000 --- a/.deploy/build-dummy.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -docker build -t norepo/dummy:666 -f ./.deploy/dummy.Dockerfile . diff --git a/.deploy/.env.example b/deploy/.env.example similarity index 100% rename from .deploy/.env.example rename to deploy/.env.example diff --git a/.deploy/.gitignore b/deploy/.gitignore similarity index 100% rename from .deploy/.gitignore rename to deploy/.gitignore diff --git a/.deploy/agg.yaml b/deploy/agg.yaml similarity index 100% rename from .deploy/agg.yaml rename to deploy/agg.yaml diff --git a/deploy/build-dummy.sh b/deploy/build-dummy.sh new file mode 100755 index 00000000..36a3fa49 --- /dev/null +++ b/deploy/build-dummy.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker build -t norepo/dummy:666 -f ./deploy/dummy.Dockerfile . diff --git a/.deploy/build.sh b/deploy/build.sh similarity index 100% rename from .deploy/build.sh rename to deploy/build.sh diff --git a/.deploy/cn1.yaml b/deploy/cn1.yaml similarity index 100% rename from .deploy/cn1.yaml rename to deploy/cn1.yaml diff --git a/.deploy/cn2.yaml b/deploy/cn2.yaml similarity index 100% rename from .deploy/cn2.yaml rename to deploy/cn2.yaml diff --git a/.deploy/cn3.yaml b/deploy/cn3.yaml similarity index 100% rename from .deploy/cn3.yaml rename to deploy/cn3.yaml diff --git a/.deploy/copy-secrets.sh b/deploy/copy-secrets.sh similarity index 100% rename from .deploy/copy-secrets.sh rename to deploy/copy-secrets.sh diff --git a/.deploy/deploy.sh b/deploy/deploy.sh similarity index 90% rename from .deploy/deploy.sh rename to deploy/deploy.sh index feaaf918..749b792c 100755 --- a/.deploy/deploy.sh +++ b/deploy/deploy.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash TIMESTAMP=$(date +%s) -RUN_FILE="./.deploy/tmp.docker-compose.${TIMESTAMP}.yml" -TEMPLATE_FILE="./.deploy/docker-compose.yml" +RUN_FILE="./deploy/tmp.docker-compose.${TIMESTAMP}.yml" +TEMPLATE_FILE="./deploy/docker-compose.yml" wait_ready() { local STACK_NAME="$1" @@ -52,4 +52,4 @@ docker stack rm $STACK_NAME wait_removed $STACK_NAME docker stack deploy -c $RUN_FILE $STACK_NAME wait_ready $STACK_NAME -rm ./.deploy/tmp.*.* +rm ./deploy/tmp.*.* diff --git a/.deploy/docker-compose.yml b/deploy/docker-compose.yml similarity index 100% rename from .deploy/docker-compose.yml rename to deploy/docker-compose.yml diff --git a/.deploy/dummy.Dockerfile b/deploy/dummy.Dockerfile similarity index 100% rename from .deploy/dummy.Dockerfile rename to deploy/dummy.Dockerfile diff --git a/.deploy/example.secrets.json b/deploy/example.secrets.json similarity index 100% rename from .deploy/example.secrets.json rename to deploy/example.secrets.json diff --git a/.deploy/inspect.sh b/deploy/inspect.sh similarity index 100% rename from .deploy/inspect.sh rename to deploy/inspect.sh diff --git a/.deploy/swarm_deployment.md b/deploy/swarm_deployment.md similarity index 100% rename from .deploy/swarm_deployment.md rename to deploy/swarm_deployment.md From ce9d78da6c85ed14c64018edaa2d64c26194d4b6 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 22 Dec 2024 21:24:16 +1100 Subject: [PATCH 25/36] Remove Dockerfile --- deploy/build-dummy.sh | 3 --- deploy/dummy.Dockerfile | 33 --------------------------------- 2 files changed, 36 deletions(-) delete mode 100755 deploy/build-dummy.sh delete mode 100644 deploy/dummy.Dockerfile diff --git a/deploy/build-dummy.sh b/deploy/build-dummy.sh deleted file mode 100755 index 36a3fa49..00000000 --- a/deploy/build-dummy.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -docker build -t norepo/dummy:666 -f ./deploy/dummy.Dockerfile . diff --git a/deploy/dummy.Dockerfile b/deploy/dummy.Dockerfile deleted file mode 100644 index 3cd39847..00000000 --- a/deploy/dummy.Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# THIS IS A DUMMY IMAGE FOR TESTING -# Recreate a runtime image to test env vars without long build times -FROM debian:stable-slim -RUN apt-get update && apt-get install -y --no-install-recommends iptables ca-certificates jq dnsutils iputils-ping && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Create non-root user -RUN useradd -m -u 1000 -s /bin/bash ciphernode - -# Create necessary directories with proper permissions -RUN mkdir -p /home/ciphernode/.config/enclave \ - /home/ciphernode/.local/share/enclave \ - /run/secrets && \ - chown -R ciphernode:ciphernode /home/ciphernode /run/secrets - -RUN echo "echo \"HELLO\" && sleep infinity" > /home/ciphernode/ciphernode-entrypoint.sh && \ - chmod +x /home/ciphernode/ciphernode-entrypoint.sh && \ - chown ciphernode:ciphernode /home/ciphernode/ciphernode-entrypoint.sh - -# Switch to non-root user -USER ciphernode -WORKDIR /home/ciphernode - -# Create the entrypoint script as root -RUN chmod +x /home/ciphernode/ciphernode-entrypoint.sh - -# Environment variables for configuration -ENV CONFIG_DIR=/home/ciphernode/.config/enclave -ENV DATA_DIR=/home/ciphernode/.local/share/enclave -ENV RUST_LOG=info - -# Add entrypoint script -ENTRYPOINT ["/bin/bash","/home/ciphernode/ciphernode-entrypoint.sh"] From c6f0e11456627ab3c6089da2dfa05c63244e8478 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 22 Dec 2024 21:58:18 +1100 Subject: [PATCH 26/36] Add docs --- packages/ciphernode/net/src/correlation_id.rs | 1 + packages/ciphernode/net/src/dialer.rs | 17 +++++++++++++++-- packages/ciphernode/net/src/events.rs | 8 ++++++++ packages/ciphernode/net/src/network_manager.rs | 3 ++- packages/ciphernode/net/src/network_peer.rs | 17 ++++++++++++++--- 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/packages/ciphernode/net/src/correlation_id.rs b/packages/ciphernode/net/src/correlation_id.rs index 2d82c37e..19dac8db 100644 --- a/packages/ciphernode/net/src/correlation_id.rs +++ b/packages/ciphernode/net/src/correlation_id.rs @@ -5,6 +5,7 @@ use std::{ static NEXT_CORRELATION_ID: AtomicUsize = AtomicUsize::new(1); +/// CorrelationId provides a way to correlate commands and the events they create. #[derive(Debug, Clone)] pub struct CorrelationId { id: usize, diff --git a/packages/ciphernode/net/src/dialer.rs b/packages/ciphernode/net/src/dialer.rs index 7447fe88..f684c50b 100644 --- a/packages/ciphernode/net/src/dialer.rs +++ b/packages/ciphernode/net/src/dialer.rs @@ -15,6 +15,7 @@ use crate::{ retry::{retry_with_backoff, to_retry, RetryError, BACKOFF_DELAY, BACKOFF_MAX_RETRIES}, }; +/// Dial a single Multiaddr with retries and return an error should those retries not work async fn dial_multiaddr( cmd_tx: &mpsc::Sender, event_tx: &broadcast::Sender, @@ -37,6 +38,12 @@ fn trace_error(r: Result<()>) { } } +/// Initiates connections to multiple network peers +/// +/// # Arguments +/// * `cmd_tx` - Sender for network peer commands +/// * `event_tx` - Broadcast sender for peer events +/// * `peers` - List of peer addresses to connect to pub async fn dial_peers( cmd_tx: &mpsc::Sender, event_tx: &broadcast::Sender, @@ -51,6 +58,7 @@ pub async fn dial_peers( Ok(()) } +/// Attempt a connection with retrys to a multiaddr return an error if the connection could not be resolved after the retries. async fn attempt_connection( cmd_tx: &mpsc::Sender, event_tx: &broadcast::Sender, @@ -68,6 +76,8 @@ async fn attempt_connection( wait_for_connection(&mut event_rx, dial_connection).await } +/// Wait for results of a retry based on a given correlation id and return the correct variant of +/// RetryError depending on the result from the downstream event async fn wait_for_connection( event_rx: &mut broadcast::Receiver, dial_connection: ConnectionId, @@ -117,6 +127,7 @@ async fn wait_for_connection( } } +/// Convert a Multiaddr to use a specific ip address with the ip4 or ip6 protocol fn dns_to_ip_addr(original: &Multiaddr, ip_str: &str) -> Result { let ip = ip_str.parse()?; let mut new_addr = Multiaddr::empty(); @@ -140,6 +151,7 @@ fn dns_to_ip_addr(original: &Multiaddr, ip_str: &str) -> Result { Ok(new_addr) } +/// Detect the DNS host from a multiaddr fn extract_dns_host(addr: &Multiaddr) -> Option { // Iterate through the protocols in the multiaddr for proto in addr.iter() { @@ -154,9 +166,10 @@ fn extract_dns_host(addr: &Multiaddr) -> Option { None } +/// If the Multiaddr uses a DNS domain look it up and return a multiaddr that uses a resolved IP +/// address fn get_resolved_multiaddr(value: &Multiaddr) -> Result { - let maybe_domain = extract_dns_host(value); - if let Some(domain) = maybe_domain { + if let Some(domain) = extract_dns_host(value) { let ip = resolve_ipv4(&domain)?; let multi = dns_to_ip_addr(value, &ip)?; return Ok(multi); diff --git a/packages/ciphernode/net/src/events.rs b/packages/ciphernode/net/src/events.rs index 06f641df..46388372 100644 --- a/packages/ciphernode/net/src/events.rs +++ b/packages/ciphernode/net/src/events.rs @@ -8,6 +8,7 @@ use libp2p::{ use crate::correlation_id::CorrelationId; +/// NetworkPeer Commands are sent to the network peer over a mspc channel pub enum NetworkPeerCommand { GossipPublish { topic: String, @@ -17,24 +18,31 @@ pub enum NetworkPeerCommand { Dial(DialOpts), } +/// NetworkPeerEvents are broadcast over a broadcast channel to whom ever wishes to listen #[derive(Message, Clone, Debug)] #[rtype(result = "anyhow::Result<()>")] pub enum NetworkPeerEvent { + /// Bytes have been broadcast over the network GossipData(Vec), + /// There was an Error publishing bytes over the network GossipPublishError { correlation_id: CorrelationId, error: Arc, }, + /// Data was successfully published over the network as far as we know. GossipPublished { correlation_id: CorrelationId, message_id: MessageId, }, + /// There was an error Dialing a peer DialError { error: Arc, }, + /// A connection was established to a peer ConnectionEstablished { connection_id: ConnectionId, }, + /// There was an error creating a connection OutgoingConnectionError { connection_id: ConnectionId, error: Arc, diff --git a/packages/ciphernode/net/src/network_manager.rs b/packages/ciphernode/net/src/network_manager.rs index 5032c36d..7c26f1e1 100644 --- a/packages/ciphernode/net/src/network_manager.rs +++ b/packages/ciphernode/net/src/network_manager.rs @@ -5,7 +5,7 @@ use crate::NetworkPeer; /// Actor for connecting to an libp2p client via it's mpsc channel interface /// This Actor should be responsible for use actix::prelude::*; -use anyhow::{anyhow, bail, Result}; +use anyhow::{bail, Result}; use cipher::Cipher; use data::Repository; use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; @@ -30,6 +30,7 @@ impl Actor for NetworkManager { type Context = Context; } +/// Libp2pEvent is used to send data to the NetworkPeer from the NetworkManager #[derive(Message, Clone, Debug, PartialEq, Eq)] #[rtype(result = "anyhow::Result<()>")] struct LibP2pEvent(pub Vec); diff --git a/packages/ciphernode/net/src/network_peer.rs b/packages/ciphernode/net/src/network_peer.rs index 00366439..677a1f06 100644 --- a/packages/ciphernode/net/src/network_peer.rs +++ b/packages/ciphernode/net/src/network_peer.rs @@ -29,14 +29,23 @@ pub struct NodeBehaviour { identify: IdentifyBehaviour, } +/// Manage the peer to peer connection. This struct wraps a libp2p Swarm and enables communication +/// with it using channels. pub struct NetworkPeer { + /// The Libp2p Swarm instance swarm: Swarm, + /// A list of peers to automatically dial peers: Vec, + /// The UDP port that the peer listens to over QUIC udp_port: Option, + /// The gossipsub topic that the peer should listen on topic: gossipsub::IdentTopic, - event_tx: broadcast::Sender, // to event bus - cmd_tx: mpsc::Sender, // to network - cmd_rx: mpsc::Receiver, // from event bus + /// Broadcast channel to report NetworkPeerEvents to listeners + event_tx: broadcast::Sender, + /// Transmission channel to send NetworkPeerCommands to the NetworkPeer + cmd_tx: mpsc::Sender, + /// Local receiver to process NetworkPeerCommands from + cmd_rx: mpsc::Receiver, } impl NetworkPeer { @@ -148,6 +157,7 @@ impl NetworkPeer { } } +/// Create the libp2p behaviour fn create_mdns_kad_behaviour( enable_mdns: bool, key: &Keypair, @@ -197,6 +207,7 @@ fn create_mdns_kad_behaviour( }) } +/// Process all swarm events async fn process_swarm_event( swarm: &mut Swarm, event_tx: &broadcast::Sender, From f80a26f48a091b135e4990437a91f72935b28d23 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 22 Dec 2024 22:05:11 +1100 Subject: [PATCH 27/36] remove println --- packages/ciphernode/config/src/app_config.rs | 1 - packages/ciphernode/net/src/dialer.rs | 15 ++++++++------- packages/ciphernode/net/src/network_peer.rs | 6 +++--- packages/ciphernode/net/src/retry.rs | 5 +++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index d821b29e..cc6eac80 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -273,7 +273,6 @@ pub fn load_config(config_file: Option<&str>) -> Result { } let with_envs = load_yaml_with_env(&defaults.config_file())?; - println!("{}", with_envs); let config = Figment::from(Serialized::defaults(&defaults)) .merge(Yaml::string(&with_envs)) diff --git a/packages/ciphernode/net/src/dialer.rs b/packages/ciphernode/net/src/dialer.rs index f684c50b..f11d6396 100644 --- a/packages/ciphernode/net/src/dialer.rs +++ b/packages/ciphernode/net/src/dialer.rs @@ -6,6 +6,7 @@ use libp2p::{ swarm::{dial_opts::DialOpts, ConnectionId, DialError}, Multiaddr, }; +use tracing::info; use std::net::ToSocketAddrs; use tokio::sync::{broadcast, mpsc}; use tracing::error; @@ -22,7 +23,7 @@ async fn dial_multiaddr( multiaddr_str: &str, ) -> Result<()> { let multiaddr = &multiaddr_str.parse()?; - println!("Now dialing in to {}", multiaddr); + info!("Now dialing in to {}", multiaddr); retry_with_backoff( || attempt_connection(cmd_tx, event_tx, multiaddr), BACKOFF_MAX_RETRIES, @@ -68,7 +69,7 @@ async fn attempt_connection( let multi = get_resolved_multiaddr(multiaddr).map_err(to_retry)?; let opts: DialOpts = multi.clone().into(); let dial_connection = opts.connection_id(); - println!("Dialing: '{}' with connection '{}'", multi, dial_connection); + info!("Dialing: '{}' with connection '{}'", multi, dial_connection); cmd_tx .send(NetworkPeerCommand::Dial(opts)) .await @@ -86,16 +87,16 @@ async fn wait_for_connection( match event_rx.recv().await.map_err(to_retry)? { NetworkPeerEvent::ConnectionEstablished { connection_id } => { if connection_id == dial_connection { - println!("Connection Established"); + info!("Connection Established"); return Ok(()); } } NetworkPeerEvent::DialError { error } => { - println!("DialError!"); + info!("DialError!"); return match error.as_ref() { // If we are dialing ourself then we should just fail DialError::NoAddresses { .. } => { - println!("DialError received. Returning RetryError::Failure"); + info!("DialError received. Returning RetryError::Failure"); Err(RetryError::Failure(error.clone().into())) } // Try again otherwise @@ -106,9 +107,9 @@ async fn wait_for_connection( connection_id, error, } => { - println!("OutgoingConnectionError!"); + info!("OutgoingConnectionError!"); if connection_id == dial_connection { - println!( + info!( "Connection {} failed because of error {}. Retrying...", connection_id, error ); diff --git a/packages/ciphernode/net/src/network_peer.rs b/packages/ciphernode/net/src/network_peer.rs index 677a1f06..e202e061 100644 --- a/packages/ciphernode/net/src/network_peer.rs +++ b/packages/ciphernode/net/src/network_peer.rs @@ -137,11 +137,11 @@ impl NetworkPeer { } }, NetworkPeerCommand::Dial(multi) => { - println!("DIAL: {:?}", multi); + info!("DIAL: {:?}", multi); match self.swarm.dial(multi) { - Ok(v) => println!("Dial returned {:?}", v), + Ok(v) => info!("Dial returned {:?}", v), Err(error) => { - println!("Dialing error! {}", error); + info!("Dialing error! {}", error); event_tx.send(NetworkPeerEvent::DialError { error: error.into() })?; } } diff --git a/packages/ciphernode/net/src/retry.rs b/packages/ciphernode/net/src/retry.rs index 5065c467..f45ef27d 100644 --- a/packages/ciphernode/net/src/retry.rs +++ b/packages/ciphernode/net/src/retry.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use tracing::{error, warn}; use std::{future::Future, time::Duration}; use tokio::time::sleep; @@ -49,7 +50,7 @@ where )); } - println!( + warn!( "Attempt {}/{} failed, retrying in {}ms: {}", current_attempt, max_attempts, delay_ms, e ); @@ -59,7 +60,7 @@ where delay_ms *= 2; // Exponential backoff } RetryError::Failure(e) => { - println!("FAILURE!: returning to caller."); + error!("FAILURE!: returning to caller."); return Err(e); } } From c374a77e0dffddc89fa546f2f19a745a09fb2208 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 22 Dec 2024 22:08:24 +1100 Subject: [PATCH 28/36] Formatting --- packages/ciphernode/net/src/correlation_id.rs | 2 +- packages/ciphernode/net/src/dialer.rs | 4 ++-- packages/ciphernode/net/src/events.rs | 10 +++------- packages/ciphernode/net/src/network_peer.rs | 2 +- packages/ciphernode/net/src/retry.rs | 2 +- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/ciphernode/net/src/correlation_id.rs b/packages/ciphernode/net/src/correlation_id.rs index 19dac8db..51265b62 100644 --- a/packages/ciphernode/net/src/correlation_id.rs +++ b/packages/ciphernode/net/src/correlation_id.rs @@ -5,7 +5,7 @@ use std::{ static NEXT_CORRELATION_ID: AtomicUsize = AtomicUsize::new(1); -/// CorrelationId provides a way to correlate commands and the events they create. +/// CorrelationId provides a way to correlate commands and the events they create. #[derive(Debug, Clone)] pub struct CorrelationId { id: usize, diff --git a/packages/ciphernode/net/src/dialer.rs b/packages/ciphernode/net/src/dialer.rs index f11d6396..ba1f6dad 100644 --- a/packages/ciphernode/net/src/dialer.rs +++ b/packages/ciphernode/net/src/dialer.rs @@ -6,10 +6,10 @@ use libp2p::{ swarm::{dial_opts::DialOpts, ConnectionId, DialError}, Multiaddr, }; -use tracing::info; use std::net::ToSocketAddrs; use tokio::sync::{broadcast, mpsc}; use tracing::error; +use tracing::info; use crate::{ events::{NetworkPeerCommand, NetworkPeerEvent}, @@ -78,7 +78,7 @@ async fn attempt_connection( } /// Wait for results of a retry based on a given correlation id and return the correct variant of -/// RetryError depending on the result from the downstream event +/// RetryError depending on the result from the downstream event async fn wait_for_connection( event_rx: &mut broadcast::Receiver, dial_connection: ConnectionId, diff --git a/packages/ciphernode/net/src/events.rs b/packages/ciphernode/net/src/events.rs index 46388372..196ea48f 100644 --- a/packages/ciphernode/net/src/events.rs +++ b/packages/ciphernode/net/src/events.rs @@ -35,14 +35,10 @@ pub enum NetworkPeerEvent { message_id: MessageId, }, /// There was an error Dialing a peer - DialError { - error: Arc, - }, + DialError { error: Arc }, /// A connection was established to a peer - ConnectionEstablished { - connection_id: ConnectionId, - }, - /// There was an error creating a connection + ConnectionEstablished { connection_id: ConnectionId }, + /// There was an error creating a connection OutgoingConnectionError { connection_id: ConnectionId, error: Arc, diff --git a/packages/ciphernode/net/src/network_peer.rs b/packages/ciphernode/net/src/network_peer.rs index e202e061..412ae2b1 100644 --- a/packages/ciphernode/net/src/network_peer.rs +++ b/packages/ciphernode/net/src/network_peer.rs @@ -40,7 +40,7 @@ pub struct NetworkPeer { udp_port: Option, /// The gossipsub topic that the peer should listen on topic: gossipsub::IdentTopic, - /// Broadcast channel to report NetworkPeerEvents to listeners + /// Broadcast channel to report NetworkPeerEvents to listeners event_tx: broadcast::Sender, /// Transmission channel to send NetworkPeerCommands to the NetworkPeer cmd_tx: mpsc::Sender, diff --git a/packages/ciphernode/net/src/retry.rs b/packages/ciphernode/net/src/retry.rs index f45ef27d..a1fdd95e 100644 --- a/packages/ciphernode/net/src/retry.rs +++ b/packages/ciphernode/net/src/retry.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use tracing::{error, warn}; use std::{future::Future, time::Duration}; use tokio::time::sleep; +use tracing::{error, warn}; pub enum RetryError { Failure(anyhow::Error), From 31ad59693b6d7f3f4538c82e2bb48d418e944fca Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 22 Dec 2024 22:13:09 +1100 Subject: [PATCH 29/36] Add timeout --- packages/ciphernode/net/src/dialer.rs | 86 ++++++++++++++++----------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/packages/ciphernode/net/src/dialer.rs b/packages/ciphernode/net/src/dialer.rs index ba1f6dad..c6933014 100644 --- a/packages/ciphernode/net/src/dialer.rs +++ b/packages/ciphernode/net/src/dialer.rs @@ -7,7 +7,9 @@ use libp2p::{ Multiaddr, }; use std::net::ToSocketAddrs; +use tokio::select; use tokio::sync::{broadcast, mpsc}; +use tokio::time::{sleep, timeout, Duration}; use tracing::error; use tracing::info; @@ -84,46 +86,58 @@ async fn wait_for_connection( dial_connection: ConnectionId, ) -> Result<(), RetryError> { loop { - match event_rx.recv().await.map_err(to_retry)? { - NetworkPeerEvent::ConnectionEstablished { connection_id } => { - if connection_id == dial_connection { - info!("Connection Established"); - return Ok(()); - } - } - NetworkPeerEvent::DialError { error } => { - info!("DialError!"); - return match error.as_ref() { - // If we are dialing ourself then we should just fail - DialError::NoAddresses { .. } => { - info!("DialError received. Returning RetryError::Failure"); - Err(RetryError::Failure(error.clone().into())) + // Create a timeout future that can be reset + select! { + result = event_rx.recv() => { + match result.map_err(to_retry)? { + NetworkPeerEvent::ConnectionEstablished { connection_id } => { + if connection_id == dial_connection { + info!("Connection Established"); + return Ok(()); + } } - // Try again otherwise - _ => Err(RetryError::Retry(error.clone().into())), - }; - } - NetworkPeerEvent::OutgoingConnectionError { - connection_id, - error, - } => { - info!("OutgoingConnectionError!"); - if connection_id == dial_connection { - info!( - "Connection {} failed because of error {}. Retrying...", - connection_id, error - ); - return match error.as_ref() { - // If we are dialing ourself then we should just fail - DialError::NoAddresses { .. } => { - Err(RetryError::Failure(error.clone().into())) + NetworkPeerEvent::DialError { error } => { + info!("DialError!"); + return match error.as_ref() { + // If we are dialing ourself then we should just fail + DialError::NoAddresses { .. } => { + info!("DialError received. Returning RetryError::Failure"); + Err(RetryError::Failure(error.clone().into())) + } + // Try again otherwise + _ => Err(RetryError::Retry(error.clone().into())), + }; + } + NetworkPeerEvent::OutgoingConnectionError { + connection_id, + error, + } => { + info!("OutgoingConnectionError!"); + if connection_id == dial_connection { + info!( + "Connection {} failed because of error {}. Retrying...", + connection_id, error + ); + return match error.as_ref() { + // If we are dialing ourself then we should just fail + DialError::NoAddresses { .. } => { + Err(RetryError::Failure(error.clone().into())) + } + // Try again otherwise + _ => Err(RetryError::Retry(error.clone().into())), + }; } - // Try again otherwise - _ => Err(RetryError::Retry(error.clone().into())), - }; + } + _ => (), } } - _ => (), + _ = sleep(Duration::from_secs(60)) => { + info!("Connection attempt timed out after 60 seconds of no events"); + return Err(RetryError::Retry(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "Connection attempt timed out", + ).into())); + } } } } From 743cb73e23a3e0182ac54c9335e91cb489638596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Mon, 23 Dec 2024 12:36:34 +1100 Subject: [PATCH 30/36] Update swarm_deployment.md --- deploy/swarm_deployment.md | 106 ++++++++++++++++++++++++++++--------- 1 file changed, 81 insertions(+), 25 deletions(-) diff --git a/deploy/swarm_deployment.md b/deploy/swarm_deployment.md index ca0d4e30..74e2d160 100644 --- a/deploy/swarm_deployment.md +++ b/deploy/swarm_deployment.md @@ -1,9 +1,83 @@ +# Setup a remote server + +Install docker + +``` +sudo apt-get update +sudo apt-get install -y ca-certificates curl gnupg lsb-release +sudo mkdir -m 0755 -p /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +sudo docker run hello-world +``` + + +Initialize swarm + +``` +docker swarm init +``` + +NOTE: If you get an error about not being able to choose between IP addresses choose the more private IP address. + + +``` +docker swarm init --advertise-addr 10.49.0.5 +``` + +## Setting up Buildkit + +NOTE: You may need to setup buildkit: + +``` +echo '{ + "builder": { + "gc": { + "enabled": true, + "defaultKeepStorage": "20GB" + } + }, + "features": { + "buildkit": true, + "containerd-snapshotter": true + } +}' | sudo tee /etc/docker/daemon.json +``` + +and then restart the docker daemon + +``` +sudo systemctl restart docker +``` + +## Setup the repo + +Clone the repo + +``` +git clone https://github.com/gnosisguild/enclave.git +``` + +Move to the new folder: + +``` +cd enclave/ +``` + +Build the app + +``` +./deploy/build.sh +``` + # Setup `.env` vars Copy the `.env.example` file to `.env` ``` -cp .env.example .env +cp ./deploy/.env.example ./deploy/.env ``` Alter the variables to reflect the correct values required for the stack: @@ -15,12 +89,14 @@ export SEPOLIA_CIPHERNODE_REGISTRY_ADDRESS=0x0952388f6028a9Eda93a5041a3B216Ea331 export SEPOLIA_FILTER_REGISTRY=0xcBaCE7C360b606bb554345b20884A28e41436934 ``` -Pay special attention to the `TAG` and `RPC_URL` vars. +Pay special attention to the `RPC_URL` vars as here we use a standin API key value. You can peruse the yaml config files for the nodes to see how the vars are used within the config. -# Secrets Setup Script +# Secrets Setup Utils Script +We have created a secrets setup utility to aid setting up the secrets for each node. + To deploy with swarm we need to set up the secrets file for our cluster. ## Run @@ -41,34 +117,14 @@ Skipping cn2.secrets.json - file already exists ==> cn1.secrets.json <== # Yellow arrows indicate files that need customization ``` -Remember to modify any highlighted files before use. - -# Initialize docker swarm - -First we need to initialize swarm. - -``` -docker swarm init -``` - -If you get an error about not being able to choose between IP addresses choose the more private IP address. - -``` -docker swarm init --advertise-addr 10.49.x.x -``` - -# Build the dockerfile image - -``` -./.deploy/build.sh ghcr.io/gnosisguild/ciphernode -``` +Remember to modify any highlighted files before use with unique secrets. # Deploy a version to the stack To deploy ``` -./.deploy/deploy.sh enclave ghcr.io/gnosisguild/ciphernode:latest +./deploy/deploy.sh enclave ghcr.io/gnosisguild/ciphernode:latest ``` This will deploy the following services: From 0da013e9012d28e5681eb6bdf136a0e9f19da1f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Mon, 23 Dec 2024 12:57:57 +1100 Subject: [PATCH 31/36] Update swarm_deployment.md --- deploy/swarm_deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/swarm_deployment.md b/deploy/swarm_deployment.md index 74e2d160..6ea63779 100644 --- a/deploy/swarm_deployment.md +++ b/deploy/swarm_deployment.md @@ -101,7 +101,7 @@ To deploy with swarm we need to set up the secrets file for our cluster. ## Run ```bash -./.deploy/copy-secrets.sh +./deploy/copy-secrets.sh ``` ## What it does From 826218f2630d332043720fa20d8038a7f0f7ff05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Mon, 23 Dec 2024 13:02:59 +1100 Subject: [PATCH 32/36] Update deploy.sh --- deploy/deploy.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 749b792c..0825ecda 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -45,8 +45,6 @@ fi sed "s|{{IMAGE}}|$2|g" $TEMPLATE_FILE > "${RUN_FILE}" -cat $RUN_FILE - STACK_NAME=$1 docker stack rm $STACK_NAME wait_removed $STACK_NAME From cb5bcd72cde5434187c2344e8bb9b24cd1bd093d Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 23 Dec 2024 12:59:20 +0500 Subject: [PATCH 33/36] Update Secrets --- deploy/copy-secrets.sh | 8 ++++++++ packages/ciphernode/ciphernode-entrypoint.sh | 9 +++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/deploy/copy-secrets.sh b/deploy/copy-secrets.sh index dadbb38b..a91509aa 100755 --- a/deploy/copy-secrets.sh +++ b/deploy/copy-secrets.sh @@ -1,5 +1,10 @@ #!/usr/bin/env bash +set_network_private_key() { + echo "Setting network private key for $1" + jq --arg key "$2" '.network_private_key = $key' "$1.secrets.json" > "$1.secrets.json.tmp" && mv "$1.secrets.json.tmp" "$1.secrets.json" +} + # Set working directory to script location cd "$(dirname "$0")" || exit 1 @@ -13,6 +18,7 @@ NC='\033[0m' # No Color # List of target files TARGETS=("cn1" "cn2" "cn3" "agg") +NET_KEYS=("0x11a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" "0x21a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" "0x31a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" "0x41a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580") # Check if source file exists if [ ! -f "$SOURCE" ]; then @@ -26,6 +32,8 @@ for target in "${TARGETS[@]}"; do echo "Skipping ${target}.secrets.json - file already exists" else cp "$SOURCE" "${target}.secrets.json" + set_network_private_key "${target}" "${NET_KEYS[${i:-0}]}" + ((i++)) echo "Created ${target}.secrets.json" fi done diff --git a/packages/ciphernode/ciphernode-entrypoint.sh b/packages/ciphernode/ciphernode-entrypoint.sh index 3fa42fdd..917b8411 100644 --- a/packages/ciphernode/ciphernode-entrypoint.sh +++ b/packages/ciphernode/ciphernode-entrypoint.sh @@ -20,9 +20,10 @@ fi # Read secrets from the JSON file PRIVATE_KEY=$(jq -r '.private_key' "$SECRETS_FILE") PASSWORD=$(jq -r '.password' "$SECRETS_FILE") +NETWORK_PRIVATE_KEY=$(jq -r '.network_private_key' "$SECRETS_FILE") -if [ -z "$PRIVATE_KEY" ] || [ -z "$PASSWORD" ]; then - echo "Error: Missing 'private_key' or 'password' in secrets file!" +if [ -z "$PRIVATE_KEY" ] || [ -z "$PASSWORD" ] || [ -z "$NETWORK_PRIVATE_KEY" ]; then + echo "Error: Missing 'private_key', 'password' or 'network_private_key' in secrets file!" exit 1 fi @@ -30,6 +31,10 @@ fi echo "Setting password" enclave password create --config "$CONFIG_FILE" --password "$PASSWORD" +# Set network private key +echo "Setting network private key" +enclave net set-key --config "$CONFIG_FILE" --net-keypair "$NETWORK_PRIVATE_KEY" + if [ "$AGGREGATOR" = "true" ]; then echo "Setting private key" enclave wallet set --config "$CONFIG_FILE" --private-key "$PRIVATE_KEY" From a6086be335050a6433fbdc38edf5b99810f65958 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 23 Dec 2024 17:45:47 +0500 Subject: [PATCH 34/36] Update Sample Keys --- deploy/copy-secrets.sh | 8 +++++++- deploy/example.secrets.json | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/deploy/copy-secrets.sh b/deploy/copy-secrets.sh index a91509aa..d4d17177 100755 --- a/deploy/copy-secrets.sh +++ b/deploy/copy-secrets.sh @@ -18,7 +18,13 @@ NC='\033[0m' # No Color # List of target files TARGETS=("cn1" "cn2" "cn3" "agg") -NET_KEYS=("0x11a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" "0x21a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" "0x31a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" "0x41a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580") + +# Sample network private keys +NETWORK_KEY_CN1="0x11a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" +NETWORK_KEY_CN2="0x21a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" +NETWORK_KEY_CN3="0x31a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" +NETWORK_KEY_AGG="0x41a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" +NET_KEYS=($NETWORK_KEY_CN1 $NETWORK_KEY_CN2 $NETWORK_KEY_CN3 $NETWORK_KEY_AGG) # Check if source file exists if [ ! -f "$SOURCE" ]; then diff --git a/deploy/example.secrets.json b/deploy/example.secrets.json index f01ec15a..72bb3ec1 100644 --- a/deploy/example.secrets.json +++ b/deploy/example.secrets.json @@ -1,4 +1,5 @@ { "password": "changeme", - "private_key": "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" -} + "private_key": "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + "network_private_key": "0x11a1e500a548b70d88184a1e042900c0ed6c57f8710bcc35dc8c85fa33d3f580" +} \ No newline at end of file From 5551020ff5239a20a5690393c3f8b19709ecc7cf Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 23 Dec 2024 18:01:19 +0500 Subject: [PATCH 35/36] Update copy secrets --- deploy/copy-secrets.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/copy-secrets.sh b/deploy/copy-secrets.sh index d4d17177..2bfc6132 100755 --- a/deploy/copy-secrets.sh +++ b/deploy/copy-secrets.sh @@ -32,6 +32,7 @@ if [ ! -f "$SOURCE" ]; then exit 1 fi +i=0 # Copy file to each target, skipping if exists for target in "${TARGETS[@]}"; do if [ -f "${target}.secrets.json" ]; then From 234f9933487f1286d027f9873f46fa0f3d009d7e Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 23 Dec 2024 20:30:38 +0500 Subject: [PATCH 36/36] update workflow --- .github/workflows/ec2-deployment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ec2-deployment.yml b/.github/workflows/ec2-deployment.yml index 793b5c01..51168c2a 100644 --- a/.github/workflows/ec2-deployment.yml +++ b/.github/workflows/ec2-deployment.yml @@ -81,6 +81,5 @@ jobs: cd /home/ec2-user/enclave git pull - - docker stack deploy -c docker-compose.yml ciphernode-stack + ./deploy/deploy.sh enclave $IMAGE_NAME:$IMAGE_TAG