Merge pull request #8004 from Agoric/mfig-debug-deploy

feat(deployment): better `docker build` caching

.dockerignore

@@ -3,14 +3,15 @@
 docker
 packages/cosmic-swingset/t[0-9]
 packages/cosmic-swingset/t[0-9].*
-packages/cosmic-swingset/lib/lib*.h
-packages/cosmic-swingset/lib/lib*.so
+golang/cosmos/build
+packages/deployment
 packages/swingset-runner
 packages/stat-logger
 **/deployment.json
 **/vars.tf
 **/*.log
 **/build
+**/bundles
 **/__pycache__
 **/*.egg-info
 **/swingset-kernel-state

.github/workflows/deployment-test.yml

@@ -66,23 +66,16 @@ jobs:
           path: testnet-load-generator
           ref: ${{steps.get-loadgen-branch.outputs.result}}
 
-      - name: Put repos under /usr/src where scripts expect them
-        run: |
-          set -e
-          sudo mv "$GITHUB_WORKSPACE/testnet-load-generator" /usr/src/testnet-load-generator
-          sudo cp -a "$GITHUB_WORKSPACE" /usr/src/agoric-sdk
-          ln -s /usr/src/agoric-sdk/packages/deployment/bin/ag-setup-cosmos /usr/local/bin/ag-setup-cosmos
-        working-directory: /
-
       - name: Build cosmic-swingset dependencies
         run: |
           set -e
           cd packages/cosmic-swingset
           make install
-        working-directory: /usr/src/agoric-sdk
-      - run: /usr/src/agoric-sdk/packages/deployment/scripts/integration-test.sh
+      - run: |
+          set -xe
+          DOCKER_VOLUMES="$PWD:/usr/src/agoric-sdk" \
+          packages/deployment/scripts/integration-test.sh
         timeout-minutes: 90
-        working-directory: /usr/src/agoric-sdk
         env:
           NETWORK_NAME: chaintest
       - name: capture results
@@ -99,14 +92,13 @@ jobs:
 
           # Tear down the nodes.
           echo yes | packages/deployment/scripts/setup.sh destroy || true
-        working-directory: /usr/src/agoric-sdk
         env:
           NETWORK_NAME: chaintest
       - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: deployment-test-results-${{ env.NOW }}
-          path: /usr/src/agoric-sdk/chaintest/results
+          path: chaintest/results
 
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'

packages/deployment/Dockerfile

@@ -2,22 +2,13 @@ ARG TAG=latest
 ARG REPO=agoric/agoric-sdk
 ARG REGISTRY=ghcr.io
 
-# FIXME: Journalbeat compilation is currently broken, but non-essential.
-# Removed from the build.
-# FROM golang:1.20-bullseye AS go-build
-
-# WORKDIR /usr/src/journalbeat
-# RUN apt-get update -y && apt-get install -y libsystemd-dev
-# RUN go get github.com/mheese/journalbeat
-
 FROM $REGISTRY/$REPO:$TAG
 
+COPY ./scripts/install-deps.sh /usr/src/agoric-sdk/packages/deployment/scripts/
 RUN /usr/src/agoric-sdk/packages/deployment/scripts/install-deps.sh
 
-# # Copy journalbeat for logging support
-# COPY --from=go-build /go/bin/journalbeat /usr/local/bin/
-
 WORKDIR /usr/src/agoric-sdk/packages/deployment
+COPY . .
 RUN ln -sf $PWD/bin/ag-setup-cosmos /usr/local/bin/ag-setup-cosmos
 
 WORKDIR /data/chains

packages/deployment/Dockerfile.sdk

@@ -62,7 +62,7 @@ RUN \
   yarn build
 
 # Remove dev dependencies.
-RUN rm -rf packages/xsnap/moddable
+RUN rm -rf packages/xsnap/moddable packages/xsnap/xsnap-native/build/tmp
 # FIXME: This causes bundling differences.  https://github.com/endojs/endo/issues/919
 # RUN yarn install --frozen-lockfile --production --network-timeout 100000
 
@@ -86,11 +86,11 @@ ARG GIT_REVISION=unknown
 RUN echo "$GIT_REVISION" > /usr/src/agoric-sdk/packages/solo/public/git-revision.txt
 
 # Compatibility links for older containers.
-RUN ln -s /data /agoric
-RUN ln -s /data/solo /usr/src/agoric-sdk/packages/cosmic-swingset/solo
-RUN ln -s /data/chains /usr/src/agoric-sdk/packages/cosmic-swingset/chains
+RUN ln -sf /data /agoric
+RUN ln -sf /data/solo /usr/src/agoric-sdk/packages/cosmic-swingset/solo
+RUN ln -sf /data/chains /usr/src/agoric-sdk/packages/cosmic-swingset/chains
 
-RUN /usr/src/agoric-sdk/packages/deployment/scripts/smoketest-binaries.sh
+RUN /usr/src/agoric-sdk/scripts/smoketest-binaries.sh
 
 # By default, run the daemon with specified arguments.
 WORKDIR /root

packages/deployment/Makefile

@@ -5,7 +5,7 @@ VERSION := $(shell node -e 'console.log(require("../../package.json").version??"
 
 TAG := $(if $(VERSION),$(VERSION),latest)
 GIT_REVISION := $(shell hash=$$(git rev-parse --short HEAD); \
-	dirty=`git diff --quiet || echo -dirty`; \
+	dirty=`git diff --quiet 2>/dev/null || echo -dirty`; \
 	echo "$$hash$$dirty")
 
 # Don't push alpha tags as ":$(TAG)".

packages/deployment/ansible/roles/copy/tasks/main.yml

@@ -7,15 +7,14 @@
     use_ssh_args: true
   when: HELPER_BINARY is defined
 
-- name: Synchronize Agoric SDK
+- name: 'Synchronize Agoric SDK {{ AGORIC_SDK }}'
   synchronize:
     src: '{{ AGORIC_SDK }}/'
     dest: '/usr/src/agoric-sdk/'
-    # dirs: yes
     delete: yes
-    rsync_opts:
-      - "--partial-dir=/usr/src/agoric-sdk/.rsync-partial/{{ hostvars[inventory_hostname]['ansible_host'] | default(inventory_hostname) }}"
-      - '--exclude=.rsync-partial'
+    #rsync_opts:
+    #  - "--partial-dir=/usr/src/agoric-sdk/.rsync-partial/{{ hostvars[inventory_hostname]['ansible_host'] | default(inventory_hostname) }}"
+    #  - '--exclude=.rsync-partial'
     # checksum: yes
     mode: push
     use_ssh_args: true

packages/deployment/docker/ag-setup-cosmos

@@ -1,5 +1,4 @@
 #! /bin/sh
-PORT=26657
 NETWORK_NAME=${NETWORK_NAME-agoric}
 SETUP_HOME=${SETUP_HOME-$NETWORK_NAME}
 IMAGE=ghcr.io/agoric/cosmic-swingset-setup:${TAG-latest}
@@ -21,10 +20,17 @@ show-*)
   TTY=-i
   ;;
 esac
+setup_volume=
+if test -f "$PWD/$SETUP_HOME/setup/deployment.json"; then
+  setup_volume=--volume="$PWD/$SETUP_HOME/setup:/data/chains/$SETUP_HOME"
+elif test -f deployment.json; then
+  setup_volume=--volume="$PWD:/data/chains/$SETUP_HOME"
+fi
 exec docker run --rm $TTY $FLAGS \
   --volume=ag-setup-cosmos-chains:/data/chains \
   --volume=ag-chain-cosmos-state:/root/.ag-chain-cosmos \
   --volume=/var/run/docker.sock:/var/run/docker.sock \
+  $setup_volume \
   --env AGD_HOME=/root/.ag-chain-cosmos \
   --env AG_SETUP_COSMOS_NAME=$NETWORK_NAME \
   --env AG_SETUP_COSMOS_HOME=$SETUP_HOME \

packages/deployment/docker/integration-test.sh

@@ -0,0 +1,38 @@
+#! /bin/sh
+NETWORK_NAME=${NETWORK_NAME-localtest}
+SETUP_HOME=${SETUP_HOME-$NETWORK_NAME}
+IMAGE=ghcr.io/agoric/cosmic-swingset-setup:${TAG-latest}
+TTY=-i
+test -t 0 && test -t 1 && TTY=-it
+FLAGS=--entrypoint=/bin/bash
+case "$1" in
+--pull)
+  shift
+  docker pull "$IMAGE"
+  ;;
+esac
+case "$1" in
+shell) shift ;;
+*)
+  set /usr/src/agoric-sdk/packages/deployment/scripts/integration-test.sh ${1+"$@"}
+  ;;
+esac
+
+setup_volume=
+if test -f "$PWD/$SETUP_HOME/setup/deployment.json"; then
+  setup_volume=--volume="$PWD/$SETUP_HOME/setup:/data/chains/$SETUP_HOME"
+elif test -f deployment.json; then
+  setup_volume=--volume="$PWD:/data/chains/$SETUP_HOME"
+fi
+if test -n "$LOADGEN"; then
+  setup_volume="$setup_volume --volume=$LOADGEN:/usr/src/testnet-load-generator"
+fi
+exec docker run --rm $TTY $FLAGS \
+  --volume=ag-setup-cosmos-chains:/data/chains \
+  --volume=ag-chain-cosmos-state:/root/.ag-chain-cosmos \
+  --volume=/var/run/docker.sock:/var/run/docker.sock \
+  $setup_volume \
+  --env AGD_HOME=/root/.ag-chain-cosmos \
+  --env NETWORK_NAME=$NETWORK_NAME \
+  -w /data/chains \
+  "$IMAGE" ${1+"$@"}

packages/deployment/scripts/integration-test.sh

@@ -7,6 +7,20 @@ thisdir=$(cd "$(dirname -- "$real0")" > /dev/null && pwd -P)
 export GOBIN="$thisdir/../../../golang/cosmos/build"
 export NETWORK_NAME=${NETWORK_NAME-localtest}
 
+SDK_SRC=${SDK_SRC-$(cd "$thisdir/../../.." > /dev/null && pwd -P)}
+
+DEFAULT_LOADGEN=/usr/src/testnet-load-generator
+LOADGEN=${LOADGEN-""}
+if [ -n "$LOADGEN" ]; then
+  LOADGEN=$(cd "$LOADGEN" > /dev/null && pwd -P)
+elif [ -d "$SDK_SRC/../testnet-load-generator" ]; then
+  LOADGEN=$(cd "$SDK_SRC/../testnet-load-generator" > /dev/null && pwd -P)
+elif [ -d "$DEFAULT_LOADGEN" ]; then
+  LOADGEN=$(cd "$DEFAULT_LOADGEN" > /dev/null && pwd -P)
+else
+  LOADGEN=
+fi
+
 SOLO_ADDR=
 VAT_CONFIG=
 RESULTSDIR=${RESULTSDIR-"$NETWORK_NAME/results"}
@@ -21,19 +35,16 @@ cd "$NETWORK_NAME/setup"
 
 export AG_SETUP_COSMOS_HOME=${AG_SETUP_COSMOS_HOME-$PWD}
 export AG_SETUP_COSMOS_STATE_SYNC_INTERVAL=20 
-AGORIC_SDK_PATH=${AGORIC_SDK_PATH-$(cd "$thisdir/../../.." > /dev/null && pwd -P)}
 
-if [ -d /usr/src/testnet-load-generator ]
+if [ -n "$LOADGEN" ]
 then
-  solodir=/usr/src/testnet-load-generator/_agstate/agoric-servers/testnet-8000
+  solodir="$LOADGEN"/_agstate/agoric-servers/testnet-8000
   "$thisdir/../../solo/bin/ag-solo" init "$solodir" --webport=8000
   SOLO_ADDR=$(cat "$solodir/ag-cosmos-helper-address")
   VAT_CONFIG="@agoric/vats/decentral-demo-config.json"
 fi
 
-# Speed up the docker deployment by pre-mounting /usr/src/agoric-sdk.
-DOCKER_VOLUMES="$AGORIC_SDK_PATH:/usr/src/agoric-sdk" \
-  "$thisdir/docker-deployment.cjs" > deployment.json
+"$thisdir/docker-deployment.cjs" > deployment.json
 
 # Set up the network from our above deployment.json.
 "$thisdir/setup.sh" init --noninteractive
@@ -44,18 +55,18 @@ VAULT_FACTORY_CONTROLLER_ADDR="$SOLO_ADDR" \
 CHAIN_BOOTSTRAP_VAT_CONFIG="$VAT_CONFIG" \
   "$thisdir/setup.sh" bootstrap ${1+"$@"}
 
-if [ -d /usr/src/testnet-load-generator ]
+if [ -n "$LOADGEN" ]
 then
-  /usr/src/agoric-sdk/packages/deployment/scripts/setup.sh show-config > "$RESULTSDIR/network-config"
+  "$SDK_SRC/packages/deployment/scripts/setup.sh" show-config > "$RESULTSDIR/network-config"
   cp ag-chain-cosmos/data/genesis.json "$RESULTSDIR/genesis.json"
   cp "$AG_SETUP_COSMOS_HOME/ag-chain-cosmos/data/genesis.json" "$RESULTSDIR/genesis.json"
-  cd /usr/src/testnet-load-generator
+  cd "$LOADGEN"
   SOLO_COINS=40000000000uist \
     "$AG_SETUP_COSMOS_HOME/faucet-helper.sh" add-egress loadgen "$SOLO_ADDR"
-  SLOGSENDER=@agoric/telemetry/src/otel-trace.js SOLO_SLOGSENDER= \
+  SLOGSENDER=@agoric/telemetry/src/otel-trace.js SOLO_SLOGSENDER="" \
   SLOGSENDER_FAIL_ON_ERROR=1 SLOGSENDER_AGENT=process \
   AG_CHAIN_COSMOS_HOME=$HOME/.agoric \
-  SDK_BUILD=0 MUST_USE_PUBLISH_BUNDLE=1 SDK_SRC=/usr/src/agoric-sdk OUTPUT_DIR="$RESULTSDIR" ./start.sh \
+  SDK_BUILD=0 MUST_USE_PUBLISH_BUNDLE=1 SDK_SRC=$SDK_SRC OUTPUT_DIR="$RESULTSDIR" ./start.sh \
     --no-stage.save-storage \
     --stages=3 --stage.duration=10 --stage.loadgen.cycles=4 \
     --stage.loadgen.faucet.interval=6 --stage.loadgen.faucet.limit=4 \

packages/deployment/scripts/setup.sh

@@ -11,8 +11,7 @@ export NETWORK_NAME
 export AG_SETUP_COSMOS_NAME=$NETWORK_NAME
 export AG_SETUP_COSMOS_HOME=${AG_SETUP_COSMOS_HOME-"$PWD/$NETWORK_NAME/setup"}
 
-# Put our bindir into the PATH so that children can find ag-setup-cosmos.
+# Put GOBIN into the PATH so that children can find ag-setup-cosmos.
 export PATH="$thisdir/../bin:${GOBIN-${GOPATH-/usr/local}/bin}:$PATH"
 
-# Run our setup command.
 exec ag-setup-cosmos ${1+"$@"}

packages/deployment/src/init.js

@@ -488,7 +488,7 @@ const doInit =
       config.OFFSETS[PLACEMENT] = offset;
     }
     Object.values(ROLE_INSTANCE).some(i => i > 0) ||
-      Fail`Aborting due to no nodes configured!`;
+      Fail`Aborting due to no nodes configured! (${ROLE_INSTANCE})`;
 
     await wr.createFile(
       `vars.tf`,

packages/solo/src/init-basedir.js

@@ -55,7 +55,7 @@ export default function initBasedir(
     `${JSON.stringify(connections)}\n`,
   );
   const dstHtmldir = path.join(basedir, 'html');
-  fs.mkdirSync(dstHtmldir);
+  fs.mkdirSync(dstHtmldir, { recursive: true });
 
   // Save the configuration options.
   fs.writeFileSync(path.join(basedir, 'options.json'), JSON.stringify(options));

scripts/run-deployment-integration.sh

@@ -1,29 +1,34 @@
 #!/bin/sh
 set -xueo pipefail
 
-SDK_REAL_DIR="$(cd "$(dirname "$(readlink -f -- "$0")")/.." > /dev/null && pwd -P)"
+SDK_SRC="$(cd "$(dirname "$(readlink -f -- "$0")")/.." > /dev/null && pwd -P)"
+export SDK_SRC
 
-# For some reason something in the integration script
-# relies on the SDK being at that location
 # Set AGORIC_SDK_PATH to the SDK path on the host if this
 # script is running inside a docker environment (and make sure to 
 # bind mount /var/run/docker.sock)
-if [ "$SDK_REAL_DIR" != "/usr/src/agoric-sdk" ]; then
-  echo 'Agoric SDK must be mounted in "/usr/src/agoric-sdk"'
-  exit 1
-fi
+export AGORIC_SDK_PATH="${AGORIC_SDK_PATH-$SDK_SRC}"
 
 export NETWORK_NAME=chaintest
 
-sudo ln -sf /usr/src/agoric-sdk/packages/deployment/bin/ag-setup-cosmos /usr/local/bin/ag-setup-cosmos
-rm -rf /usr/src/agoric-sdk/chaintest  ~/.ag-chain-cosmos/ /usr/src/testnet-load-generator/_agstate/agoric-servers/testnet-8000
+sudo ln -sf "$SDK_SRC/packages/deployment/bin/ag-setup-cosmos" /usr/local/bin/ag-setup-cosmos
 
-cd /usr/src/agoric-sdk/
+# Note: the deployment test and the loadgen test in testnet mode modify some
+# directories in $HOME so provide an empty $HOME for them.
+export HOME="$(mktemp -d -t deployment-integration-home.XXXXX)"
+
+# While it'd be great if these [tests were more hermetic](https://github.com/Agoric/agoric-sdk/issues/8059),
+# this manual runner must currently reset paths relative to the SDK to ensure
+# reproducible tests.
+rm -rf "$SDK_SRC/chaintest" "$SDK_SRC/../testnet-load-generator/_agstate/agoric-servers/testnet-8000"
+
+cd "$SDK_SRC"
 sudo ./packages/deployment/scripts/install-deps.sh
 yarn install && XSNAP_RANDOM_INIT=1 yarn build && make -C packages/cosmic-swingset/
 # change to "false" to skip extraction on success like in CI
 testfailure="unknown"
-/usr/src/agoric-sdk/packages/deployment/scripts/integration-test.sh || {
+DOCKER_VOLUMES="$AGORIC_SDK_PATH:/usr/src/agoric-sdk" \
+packages/deployment/scripts/integration-test.sh || {
   echo "Test failed!!!"
   testfailure="true"
 }
@@ -33,4 +38,4 @@ packages/deployment/scripts/capture-integration-results.sh $testfailure
 echo yes | packages/deployment/scripts/setup.sh destroy || true
 
 # Not part of CI
-/usr/src/agoric-sdk/scripts/process-integration-results.sh $NETWORK_NAME/results
+scripts/process-integration-results.sh $NETWORK_NAME/results