From 64be0aef716ab8e94aeccde9390c8abdf731c89b Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 2 Oct 2024 11:55:29 +0200 Subject: [PATCH 01/21] Refactor formatting & docs --- docker/Dockerfile | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c71e3e422f5..ad99807e068 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -70,7 +70,7 @@ ENV CARGO_HOME="${APP_HOME}/.cargo/" # Copy the entrypoint script to be used on both images COPY ./docker/entrypoint.sh /etc/zebrad/entrypoint.sh -# In this stage we build tests (without running then) +# This stage builds tests without running them. # # We also download needed dependencies for tests to work, from other images. # An entrypoint.sh is only available in this step for easier test handling with variables. @@ -116,9 +116,9 @@ RUN --mount=type=bind,source=zebrad,target=zebrad \ --mount=type=cache,target=${APP_HOME}/target/ \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry/ \ -cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace --no-run && \ -cp ${APP_HOME}/target/release/zebrad /usr/local/bin && \ -cp ${APP_HOME}/target/release/zebra-checkpoints /usr/local/bin + cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace --no-run && \ + cp ${APP_HOME}/target/release/zebrad /usr/local/bin && \ + cp ${APP_HOME}/target/release/zebra-checkpoints /usr/local/bin # Copy the lightwalletd binary and source files to be able to run tests COPY --from=electriccoinco/lightwalletd:latest /usr/local/bin/lightwalletd /usr/local/bin/ @@ -161,13 +161,11 @@ RUN --mount=type=bind,source=tower-batch-control,target=tower-batch-control \ --mount=type=cache,target=${APP_HOME}/target/ \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry/ \ -cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad && \ -cp ${APP_HOME}/target/release/zebrad /usr/local/bin + cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad && \ + cp ${APP_HOME}/target/release/zebrad /usr/local/bin -# This stage is only used when deploying nodes or when only the resulting zebrad binary is needed -# -# To save space, this step starts from scratch using debian, and only adds the resulting -# binary from the `release` stage +# This step starts from scratch using Debian and only adds the resulting binary +# from the `release` stage. FROM debian:bookworm-slim AS runtime # Set the default path for the zebrad binary From a248b14f2b7b5f0a4defbf037e3d2c4a35919677 Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 2 Oct 2024 12:00:17 +0200 Subject: [PATCH 02/21] Refactor the `runtime` stage in Dockerfile --- docker/Dockerfile | 78 ++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ad99807e068..1523e20a1e5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -67,8 +67,7 @@ ENV SHORT_SHA=${SHORT_SHA:-} ENV CARGO_HOME="${APP_HOME}/.cargo/" -# Copy the entrypoint script to be used on both images -COPY ./docker/entrypoint.sh /etc/zebrad/entrypoint.sh +COPY ./docker/entrypoint.sh /usr/local/bin/entrypoint.sh # This stage builds tests without running them. # @@ -131,7 +130,7 @@ ARG EXPERIMENTAL_FEATURES="journald prometheus filter-reload" ENV ENTRYPOINT_FEATURES_EXPERIMENTAL="${ENTRYPOINT_FEATURES} ${EXPERIMENTAL_FEATURES}" # By default, runs the entrypoint tests specified by the environmental variables (if any are set) -ENTRYPOINT [ "/etc/zebrad/entrypoint.sh" ] +ENTRYPOINT [ "entrypoint.sh" ] # In this stage we build a release (generate the zebrad binary) # @@ -168,59 +167,40 @@ RUN --mount=type=bind,source=tower-batch-control,target=tower-batch-control \ # from the `release` stage. FROM debian:bookworm-slim AS runtime -# Set the default path for the zebrad binary -ARG APP_HOME -ENV APP_HOME=${APP_HOME} -WORKDIR ${APP_HOME} - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ +RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ - rocksdb-tools \ - gosu \ - && rm -rf /var/lib/apt/lists/* /tmp/* + gosu + +RUN rm -rf /var/lib/apt/lists/* /tmp/* + +COPY --from=release /usr/local/bin/zebrad /usr/local/bin/ +COPY --from=release /usr/local/bin/entrypoint.sh /usr/local/bin/ + +EXPOSE 8233 18233 -# Create a non-privileged user that the app will run under. -# Running as root inside the container is running as root in the Docker host -# If an attacker manages to break out of the container, they will have root access to the host -# See https://docs.docker.com/go/dockerfile-user-best-practices/ -ARG USER=zebra -ENV USER=${USER} -ARG UID=10001 -ENV UID=${UID} -ARG GID=10001 -ENV GID=${GID} - -RUN addgroup --system --gid ${GID} ${USER} \ - && adduser \ - --system \ - --disabled-login \ - --shell /bin/bash \ - --home ${APP_HOME} \ - --uid "${UID}" \ - --gid "${GID}" \ - ${USER} - -# Config settings for zebrad ARG FEATURES ENV FEATURES=${FEATURES} -# Path and name of the config file -# These are set to a default value when not defined in the environment -ENV ZEBRA_CONF_DIR=${ZEBRA_CONF_DIR:-/etc/zebrad} -ENV ZEBRA_CONF_FILE=${ZEBRA_CONF_FILE:-zebrad.toml} +# Create a non-privileged user for running `zebrad`. +# +# ## Security +# +# If an attacker manages to break out of the container, they will have root +# access to the host. [1][1] +# +# [1]: https://docs.docker.com/go/dockerfile-user-best-practices/ +ENV USER="zebra" + +RUN adduser --system ${USER} -RUN mkdir -p ${ZEBRA_CONF_DIR} && chown ${UID}:${UID} ${ZEBRA_CONF_DIR} \ - && chown ${UID}:${UID} ${APP_HOME} +ENV ZEBRA_CONF_DIR="/etc/zebrad" +ENV ZEBRA_CACHED_STATE_DIR="/var/cache/zebrad" -COPY --from=release /usr/local/bin/zebrad /usr/local/bin -COPY --from=release /etc/zebrad/entrypoint.sh /etc/zebrad +RUN mkdir -p ${ZEBRA_CONF_DIR} && chown ${USER} ${ZEBRA_CONF_DIR} +# TODO: Shorten `ZEBRA_CACHED_STATE_DIR` to `ZEBRA_CACHE_DIR` +RUN mkdir -p ${ZEBRA_CACHED_STATE_DIR} && chown ${USER} ${ZEBRA_CACHED_STATE_DIR} -# Expose configured ports -EXPOSE 8233 18233 +USER $USER -# Update the config file based on the Docker run variables, -# and launch zebrad with it -ENTRYPOINT [ "/etc/zebrad/entrypoint.sh" ] +ENTRYPOINT [ "entrypoint.sh" ] CMD ["zebrad"] From ca1620cfa9556b81dea0cd1d751b9646c3406bd5 Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 9 Oct 2024 10:00:28 +0200 Subject: [PATCH 03/21] Remove unused code from `entrypoint.sh` --- docker/entrypoint.sh | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index d71be57805d..bacc52c5593 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -50,34 +50,6 @@ if [[ -z "${RPC_PORT}" ]]; then fi fi -#### -# Test Variables -# These variables are used to run tests in the Dockerfile. -#### - -: "${RUN_ALL_TESTS:=}" -: "${RUN_ALL_EXPERIMENTAL_TESTS:=}" -: "${TEST_FAKE_ACTIVATION_HEIGHTS:=}" -: "${TEST_ZEBRA_EMPTY_SYNC:=}" -: "${ZEBRA_TEST_LIGHTWALLETD:=}" -: "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES:=}" -: "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES:=}" -: "${TEST_DISK_REBUILD:=}" -: "${TEST_UPDATE_SYNC:=}" -: "${TEST_CHECKPOINT_SYNC:=}" -: "${GENERATE_CHECKPOINTS_MAINNET:=}" -: "${GENERATE_CHECKPOINTS_TESTNET:=}" -: "${TEST_LWD_RPC_CALL:=}" -: "${TEST_LWD_FULL_SYNC:=}" -: "${TEST_LWD_UPDATE_SYNC:=}" -: "${TEST_LWD_GRPC:=}" -: "${TEST_LWD_TRANSACTIONS:=}" -: "${TEST_GET_BLOCK_TEMPLATE:=}" -: "${TEST_SUBMIT_BLOCK:=}" -: "${TEST_SCAN_START_WHERE_LEFT:=}" -: "${ENTRYPOINT_FEATURES:=}" -: "${TEST_SCAN_TASK_COMMANDS:=}" - # Configuration file path if [[ -n "${ZEBRA_CONF_DIR}" ]] && [[ -n "${ZEBRA_CONF_FILE}" ]] && [[ -z "${ZEBRA_CONF_PATH}" ]]; then ZEBRA_CONF_PATH="${ZEBRA_CONF_DIR}/${ZEBRA_CONF_FILE}" From ea8b11945c303e45e33ee4f60197c9d33915806d Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 9 Oct 2024 10:01:38 +0200 Subject: [PATCH 04/21] Simplify `entrypoint.sh` setup --- docker/entrypoint.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index bacc52c5593..f97820b19e1 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -8,12 +8,8 @@ # 2. Configuration Management: Dynamically generates the `zebrad.toml` configuration file based on environment variables, ensuring the node starts with the desired settings. # 3. Test Execution: Can run a series of tests to validate functionality based on specified environment variables. # 4. Node Startup: Starts the node, allowing it to begin its operations. -# -# Exit if a command fails -set -e -# Exit if any command in a pipeline fails -set -o pipefail +set -eo pipefail #### # General Variables From 7cba8cf278fdc90e48a1c34a9ea9a60b58924be7 Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 9 Oct 2024 11:19:45 +0200 Subject: [PATCH 05/21] Revise docs & formatting --- docker/entrypoint.sh | 75 ++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f97820b19e1..f241a7c55dd 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -# This script serves as the entrypoint for the Zebra Docker container. -# -# Description: # This script serves as the primary entrypoint for the Docker container. Its main responsibilities include: # 1. Environment Setup: Prepares the environment by setting various flags and parameters. # 2. Configuration Management: Dynamically generates the `zebrad.toml` configuration file based on environment variables, ensuring the node starts with the desired settings. @@ -37,7 +34,7 @@ set -eo pipefail : "${TRACING_ENDPOINT_PORT:=3000}" # [rpc] : "${RPC_LISTEN_ADDR:=0.0.0.0}" -# if ${RPC_PORT} is not set, use the default value for the current network +# if `${RPC_PORT}` is not set, use the default value for the current network if [[ -z "${RPC_PORT}" ]]; then if [[ "${NETWORK}" = "Mainnet" ]]; then : "${RPC_PORT:=8232}" @@ -68,57 +65,60 @@ listen_addr = "${ZEBRA_LISTEN_ADDR}" cache_dir = "${ZEBRA_CACHED_STATE_DIR}" EOF - if [[ " ${FEATURES} " =~ " prometheus " ]]; then # spaces are important here to avoid partial matches - cat <> "${ZEBRA_CONF_PATH}" + # Spaces are important here to avoid partial matches. + if [[ " ${FEATURES} " =~ " prometheus " ]]; then + cat <>"${ZEBRA_CONF_PATH}" [metrics] endpoint_addr = "${METRICS_ENDPOINT_ADDR}:${METRICS_ENDPOINT_PORT}" EOF - fi + fi - if [[ -n "${RPC_PORT}" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + if [[ -n "${RPC_PORT}" ]]; then + cat <>"${ZEBRA_CONF_PATH}" [rpc] listen_addr = "${RPC_LISTEN_ADDR}:${RPC_PORT}" EOF - fi + fi - if [[ -n "${LOG_FILE}" ]] || [[ -n "${LOG_COLOR}" ]] || [[ -n "${TRACING_ENDPOINT_ADDR}" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + if [[ -n "${LOG_FILE}" ]] || [[ -n "${LOG_COLOR}" ]] || [[ -n "${TRACING_ENDPOINT_ADDR}" ]]; then + cat <>"${ZEBRA_CONF_PATH}" [tracing] EOF - if [[ " ${FEATURES} " =~ " filter-reload " ]]; then # spaces are important here to avoid partial matches - cat <> "${ZEBRA_CONF_PATH}" + # Spaces are important here to avoid partial matches. + if [[ " ${FEATURES} " =~ " filter-reload " ]]; then + cat <>"${ZEBRA_CONF_PATH}" endpoint_addr = "${TRACING_ENDPOINT_ADDR}:${TRACING_ENDPOINT_PORT}" EOF - fi - # Set this to log to a file, if not set, logs to standard output - if [[ -n "${LOG_FILE}" ]]; then - mkdir -p "$(dirname "${LOG_FILE}")" - cat <> "${ZEBRA_CONF_PATH}" + fi + # Set this to log to a file, if not set, logs to standard output. + if [[ -n "${LOG_FILE}" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" + cat <>"${ZEBRA_CONF_PATH}" log_file = "${LOG_FILE}" EOF - fi - # Zebra automatically detects if it is attached to a terminal, and uses colored output. - # Set this to 'true' to force using color even if the output is not a terminal. - # Set this to 'false' to disable using color even if the output is a terminal. - if [[ "${LOG_COLOR}" = "true" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + fi + # Zebra automatically detects if it is attached to a terminal, and uses colored output. + # Set this to 'true' to force using color even if the output is not a terminal. + # Set this to 'false' to disable using color even if the output is a terminal. + if [[ "${LOG_COLOR}" = "true" ]]; then + cat <>"${ZEBRA_CONF_PATH}" force_use_color = true EOF - elif [[ "${LOG_COLOR}" = "false" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + elif [[ "${LOG_COLOR}" = "false" ]]; then + cat <>"${ZEBRA_CONF_PATH}" use_color = false EOF + fi fi - fi - if [[ -n "${MINER_ADDRESS}" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + if [[ -n "${MINER_ADDRESS}" ]]; then + cat <>"${ZEBRA_CONF_PATH}" [mining] miner_address = "${MINER_ADDRESS}" EOF + fi fi -fi +} if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then # Print the config file @@ -126,22 +126,19 @@ if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then cat "${ZEBRA_CONF_PATH}" fi -# Function to list directory +# Checks if a directory contains subdirectories check_directory_files() { local dir="$1" # Check if the directory exists if [[ -d "${dir}" ]]; then # Check if there are any subdirectories if find "${dir}" -mindepth 1 -type d | read -r; then - # Subdirectories exist, so we continue : else - # No subdirectories, print message and exit with status 1 echo "No subdirectories found in ${dir}." exit 1 fi else - # Directory doesn't exist, print message and exit with status 1 echo "Directory ${dir} does not exist." exit 1 fi @@ -163,8 +160,12 @@ run_cargo_test() { fi done - # Run the command using eval, this will replace the current process with the cargo command - eval "${cmd}" || { echo "Cargo test failed"; exit 1; } + # Run the command using eval. This will replace the current process with the + # cargo command. + eval "${cmd}" || { + echo "Cargo test failed" + exit 1 + } } # Main Execution Logic: From 56c65e6141bf163f1667d3ba1864b08bbd2d5627 Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 9 Oct 2024 11:20:31 +0200 Subject: [PATCH 06/21] Adjust default values for env vars --- docker/entrypoint.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f241a7c55dd..0797fc6801e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -13,18 +13,17 @@ set -eo pipefail # These variables are used to run the Zebra node. #### -# Path and name of the config file. These two have defaults set in the Dockerfile. +# Path and name of the config file. : "${ZEBRA_CONF_DIR:=}" -: "${ZEBRA_CONF_FILE:=}" +: "${ZEBRA_CONF_FILE:=zebrad.toml}" # [network] : "${NETWORK:=Mainnet}" : "${ZEBRA_LISTEN_ADDR:=0.0.0.0}" # [consensus] : "${ZEBRA_CHECKPOINT_SYNC:=true}" # [state] -# Set this to change the default cached state directory -: "${ZEBRA_CACHED_STATE_DIR:=/var/cache/zebrad-cache}" -: "${LIGHTWALLETD_DATA_DIR:=/var/cache/lwd-cache}" +: "${ZEBRA_CACHED_STATE_DIR:=/var/cache/zebrad}" +: "${LIGHTWALLETD_DATA_DIR:=/var/cache/lwd}" # [metrics] : "${METRICS_ENDPOINT_ADDR:=0.0.0.0}" : "${METRICS_ENDPOINT_PORT:=9999}" From be381320d2918b390b34ab772e6854f4bc467017 Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 9 Oct 2024 19:38:02 +0200 Subject: [PATCH 07/21] Bump Rust v from 1.79 to 1.81 in Dockerfile --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1523e20a1e5..3c86e91751e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,7 +21,8 @@ ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints" ARG EXPERIMENTAL_FEATURES="" ARG APP_HOME="/opt/zebrad" -ARG RUST_VERSION=1.79.0 +ARG RUST_VERSION=1.81.0 + # In this stage we download all system requirements to build the project # # It also captures all the build arguments to be used as environment variables. From 0492b7a9f33bc6301cc99c232cf451f9984d3130 Mon Sep 17 00:00:00 2001 From: Marek Date: Thu, 10 Oct 2024 12:39:30 +0200 Subject: [PATCH 08/21] Refactor `entrypoint.sh` --- docker/entrypoint.sh | 537 +++++++++++++++++++++++-------------------- 1 file changed, 289 insertions(+), 248 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 0797fc6801e..46d1f3a8460 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,62 +1,75 @@ #!/usr/bin/env bash -# This script serves as the primary entrypoint for the Docker container. Its main responsibilities include: -# 1. Environment Setup: Prepares the environment by setting various flags and parameters. -# 2. Configuration Management: Dynamically generates the `zebrad.toml` configuration file based on environment variables, ensuring the node starts with the desired settings. -# 3. Test Execution: Can run a series of tests to validate functionality based on specified environment variables. -# 4. Node Startup: Starts the node, allowing it to begin its operations. +# Entrypoint for the Docker container. +# +# The main execution logic is at the bottom. +# +# ## Notes +# +# - If `$ZEBRA_CONF_PATH` is set, it must point to a Zebra conf file readable by +# `$USER`. +# - If `$ZEBRA_CONF_DIR` is set, it must point to a dir writable by `$USER`. +# - If `$ZEBRA_CACHED_STATE_DIR` or `$LIGHTWALLETD_DATA_DIR` are set, they must +# point to dirs writable by `$USER`. If they're not set, `/var/cache/zebrad` +# and `var/cache/lwd` must be writable by `$USER`. set -eo pipefail -#### -# General Variables -# These variables are used to run the Zebra node. -#### - -# Path and name of the config file. -: "${ZEBRA_CONF_DIR:=}" -: "${ZEBRA_CONF_FILE:=zebrad.toml}" -# [network] -: "${NETWORK:=Mainnet}" -: "${ZEBRA_LISTEN_ADDR:=0.0.0.0}" -# [consensus] -: "${ZEBRA_CHECKPOINT_SYNC:=true}" -# [state] -: "${ZEBRA_CACHED_STATE_DIR:=/var/cache/zebrad}" -: "${LIGHTWALLETD_DATA_DIR:=/var/cache/lwd}" -# [metrics] -: "${METRICS_ENDPOINT_ADDR:=0.0.0.0}" -: "${METRICS_ENDPOINT_PORT:=9999}" -# [tracing] -: "${LOG_COLOR:=false}" -: "${TRACING_ENDPOINT_ADDR:=0.0.0.0}" -: "${TRACING_ENDPOINT_PORT:=3000}" -# [rpc] -: "${RPC_LISTEN_ADDR:=0.0.0.0}" -# if `${RPC_PORT}` is not set, use the default value for the current network -if [[ -z "${RPC_PORT}" ]]; then - if [[ "${NETWORK}" = "Mainnet" ]]; then - : "${RPC_PORT:=8232}" - elif [[ "${NETWORK}" = "Testnet" ]]; then - : "${RPC_PORT:=18232}" +# Sets default values for various env vars. +# +# Takes no arguments. +prepare_env_vars() { + # [network] + : "${NETWORK:=Mainnet}" + : "${ZEBRA_LISTEN_ADDR:=0.0.0.0}" + + # [consensus] + : "${ZEBRA_CHECKPOINT_SYNC:=true}" + + # [state] + : "${ZEBRA_CACHED_STATE_DIR:=/var/cache/zebrad}" + : "${LIGHTWALLETD_DATA_DIR:=/var/cache/lwd}" + + # [metrics] + : "${METRICS_ENDPOINT_ADDR:=0.0.0.0}" + : "${METRICS_ENDPOINT_PORT:=9999}" + + # [tracing] + : "${LOG_COLOR:=false}" + : "${TRACING_ENDPOINT_ADDR:=0.0.0.0}" + : "${TRACING_ENDPOINT_PORT:=3000}" + + # [rpc] + : "${RPC_LISTEN_ADDR:=0.0.0.0}" + + # if `${RPC_PORT}` is not set, use the default value for the current network + if [[ -z "${RPC_PORT}" ]]; then + if [[ "${NETWORK}" = "Mainnet" ]]; then + : "${RPC_PORT:=8232}" + elif [[ "${NETWORK}" = "Testnet" ]]; then + : "${RPC_PORT:=18232}" + fi fi -fi -# Configuration file path -if [[ -n "${ZEBRA_CONF_DIR}" ]] && [[ -n "${ZEBRA_CONF_FILE}" ]] && [[ -z "${ZEBRA_CONF_PATH}" ]]; then - ZEBRA_CONF_PATH="${ZEBRA_CONF_DIR}/${ZEBRA_CONF_FILE}" -fi + echo "Using the following env vars:" + echo "" + printenv + echo "" +} -# Populate `zebrad.toml` before starting zebrad, using the environmental -# variables set by the Dockerfile or the user. If the user has already created a config, don't replace it. +# Populates the config file for Zebra, using the env vars set by the Dockerfile +# or user. # -# We disable most ports by default, so the default config is secure. -# Users have to opt-in to additional functionality by setting environmental variables. -if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ ! -f "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then - # Create the conf path and file - (mkdir -p "$(dirname "${ZEBRA_CONF_PATH}")" && touch "${ZEBRA_CONF_PATH}") || { echo "Error creating file ${ZEBRA_CONF_PATH}"; exit 1; } - # Populate the conf file - cat < "${ZEBRA_CONF_PATH}" +# The default settings are minimal. Users have to opt-in to additional +# functionality by setting the environment variables. +# +# Also prints the content of the generated config file. +# +# ## Positional Parameters +# +# - "$1": file to write the config to +prepare_default_conf_file() { + cat <"$1" [network] network = "${NETWORK}" listen_addr = "${ZEBRA_LISTEN_ADDR}" @@ -64,68 +77,68 @@ listen_addr = "${ZEBRA_LISTEN_ADDR}" cache_dir = "${ZEBRA_CACHED_STATE_DIR}" EOF - # Spaces are important here to avoid partial matches. - if [[ " ${FEATURES} " =~ " prometheus " ]]; then - cat <>"${ZEBRA_CONF_PATH}" + # Spaces are important here to avoid partial matches. + if [[ " ${FEATURES} " =~ " prometheus " ]]; then + cat <>"$1" [metrics] endpoint_addr = "${METRICS_ENDPOINT_ADDR}:${METRICS_ENDPOINT_PORT}" EOF - fi + fi - if [[ -n "${RPC_PORT}" ]]; then - cat <>"${ZEBRA_CONF_PATH}" + if [[ -n "${RPC_PORT}" ]]; then + cat <>"$1" [rpc] listen_addr = "${RPC_LISTEN_ADDR}:${RPC_PORT}" EOF - fi + fi - if [[ -n "${LOG_FILE}" ]] || [[ -n "${LOG_COLOR}" ]] || [[ -n "${TRACING_ENDPOINT_ADDR}" ]]; then - cat <>"${ZEBRA_CONF_PATH}" + if [[ -n "${LOG_FILE}" ]] || [[ -n "${LOG_COLOR}" ]] || [[ -n "${TRACING_ENDPOINT_ADDR}" ]]; then + cat <>"$1" [tracing] EOF - # Spaces are important here to avoid partial matches. - if [[ " ${FEATURES} " =~ " filter-reload " ]]; then - cat <>"${ZEBRA_CONF_PATH}" + # Spaces are important here to avoid partial matches. + if [[ " ${FEATURES} " =~ " filter-reload " ]]; then + cat <>"$1" endpoint_addr = "${TRACING_ENDPOINT_ADDR}:${TRACING_ENDPOINT_PORT}" EOF - fi - # Set this to log to a file, if not set, logs to standard output. - if [[ -n "${LOG_FILE}" ]]; then - mkdir -p "$(dirname "${LOG_FILE}")" - cat <>"${ZEBRA_CONF_PATH}" + fi + # Set this to log to a file, if not set, logs to standard output. + if [[ -n "${LOG_FILE}" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" + cat <>"$1" log_file = "${LOG_FILE}" EOF - fi - # Zebra automatically detects if it is attached to a terminal, and uses colored output. - # Set this to 'true' to force using color even if the output is not a terminal. - # Set this to 'false' to disable using color even if the output is a terminal. - if [[ "${LOG_COLOR}" = "true" ]]; then - cat <>"${ZEBRA_CONF_PATH}" + fi + # Zebra automatically detects if it is attached to a terminal, and uses colored output. + # Set this to 'true' to force using color even if the output is not a terminal. + # Set this to 'false' to disable using color even if the output is a terminal. + if [[ "${LOG_COLOR}" = "true" ]]; then + cat <>"$1" force_use_color = true EOF - elif [[ "${LOG_COLOR}" = "false" ]]; then - cat <>"${ZEBRA_CONF_PATH}" + elif [[ "${LOG_COLOR}" = "false" ]]; then + cat <>"$1" use_color = false EOF - fi fi + fi - if [[ -n "${MINER_ADDRESS}" ]]; then - cat <>"${ZEBRA_CONF_PATH}" + if [[ -n "${MINER_ADDRESS}" ]]; then + cat <>"$1" [mining] miner_address = "${MINER_ADDRESS}" EOF - fi fi -} -if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then - # Print the config file - echo "Using zebrad.toml:" - cat "${ZEBRA_CONF_PATH}" -fi + echo "Prepared the following Zebra config:" + echo "" + cat "$1" + echo "" +} # Checks if a directory contains subdirectories +# +# Exits with 0 if it does, and 1 otherwise. check_directory_files() { local dir="$1" # Check if the directory exists @@ -143,9 +156,18 @@ check_directory_files() { fi } -# Function to run cargo test with an arbitrary number of arguments +# Runs cargo test with an arbitrary number of arguments. +# +# ## Positional Parameters +# +# - '$1' must contain cargo FEATURES as desribed here: +# https://doc.rust-lang.org/cargo/reference/features.html#command-line-feature-options, +# or be empty +# - the remaining params will be appended to a command starting with `exec cargo +# test ... -- ...` run_cargo_test() { - # Start constructing the command, ensuring that $1 is enclosed in single quotes as it's a feature list + # Start constructing the command, ensuring that $1 is enclosed in single + # quotes as it's a feature list local cmd="exec cargo test --locked --release --features '$1' --package zebrad --test acceptance -- --nocapture --include-ignored" # Shift the first argument, as it's already included in the cmd @@ -167,167 +189,186 @@ run_cargo_test() { } } -# Main Execution Logic: -# This script orchestrates the execution flow based on the provided arguments and environment variables. -# - If "$1" is '--', '-', or 'zebrad', the script processes the subsequent arguments for the 'zebrad' command. -# - If ENTRYPOINT_FEATURES is unset, it checks for ZEBRA_CONF_PATH. If set, 'zebrad' runs with this custom configuration; otherwise, it runs with the provided arguments. -# - If "$1" is an empty string and ENTRYPOINT_FEATURES is set, the script enters the testing phase, checking various environment variables to determine the specific tests to run. -# - Different tests or operations are triggered based on the respective conditions being met. -# - If "$1" doesn't match any of the above, it's assumed to be a command, which is executed directly. -# This structure ensures a flexible execution strategy, accommodating various scenarios such as custom configurations, different testing phases, or direct command execution. +# Runs tests depending on the env vars. +run_tests() { + # Validate the test variables. For these tests, we activate the test features + # to avoid recompiling `zebrad`, but we don't actually run any gRPC tests. + if [[ "${RUN_ALL_TESTS}" -eq "1" ]]; then + # Run unit, basic acceptance tests, and ignored tests, only showing command + # output if the test fails. If the lightwalletd environment variables are + # set, we will also run those tests. + exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" \ + --workspace -- --nocapture --include-ignored + + elif [[ "${RUN_ALL_EXPERIMENTAL_TESTS}" -eq "1" ]]; then + # Run unit, basic acceptance tests, and ignored tests with experimental + # features. If the lightwalletd environment variables are set, we will + # also run those tests. + exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES_EXPERIMENTAL}" \ + --workspace -- --nocapture --include-ignored + + elif [[ "${TEST_FAKE_ACTIVATION_HEIGHTS}" -eq "1" ]]; then + # Run state tests with fake activation heights. + exec cargo test --locked --release --features "zebra-test" --package zebra-state \ + --lib -- --nocapture --include-ignored with_fake_activation_heights + + elif [[ "${TEST_ZEBRA_EMPTY_SYNC}" -eq "1" ]]; then + # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state. + run_cargo_test "${ENTRYPOINT_FEATURES}" "sync_large_checkpoints_" + + elif [[ "${ZEBRA_TEST_LIGHTWALLETD}" -eq "1" ]]; then + # Test launching lightwalletd with an empty lightwalletd and Zebra state. + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_integration" + + elif [[ -n "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES}" ]]; then + # Run a Zebra full sync test on mainnet. + run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_mainnet" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + + elif [[ -n "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES}" ]]; then + # Run a Zebra full sync test on testnet. + run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_testnet" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + + elif [[ "${TEST_DISK_REBUILD}" -eq "1" ]]; then + # Run a Zebra sync up to the mandatory checkpoint. + # + # TODO: use environment variables instead of Rust features (part of #2995) + run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" \ + "sync_to_mandatory_checkpoint_${NETWORK,,}" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + + elif [[ "${TEST_UPDATE_SYNC}" -eq "1" ]]; then + # Run a Zebra sync starting at the cached tip, and syncing to the latest tip. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "zebrad_update_sync" + + elif [[ "${TEST_CHECKPOINT_SYNC}" -eq "1" ]]; then + # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + # TODO: use environment variables instead of Rust features (part of #2995) + run_cargo_test "test_sync_past_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" \ + "sync_past_mandatory_checkpoint_${NETWORK,,}" + + elif [[ "${GENERATE_CHECKPOINTS_MAINNET}" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra from a cached state on mainnet. + # + # TODO: disable or filter out logs like: + # test generate_checkpoints_mainnet has been running for over 60 seconds + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_mainnet" + + elif [[ "${GENERATE_CHECKPOINTS_TESTNET}" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra on testnet. + # + # This test might fail if testnet is unstable. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_testnet" + + elif [[ "${TEST_LWD_RPC_CALL}" -eq "1" ]]; then + # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + # Run both the fully synced RPC test and the subtree snapshot test, one test + # at a time. Since these tests use the same cached state, a state problem in + # the first test can fail the second test. + run_cargo_test "${ENTRYPOINT_FEATURES}" "--test-threads" "1" "fully_synced_rpc_" + + elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then + # Starting at a cached Zebra tip, run a lightwalletd sync to tip. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_full_sync" + check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" + + elif [[ "${TEST_LWD_UPDATE_SYNC}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_update_sync" + + # These tests actually use gRPC. + elif [[ "${TEST_LWD_GRPC}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to + # lightwalletd, which calls Zebra. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_wallet_grpc_tests" + + elif [[ "${TEST_LWD_TRANSACTIONS}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, test sending + # transactions gRPC call to lightwalletd, which calls Zebra. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" + run_cargo_test "${ENTRYPOINT_FEATURES}" "sending_transactions_using_lightwalletd" + + # These tests use mining code, but don't use gRPC. + elif [[ "${TEST_GET_BLOCK_TEMPLATE}" -eq "1" ]]; then + # Starting with a cached Zebra tip, test getting a block template from + # Zebra's RPC server. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "get_block_template" + + elif [[ "${TEST_SUBMIT_BLOCK}" -eq "1" ]]; then + # Starting with a cached Zebra tip, test sending a block to Zebra's RPC + # port. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "submit_block" + + elif [[ "${TEST_SCAN_START_WHERE_LEFT}" -eq "1" ]]; then + # Test that the scanner can continue scanning where it was left when + # zebra-scanner restarts. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + exec cargo test --locked --release --features "zebra-test" --package zebra-scan \ + -- --nocapture --include-ignored scan_start_where_left + + elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then + # Test that the scan task commands are working. + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + exec cargo test --locked --release --features "zebra-test" --package zebra-scan \ + -- --nocapture --include-ignored scan_task_commands + fi +} +# Main Execution Logic +# +# - If "$1" is "--", "-", or "zebrad", run `zebrad` with the remaining params, +# and: +# - if `$ZEBRA_CONF_PATH` points to a file, use it for `zebrad` as the config +# file, +# - else if `$ZEBRA_CONF_DIR` points to a dir, generate a default config file +# and use it for `zebrad`. +# - If "$1" is "tests", run tests. +# - TODO: If "$1" is "monitoring", start a monitoring node. +# - If "$1" doesn't match any of the above, run "$@" directly. case "$1" in - --* | -* | zebrad) - shift - if [[ -n "${ZEBRA_CONF_PATH}" ]]; then - exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } - else - exec zebrad "$@" || { echo "Execution failed"; exit 1; } - fi - ;; - "") - if [[ -n "${ENTRYPOINT_FEATURES}" ]]; then - # Validate the test variables - # For these tests, we activate the test features to avoid recompiling `zebrad`, - # but we don't actually run any gRPC tests. - if [[ "${RUN_ALL_TESTS}" -eq "1" ]]; then - # Run unit, basic acceptance tests, and ignored tests, only showing command output if the test fails. - # If the lightwalletd environmental variables are set, we will also run those tests. - exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace -- --nocapture --include-ignored - - elif [[ "${RUN_ALL_EXPERIMENTAL_TESTS}" -eq "1" ]]; then - # Run unit, basic acceptance tests, and ignored tests with experimental features. - # If the lightwalletd environmental variables are set, we will also run those tests. - exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES_EXPERIMENTAL}" --workspace -- --nocapture --include-ignored - - elif [[ "${TEST_FAKE_ACTIVATION_HEIGHTS}" -eq "1" ]]; then - # Run state tests with fake activation heights. - exec cargo test --locked --release --features "zebra-test" --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights - - elif [[ "${TEST_ZEBRA_EMPTY_SYNC}" -eq "1" ]]; then - # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state. - run_cargo_test "${ENTRYPOINT_FEATURES}" "sync_large_checkpoints_" - - elif [[ "${ZEBRA_TEST_LIGHTWALLETD}" -eq "1" ]]; then - # Test launching lightwalletd with an empty lightwalletd and Zebra state. - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_integration" - - elif [[ -n "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES}" ]]; then - # Run a Zebra full sync test on mainnet. - run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_mainnet" - # List directory generated by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - - elif [[ -n "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES}" ]]; then - # Run a Zebra full sync test on testnet. - run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_testnet" - # List directory generated by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - - elif [[ "${TEST_DISK_REBUILD}" -eq "1" ]]; then - # Run a Zebra sync up to the mandatory checkpoint. - # - # TODO: use environmental variables instead of Rust features (part of #2995) - run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_to_mandatory_checkpoint_${NETWORK,,}" - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - - elif [[ "${TEST_UPDATE_SYNC}" -eq "1" ]]; then - # Run a Zebra sync starting at the cached tip, and syncing to the latest tip. - # - # List directory used by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "zebrad_update_sync" - - elif [[ "${TEST_CHECKPOINT_SYNC}" -eq "1" ]]; then - # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it. - # - # List directory used by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - # TODO: use environmental variables instead of Rust features (part of #2995) - run_cargo_test "test_sync_past_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" - - elif [[ "${GENERATE_CHECKPOINTS_MAINNET}" -eq "1" ]]; then - # Generate checkpoints after syncing Zebra from a cached state on mainnet. - # - # TODO: disable or filter out logs like: - # test generate_checkpoints_mainnet has been running for over 60 seconds - # - # List directory used by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_mainnet" - - elif [[ "${GENERATE_CHECKPOINTS_TESTNET}" -eq "1" ]]; then - # Generate checkpoints after syncing Zebra on testnet. - # - # This test might fail if testnet is unstable. - # - # List directory used by test - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_testnet" - - elif [[ "${TEST_LWD_RPC_CALL}" -eq "1" ]]; then - # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. - # Since these tests use the same cached state, a state problem in the first test can fail the second test. - run_cargo_test "${ENTRYPOINT_FEATURES}" "--test-threads" "1" "fully_synced_rpc_" - - elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then - # Starting at a cached Zebra tip, run a lightwalletd sync to tip. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_full_sync" - check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" - - elif [[ "${TEST_LWD_UPDATE_SYNC}" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_update_sync" - - # These tests actually use gRPC. - elif [[ "${TEST_LWD_GRPC}" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_wallet_grpc_tests" - - elif [[ "${TEST_LWD_TRANSACTIONS}" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" - run_cargo_test "${ENTRYPOINT_FEATURES}" "sending_transactions_using_lightwalletd" - - # These tests use mining code, but don't use gRPC. - elif [[ "${TEST_GET_BLOCK_TEMPLATE}" -eq "1" ]]; then - # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "get_block_template" - - elif [[ "${TEST_SUBMIT_BLOCK}" -eq "1" ]]; then - # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "submit_block" - - elif [[ "${TEST_SCAN_START_WHERE_LEFT}" -eq "1" ]]; then - # Test that the scanner can continue scanning where it was left when zebra-scanner restarts. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_start_where_left - - elif [[ "${TEST_SCAN_TASK_COMMANDS}" -eq "1" ]]; then - # Test that the scan task commands are working. - check_directory_files "${ZEBRA_CACHED_STATE_DIR}" - exec cargo test --locked --release --features "zebra-test" --package zebra-scan -- --nocapture --include-ignored scan_task_commands - - else - exec "$@" - fi - fi - ;; - *) - if command -v gosu >/dev/null 2>&1; then - exec gosu "$USER" "$@" - else - exec "$@" - fi - ;; +--* | -* | zebrad) + shift + + prepare_env_vars + + if [[ ! -f "${ZEBRA_CONF_PATH}" ]] && [[ -d "${ZEBRA_CONF_DIR}" ]]; then + ZEBRA_CONF_PATH="${ZEBRA_CONF_DIR}/zebrad.toml" + prepare_default_conf_file "$ZEBRA_CONF_PATH" + fi + + if [[ -f "${ZEBRA_CONF_PATH}" ]]; then + exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" + else + exec zebrad "$@" + fi + ;; +tests) + prepare_env_vars + run_tests + ;; +monitoring) + # TODO: Impl logic for starting a monitoring node. + : + ;; +*) + if command -v gosu >/dev/null 2>&1; then + exec gosu "$USER" "$@" + else + exec "$@" + fi + ;; esac From 6595740ef5391bd6d09750d66f2a53f41cebb837 Mon Sep 17 00:00:00 2001 From: Marek Date: Thu, 10 Oct 2024 12:40:33 +0200 Subject: [PATCH 09/21] Refactor `Dockerfile` --- docker/Dockerfile | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3c86e91751e..4f27fa7073a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -130,8 +130,8 @@ ENV ENTRYPOINT_FEATURES=${ENTRYPOINT_FEATURES} ARG EXPERIMENTAL_FEATURES="journald prometheus filter-reload" ENV ENTRYPOINT_FEATURES_EXPERIMENTAL="${ENTRYPOINT_FEATURES} ${EXPERIMENTAL_FEATURES}" -# By default, runs the entrypoint tests specified by the environmental variables (if any are set) -ENTRYPOINT [ "entrypoint.sh" ] +# Run tests specified by the env vars +ENTRYPOINT [ "entrypoint.sh" "test" ] # In this stage we build a release (generate the zebrad binary) # @@ -177,11 +177,6 @@ RUN rm -rf /var/lib/apt/lists/* /tmp/* COPY --from=release /usr/local/bin/zebrad /usr/local/bin/ COPY --from=release /usr/local/bin/entrypoint.sh /usr/local/bin/ -EXPOSE 8233 18233 - -ARG FEATURES -ENV FEATURES=${FEATURES} - # Create a non-privileged user for running `zebrad`. # # ## Security @@ -190,15 +185,18 @@ ENV FEATURES=${FEATURES} # access to the host. [1][1] # # [1]: https://docs.docker.com/go/dockerfile-user-best-practices/ -ENV USER="zebra" +ARG USER="zebra" +ENV USER=${USER} RUN adduser --system ${USER} -ENV ZEBRA_CONF_DIR="/etc/zebrad" -ENV ZEBRA_CACHED_STATE_DIR="/var/cache/zebrad" - +ARG ZEBRA_CONF_DIR="/etc/zebrad" +ENV ZEBRA_CONF_DIR=${ZEBRA_CONF_DIR} RUN mkdir -p ${ZEBRA_CONF_DIR} && chown ${USER} ${ZEBRA_CONF_DIR} + # TODO: Shorten `ZEBRA_CACHED_STATE_DIR` to `ZEBRA_CACHE_DIR` +ARG ZEBRA_CACHED_STATE_DIR="/var/cache/zebrad" +ENV ZEBRA_CACHED_STATE_DIR=${ZEBRA_CACHED_STATE_DIR} RUN mkdir -p ${ZEBRA_CACHED_STATE_DIR} && chown ${USER} ${ZEBRA_CACHED_STATE_DIR} USER $USER From 22dc7385f8c805d3a774a58942f49ce69cd80954 Mon Sep 17 00:00:00 2001 From: Marek Date: Thu, 10 Oct 2024 12:40:54 +0200 Subject: [PATCH 10/21] Add TODOs for monitoring stage to Dockerfile --- docker/Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4f27fa7073a..cb7f33e71c8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,11 +3,12 @@ # If you want to include a file in the Docker image, add it to .dockerignore. # -# We are using 4 stages: +# We are using 4 (TODO: 5) stages: # - deps: install build dependencies and sets the needed variables # - tests: builds tests binaries # - release: builds release binaries # - runtime: runs the release binaries +# - TODO: Add a `monitoring` stage # # We first set default values for build arguments used across the stages. # Each stage must define the build arguments (ARGs) it uses. @@ -203,3 +204,12 @@ USER $USER ENTRYPOINT [ "entrypoint.sh" ] CMD ["zebrad"] + +# TODO: Add a `monitoring` stage +# +# This stage will be based on `runtime`, and initially: +# +# - run `zebrad` on Testnet +# - with mining enabled using S-nomp and `nheqminer`. +# +# We can add further functionality to this stage for further purposes. From 621754b01e4f658abd280aff0c7d5f3f86e98d28 Mon Sep 17 00:00:00 2001 From: Marek Date: Thu, 10 Oct 2024 12:40:33 +0200 Subject: [PATCH 11/21] Refactor `Dockerfile` --- docker/Dockerfile | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3c86e91751e..797f21e3177 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -130,8 +130,8 @@ ENV ENTRYPOINT_FEATURES=${ENTRYPOINT_FEATURES} ARG EXPERIMENTAL_FEATURES="journald prometheus filter-reload" ENV ENTRYPOINT_FEATURES_EXPERIMENTAL="${ENTRYPOINT_FEATURES} ${EXPERIMENTAL_FEATURES}" -# By default, runs the entrypoint tests specified by the environmental variables (if any are set) -ENTRYPOINT [ "entrypoint.sh" ] +# Run tests specified by the env vars +ENTRYPOINT [ "entrypoint.sh", "test" ] # In this stage we build a release (generate the zebrad binary) # @@ -177,11 +177,6 @@ RUN rm -rf /var/lib/apt/lists/* /tmp/* COPY --from=release /usr/local/bin/zebrad /usr/local/bin/ COPY --from=release /usr/local/bin/entrypoint.sh /usr/local/bin/ -EXPOSE 8233 18233 - -ARG FEATURES -ENV FEATURES=${FEATURES} - # Create a non-privileged user for running `zebrad`. # # ## Security @@ -190,15 +185,18 @@ ENV FEATURES=${FEATURES} # access to the host. [1][1] # # [1]: https://docs.docker.com/go/dockerfile-user-best-practices/ -ENV USER="zebra" +ARG USER="zebra" +ENV USER=${USER} RUN adduser --system ${USER} -ENV ZEBRA_CONF_DIR="/etc/zebrad" -ENV ZEBRA_CACHED_STATE_DIR="/var/cache/zebrad" - +ARG ZEBRA_CONF_DIR="/etc/zebrad" +ENV ZEBRA_CONF_DIR=${ZEBRA_CONF_DIR} RUN mkdir -p ${ZEBRA_CONF_DIR} && chown ${USER} ${ZEBRA_CONF_DIR} + # TODO: Shorten `ZEBRA_CACHED_STATE_DIR` to `ZEBRA_CACHE_DIR` +ARG ZEBRA_CACHED_STATE_DIR="/var/cache/zebrad" +ENV ZEBRA_CACHED_STATE_DIR=${ZEBRA_CACHED_STATE_DIR} RUN mkdir -p ${ZEBRA_CACHED_STATE_DIR} && chown ${USER} ${ZEBRA_CACHED_STATE_DIR} USER $USER From 6b685927df3a7f01ebbf634fe330dbcc478f0b7c Mon Sep 17 00:00:00 2001 From: Marek Date: Thu, 10 Oct 2024 12:40:54 +0200 Subject: [PATCH 12/21] Add TODOs for monitoring stage to Dockerfile --- docker/Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 797f21e3177..27a098ab2ec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,11 +3,12 @@ # If you want to include a file in the Docker image, add it to .dockerignore. # -# We are using 4 stages: +# We are using 4 (TODO: 5) stages: # - deps: install build dependencies and sets the needed variables # - tests: builds tests binaries # - release: builds release binaries # - runtime: runs the release binaries +# - TODO: Add a `monitoring` stage # # We first set default values for build arguments used across the stages. # Each stage must define the build arguments (ARGs) it uses. @@ -203,3 +204,12 @@ USER $USER ENTRYPOINT [ "entrypoint.sh" ] CMD ["zebrad"] + +# TODO: Add a `monitoring` stage +# +# This stage will be based on `runtime`, and initially: +# +# - run `zebrad` on Testnet +# - with mining enabled using S-nomp and `nheqminer`. +# +# We can add further functionality to this stage for further purposes. From 38837e4cb714a75635cca64300d725e4eb9d997a Mon Sep 17 00:00:00 2001 From: Marek Date: Thu, 10 Oct 2024 13:46:22 +0200 Subject: [PATCH 13/21] Fix a typo --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 27a098ab2ec..9ab6fda4d78 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -132,7 +132,7 @@ ARG EXPERIMENTAL_FEATURES="journald prometheus filter-reload" ENV ENTRYPOINT_FEATURES_EXPERIMENTAL="${ENTRYPOINT_FEATURES} ${EXPERIMENTAL_FEATURES}" # Run tests specified by the env vars -ENTRYPOINT [ "entrypoint.sh", "test" ] +ENTRYPOINT [ "entrypoint.sh", "tests" ] # In this stage we build a release (generate the zebrad binary) # From 2ada296260678a6fd824e0b103520f4ee30f0227 Mon Sep 17 00:00:00 2001 From: Marek Date: Fri, 11 Oct 2024 12:44:43 +0200 Subject: [PATCH 14/21] Allow running `zebrad` in test mode --- docker/Dockerfile | 2 +- docker/entrypoint.sh | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9ab6fda4d78..27a098ab2ec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -132,7 +132,7 @@ ARG EXPERIMENTAL_FEATURES="journald prometheus filter-reload" ENV ENTRYPOINT_FEATURES_EXPERIMENTAL="${ENTRYPOINT_FEATURES} ${EXPERIMENTAL_FEATURES}" # Run tests specified by the env vars -ENTRYPOINT [ "entrypoint.sh", "tests" ] +ENTRYPOINT [ "entrypoint.sh", "test" ] # In this stage we build a release (generate the zebrad binary) # diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 46d1f3a8460..9414f6e3478 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -190,6 +190,10 @@ run_cargo_test() { } # Runs tests depending on the env vars. +# +# Positional Parameters +# +# - $@: Arbitrary command that will be executed if no test env var is set. run_tests() { # Validate the test variables. For these tests, we activate the test features # to avoid recompiling `zebrad`, but we don't actually run any gRPC tests. @@ -325,6 +329,8 @@ run_tests() { check_directory_files "${ZEBRA_CACHED_STATE_DIR}" exec cargo test --locked --release --features "zebra-test" --package zebra-scan \ -- --nocapture --include-ignored scan_task_commands + else + exec "$@" fi } @@ -356,9 +362,12 @@ case "$1" in exec zebrad "$@" fi ;; -tests) +test) + shift + prepare_env_vars - run_tests + + run_tests "$@" ;; monitoring) # TODO: Impl logic for starting a monitoring node. From 69b03d4e18f88af3ef4c10ebeee92a0ac0e2a4d0 Mon Sep 17 00:00:00 2001 From: Marek Date: Fri, 11 Oct 2024 15:27:06 +0200 Subject: [PATCH 15/21] Allow custom config for `zebrad` in test mode --- docker/entrypoint.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 9414f6e3478..1ffad39ef9e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -330,7 +330,18 @@ run_tests() { exec cargo test --locked --release --features "zebra-test" --package zebra-scan \ -- --nocapture --include-ignored scan_task_commands else - exec "$@" + if [[ "$1" == "zebrad" ]] && [[ -f "${ZEBRA_CONF_PATH}" ]]; then + shift + echo "if" + echo "conf path: " "${ZEBRA_CONF_PATH}" + echo "pos param: " "$@" + exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" + else + echo "else" + echo "conf path: " "${ZEBRA_CONF_PATH}" + echo "pos param: " "$@" + exec "$@" + fi fi } From 6932d9acca714cfd323e6291d862d8121a5cb4df Mon Sep 17 00:00:00 2001 From: Marek Date: Fri, 11 Oct 2024 15:44:44 +0200 Subject: [PATCH 16/21] Remove `curl` from the `runtime` Docker image --- docker/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 27a098ab2ec..27775fe9611 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -169,9 +169,7 @@ RUN --mount=type=bind,source=tower-batch-control,target=tower-batch-control \ # from the `release` stage. FROM debian:bookworm-slim AS runtime -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl \ - gosu +RUN apt-get update && apt-get install -y --no-install-recommends gosu RUN rm -rf /var/lib/apt/lists/* /tmp/* From 6fe460daf3ba6383a959a815124c61c789335df3 Mon Sep 17 00:00:00 2001 From: Marek Date: Fri, 11 Oct 2024 19:35:47 +0200 Subject: [PATCH 17/21] Remove redundant echos --- docker/entrypoint.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 1ffad39ef9e..8fa4d30bc6a 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -332,14 +332,8 @@ run_tests() { else if [[ "$1" == "zebrad" ]] && [[ -f "${ZEBRA_CONF_PATH}" ]]; then shift - echo "if" - echo "conf path: " "${ZEBRA_CONF_PATH}" - echo "pos param: " "$@" exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" else - echo "else" - echo "conf path: " "${ZEBRA_CONF_PATH}" - echo "pos param: " "$@" exec "$@" fi fi From c5010b8daf9d2e6ad8d3a4445f32fba65b6c709a Mon Sep 17 00:00:00 2001 From: Marek Date: Sat, 12 Oct 2024 16:52:40 +0200 Subject: [PATCH 18/21] Remove a malfunctioning CD test The test was using a custom config file set in `test_variables`. However, the file was not included in the Docker image, and the entrypoint script created a new, default one under the original file's path. Zebra then loaded this new file, and the test passed because the pattern in `grep_patterns` matched Zebra's output containing the original path, even though the config file was different. --- .github/workflows/cd-deploy-nodes-gcp.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/cd-deploy-nodes-gcp.yml b/.github/workflows/cd-deploy-nodes-gcp.yml index c096d531ddf..26e86c02b9a 100644 --- a/.github/workflows/cd-deploy-nodes-gcp.yml +++ b/.github/workflows/cd-deploy-nodes-gcp.yml @@ -163,18 +163,6 @@ jobs: test_variables: '-e NETWORK' network: 'Testnet' - # Test that Zebra works using $ZEBRA_CONF_PATH config - test-zebra-conf-path: - name: Test CD custom Docker config file - needs: build - uses: ./.github/workflows/sub-test-zebra-config.yml - with: - test_id: 'custom-conf' - docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} - grep_patterns: '-e "loaded zebrad config.*config_path.*=.*v1.0.0-rc.2.toml"' - test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v1.0.0-rc.2.toml"' - network: ${{ inputs.network || vars.ZCASH_NETWORK }} - # Finds a `tip` cached state disk for zebra from the main branch # # Passes the disk name to subsequent jobs using `cached_disk_name` output From e05df78bb14f4961166f60660bbf823f604520df Mon Sep 17 00:00:00 2001 From: Marek Date: Sat, 12 Oct 2024 17:59:33 +0200 Subject: [PATCH 19/21] Remove a redundant CI test --- .github/workflows/sub-ci-unit-tests-docker.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/sub-ci-unit-tests-docker.yml b/.github/workflows/sub-ci-unit-tests-docker.yml index 3f80d24ebbd..f77f3b6408b 100644 --- a/.github/workflows/sub-ci-unit-tests-docker.yml +++ b/.github/workflows/sub-ci-unit-tests-docker.yml @@ -151,19 +151,6 @@ jobs: test_variables: '-e NETWORK' network: 'Mainnet' - # Test reconfiguring the the docker image for tesnet. - test-configuration-file-testnet: - name: Test CI testnet Docker config file - # Make sure Zebra can sync the genesis block on testnet - uses: ./.github/workflows/sub-test-zebra-config.yml - with: - test_id: 'testnet-conf' - docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} - grep_patterns: '-e "net.*=.*Test.*estimated progress to chain tip.*Genesis" -e "net.*=.*Test.*estimated progress to chain tip.*BeforeOverwinter"' - # TODO: improve the entrypoint to avoid using `ENTRYPOINT_FEATURES=""` - test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="/etc/zebrad/zebrad.toml" -e ENTRYPOINT_FEATURES=""' - network: 'Testnet' - # Test that Zebra works using $ZEBRA_CONF_PATH config test-zebra-conf-path: name: Test CI custom Docker config file From e9f0479b5d2ce0214cd3e2a52b899f8df5e0a2f7 Mon Sep 17 00:00:00 2001 From: Marek Date: Sat, 12 Oct 2024 18:00:54 +0200 Subject: [PATCH 20/21] Remove all packages from the `runtime` stage --- docker/Dockerfile | 4 ---- docker/entrypoint.sh | 4 ---- 2 files changed, 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 27775fe9611..39cef822276 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -169,10 +169,6 @@ RUN --mount=type=bind,source=tower-batch-control,target=tower-batch-control \ # from the `release` stage. FROM debian:bookworm-slim AS runtime -RUN apt-get update && apt-get install -y --no-install-recommends gosu - -RUN rm -rf /var/lib/apt/lists/* /tmp/* - COPY --from=release /usr/local/bin/zebrad /usr/local/bin/ COPY --from=release /usr/local/bin/entrypoint.sh /usr/local/bin/ diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 8fa4d30bc6a..0ebe974bfa2 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -379,10 +379,6 @@ monitoring) : ;; *) - if command -v gosu >/dev/null 2>&1; then - exec gosu "$USER" "$@" - else exec "$@" - fi ;; esac From 4fa064c646c4894c5bb5176f5069848d65165e0a Mon Sep 17 00:00:00 2001 From: Marek Date: Mon, 14 Oct 2024 13:38:24 +0200 Subject: [PATCH 21/21] Docs cosmetics --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 0ebe974bfa2..68cda391439 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -67,7 +67,7 @@ prepare_env_vars() { # # ## Positional Parameters # -# - "$1": file to write the config to +# - "$1": the file to write the config to prepare_default_conf_file() { cat <"$1" [network]