From b03e69b4cef17ff4c3593787f93a99ec8ac15d58 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Tue, 7 Jan 2025 01:02:36 +0100 Subject: [PATCH] fix(ci): `prepare-reth` worfklow (#21) **Description** Adds back hive assets reused in workflow `kurtosis-op`. **Tests** **Additional context** https://github.com/ethereum-optimism/op-reth/actions/runs/12640451509/job/35220890751 **Metadata** --- .github/assets/hive/Dockerfile | 9 ++++ .github/assets/hive/build_simulators.sh | 38 ++++++++++++++++ .github/assets/hive/expected_failures.yaml | 53 ++++++++++++++++++++++ .github/assets/hive/load_images.sh | 25 ++++++++++ .github/assets/hive/no_sim_build.diff | 52 +++++++++++++++++++++ .github/assets/hive/parse.py | 47 +++++++++++++++++++ .github/assets/hive/run_simulator.sh | 38 ++++++++++++++++ 7 files changed, 262 insertions(+) create mode 100644 .github/assets/hive/Dockerfile create mode 100755 .github/assets/hive/build_simulators.sh create mode 100644 .github/assets/hive/expected_failures.yaml create mode 100755 .github/assets/hive/load_images.sh create mode 100644 .github/assets/hive/no_sim_build.diff create mode 100644 .github/assets/hive/parse.py create mode 100755 .github/assets/hive/run_simulator.sh diff --git a/.github/assets/hive/Dockerfile b/.github/assets/hive/Dockerfile new file mode 100644 index 000000000000..25b71bf21872 --- /dev/null +++ b/.github/assets/hive/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu + +COPY dist/reth /usr/local/bin + +COPY LICENSE-* ./ + +EXPOSE 30303 30303/udp 9001 8545 8546 +ENV RUST_LOG=debug +ENTRYPOINT ["/usr/local/bin/reth"] diff --git a/.github/assets/hive/build_simulators.sh b/.github/assets/hive/build_simulators.sh new file mode 100755 index 000000000000..b33f4db4ee79 --- /dev/null +++ b/.github/assets/hive/build_simulators.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Create the hive_assets directory +mkdir hive_assets/ + +cd hivetests +go build . + +./hive -client reth # first builds and caches the client + +# Run each hive command in the background for each simulator and wait +echo "Building images" +./hive -client reth --sim "pyspec" -sim.timelimit 1s || true & +./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true & +./hive -client reth --sim "devp2p" -sim.timelimit 1s || true & +./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true & +./hive -client reth --sim "smoke/genesis" -sim.timelimit 1s || true & +./hive -client reth --sim "smoke/network" -sim.timelimit 1s || true & +./hive -client reth --sim "ethereum/sync" -sim.timelimit 1s || true & +wait + +# Run docker save in parallel and wait +echo "Saving images" +docker save hive/hiveproxy:latest -o ../hive_assets/hiveproxy.tar & +docker save hive/simulators/devp2p:latest -o ../hive_assets/devp2p.tar & +docker save hive/simulators/ethereum/engine:latest -o ../hive_assets/engine.tar & +docker save hive/simulators/ethereum/rpc-compat:latest -o ../hive_assets/rpc_compat.tar & +docker save hive/simulators/ethereum/pyspec:latest -o ../hive_assets/pyspec.tar & +docker save hive/simulators/smoke/genesis:latest -o ../hive_assets/smoke_genesis.tar & +docker save hive/simulators/smoke/network:latest -o ../hive_assets/smoke_network.tar & +docker save hive/simulators/ethereum/sync:latest -o ../hive_assets/ethereum_sync.tar & +wait + +# Make sure we don't rebuild images on the CI jobs +git apply ../.github/assets/hive/no_sim_build.diff +go build . +mv ./hive ../hive_assets/ diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml new file mode 100644 index 000000000000..c1c7ff71cf8e --- /dev/null +++ b/.github/assets/hive/expected_failures.yaml @@ -0,0 +1,53 @@ +# https://github.com/paradigmxyz/reth/issues/7015 +# https://github.com/paradigmxyz/reth/issues/6332 +rpc-compat: + - debug_getRawBlock/get-invalid-number (reth) + - debug_getRawHeader/get-invalid-number (reth) + - debug_getRawReceipts/get-invalid-number (reth) + - debug_getRawTransaction/get-invalid-hash (reth) + + - eth_call/call-callenv (reth) + - eth_feeHistory/fee-history (reth) + - eth_getStorageAt/get-storage-invalid-key-too-large (reth) + - eth_getStorageAt/get-storage-invalid-key (reth) + - eth_getTransactionReceipt/get-access-list (reth) + - eth_getTransactionReceipt/get-blob-tx (reth) + - eth_getTransactionReceipt/get-dynamic-fee (reth) + +# https://github.com/paradigmxyz/reth/issues/8732 +engine-withdrawals: + - Withdrawals Fork On Genesis (Paris) (reth) + - Withdrawals Fork on Block 1 (Paris) (reth) + - Withdrawals Fork on Block 2 (Paris) (reth) + - Withdrawals Fork on Block 3 (Paris) (reth) + - Withdraw to a single account (Paris) (reth) + - Withdraw to two accounts (Paris) (reth) + - Withdraw many accounts (Paris) (reth) + - Withdraw zero amount (Paris) (reth) + - Empty Withdrawals (Paris) (reth) + - Corrupted Block Hash Payload (INVALID) (Paris) (reth) + - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth) + - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth) + +engine-api: [] + +# https://github.com/paradigmxyz/reth/issues/8305 +# https://github.com/paradigmxyz/reth/issues/6217 +# https://github.com/paradigmxyz/reth/issues/8306 +# https://github.com/paradigmxyz/reth/issues/7144 +engine-cancun: + - Blob Transaction Ordering, Multiple Clients (Cancun) (reth) + - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth) + - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, VersionedHashes Version, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, Incomplete VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, Extra VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + +sync: [] diff --git a/.github/assets/hive/load_images.sh b/.github/assets/hive/load_images.sh new file mode 100755 index 000000000000..05e1cb9905fa --- /dev/null +++ b/.github/assets/hive/load_images.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -eo pipefail + +# List of tar files to load +IMAGES=( + "/tmp/hiveproxy.tar" + "/tmp/devp2p.tar" + "/tmp/engine.tar" + "/tmp/rpc_compat.tar" + "/tmp/pyspec.tar" + "/tmp/smoke_genesis.tar" + "/tmp/smoke_network.tar" + "/tmp/ethereum_sync.tar" + "/tmp/reth_image.tar" +) + +# Loop through the images and load them +for IMAGE_TAR in "${IMAGES[@]}"; do + echo "Loading image $IMAGE_TAR..." + docker load -i "$IMAGE_TAR" & +done + +wait + +docker image ls -a \ No newline at end of file diff --git a/.github/assets/hive/no_sim_build.diff b/.github/assets/hive/no_sim_build.diff new file mode 100644 index 000000000000..6127a4ecb732 --- /dev/null +++ b/.github/assets/hive/no_sim_build.diff @@ -0,0 +1,52 @@ +diff --git a/internal/libdocker/builder.go b/internal/libdocker/builder.go +index e4bf99b6..2023f7e2 100644 +--- a/internal/libdocker/builder.go ++++ b/internal/libdocker/builder.go +@@ -8,7 +8,6 @@ import ( + "io" + "io/fs" + "log/slog" +- "os" + "path/filepath" + "slices" + "strings" +@@ -49,25 +48,8 @@ func (b *Builder) BuildClientImage(ctx context.Context, client libhive.ClientDes + + // BuildSimulatorImage builds a docker image of a simulator. + func (b *Builder) BuildSimulatorImage(ctx context.Context, name string, buildArgs map[string]string) (string, error) { +- dir := b.config.Inventory.SimulatorDirectory(name) +- buildContextPath := dir +- buildDockerfile := "Dockerfile" +- +- // build context dir of simulator can be overridden with "hive_context.txt" file containing the desired build path +- if contextPathBytes, err := os.ReadFile(filepath.Join(filepath.FromSlash(dir), "hive_context.txt")); err == nil { +- buildContextPath = filepath.Join(dir, strings.TrimSpace(string(contextPathBytes))) +- if strings.HasPrefix(buildContextPath, "../") { +- return "", fmt.Errorf("cannot access build directory outside of Hive root: %q", buildContextPath) +- } +- if p, err := filepath.Rel(buildContextPath, filepath.Join(filepath.FromSlash(dir), "Dockerfile")); err != nil { +- return "", fmt.Errorf("failed to derive relative simulator Dockerfile path: %v", err) +- } else { +- buildDockerfile = p +- } +- } + tag := fmt.Sprintf("hive/simulators/%s:latest", name) +- err := b.buildImage(ctx, buildContextPath, buildDockerfile, tag, buildArgs) +- return tag, err ++ return tag, nil + } + + // BuildImage creates a container by archiving the given file system, +diff --git a/internal/libdocker/proxy.go b/internal/libdocker/proxy.go +index d3a14ae6..8779671e 100644 +--- a/internal/libdocker/proxy.go ++++ b/internal/libdocker/proxy.go +@@ -16,7 +16,7 @@ const hiveproxyTag = "hive/hiveproxy" + + // Build builds the hiveproxy image. + func (cb *ContainerBackend) Build(ctx context.Context, b libhive.Builder) error { +- return b.BuildImage(ctx, hiveproxyTag, hiveproxy.Source) ++ return nil + } + + // ServeAPI starts the API server. diff --git a/.github/assets/hive/parse.py b/.github/assets/hive/parse.py new file mode 100644 index 000000000000..c408a4d13361 --- /dev/null +++ b/.github/assets/hive/parse.py @@ -0,0 +1,47 @@ +import json +import yaml +import sys +import argparse + +# Argument parser setup +parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.") +parser.add_argument("report_json", help="Path to the hive report JSON file.") +parser.add_argument("--exclusion", required=True, help="Path to the exclusion YAML file.") +args = parser.parse_args() + +# Load hive JSON +with open(args.report_json, 'r') as file: + report = json.load(file) + +# Load exclusion YAML +with open(args.exclusion, 'r') as file: + exclusion_data = yaml.safe_load(file) + exclusions = exclusion_data.get(report['name'], []) + +# Collect unexpected failures and passes +unexpected_failures = [] +unexpected_passes = [] + +for test in report['testCases'].values(): + test_name = test['name'] + test_pass = test['summaryResult']['pass'] + if test_name in exclusions: + if test_pass: + unexpected_passes.append(test_name) + else: + if not test_pass: + unexpected_failures.append(test_name) + +# Check if there are any unexpected failures or passes and exit with error +if unexpected_failures or unexpected_passes: + if unexpected_failures: + print("Unexpected Failures:") + for test in unexpected_failures: + print(f" {test}") + if unexpected_passes: + print("Unexpected Passes:") + for test in unexpected_passes: + print(f" {test}") + sys.exit(1) + +print("Success.") diff --git a/.github/assets/hive/run_simulator.sh b/.github/assets/hive/run_simulator.sh new file mode 100755 index 000000000000..731c94c3f69b --- /dev/null +++ b/.github/assets/hive/run_simulator.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# set -x + +cd hivetests/ + +sim="${1}" +limit="${2}" + +run_hive() { + hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism 4 --client reth 2>&1 | tee /tmp/log || true +} + +check_log() { + tail -n 1 /tmp/log | sed -r 's/\x1B\[[0-9;]*[mK]//g' +} + +attempt=0 +max_attempts=5 + +while [ $attempt -lt $max_attempts ]; do + run_hive + + # Check if no tests were run. sed removes ansi colors + if check_log | grep -q "suites=0"; then + echo "no tests were run, retrying in 10 seconds" + sleep 10 + attempt=$((attempt + 1)) + continue + fi + + # Check the last line of the log for "finished", "tests failed", or "test failed" + if check_log | grep -Eq "(finished|tests? failed)"; then + exit 0 + else + exit 1 + fi +done +exit 1