Working deneb support

Dockerfile

@@ -23,12 +23,12 @@
 ARG DEBIAN_RELEASE="bookworm"
 
 # Lighthouse testnet bakery helper
-ARG LCLI_VERSION="4.6.0-rc.0"
+ARG LCLI_VERSION="5.0.0"
 
 # Ethereum clients
-ARG GETH_VERSION="1.13.10"
-ARG LIGHTHOUSE_VERSION="4.6.0-rc.0"
-ARG TEKU_VERSION="24.1.0"
+ARG GETH_VERSION="1.13.13"
+ARG LIGHTHOUSE_VERSION="5.0.0"
+ARG TEKU_VERSION="24.2.0"
 ARG MEV_BOOST_VERSION="1.7a1"
 
 # prysm image
@@ -62,7 +62,7 @@ RUN find -L /usr/local/prysm
 
 # builder and relay
 FROM bitnami/minideb:${DEBIAN_RELEASE} AS mevbuilder
-ARG FLASHBOTS_BUILDER_REF="b2dcbddfb1c81b1c20f0870d6fca414b9016433b"
+ARG FLASHBOTS_BUILDER_REF="7845c515c32ef08f15c420fda0f5de4f2e6800cd"
 ARG MAINIFOLD_FREELAY_REF="support-privatenet"
 ENV GO_1_20_SHA256="5a9ebcc65c1cce56e0d2dc616aff4c4cedcfbda8cc6f0288cc08cda3b18dcbf1"
 
@@ -92,31 +92,31 @@ RUN PATH="/usr/local/go/bin/:$PATH" go run build/ci.go install -static ./cmd/get
 # genesis & zcli tools
 FROM bitnami/minideb:${DEBIAN_RELEASE} AS genesisbuilder
 # Testnet baking accessories
-ARG ZCLI_REF="refs/tags/v0.6.0"
-ARG ETH2_TESTNET_GENESIS_REF="955d4a81095c019dd712ba96316c426330133240"
-ENV GO_1_19_SHA256="464b6b66591f6cf055bc5df90a9750bf5fbc9d038722bb84a9d56a2bea974be6"
+ARG ZCLI_REF="refs/tags/v0.7.1"
+ARG ETH2_TESTNET_GENESIS_REF="4b3498476f14b872b43080eee319adea45286daf"
+ENV GO_1_21_SHA256="13b76a9b2a26823e53062fa841b07087d48ae2ef2936445dc34c4ae03293702c"
 ENV ZCLI_REF="${ZCLI_REF}"
 ENV ETH2_TESTNET_GENESIS_REF="${ETH2_TESTNET_GENESIS_REF}"
 WORKDIR /usr/local/src/
 
 RUN install_packages curl ca-certificates git build-essential
 
 # Install golang
-RUN cd /tmp && curl -OL https://golang.org/dl/go1.19.linux-amd64.tar.gz
-RUN echo "${GO_1_19_SHA256} /tmp/go1.19.linux-amd64.tar.gz" | sha256sum -c
-RUN cd /tmp && tar -C /usr/local -xvf go1.19.linux-amd64.tar.gz
+RUN cd /tmp && curl -OL https://golang.org/dl/go1.21.7.linux-amd64.tar.gz
+RUN echo "${GO_1_21_SHA256} /tmp/go1.21.7.linux-amd64.tar.gz" | sha256sum -c
+RUN cd /tmp && tar -C /usr/local -xvf go1.21.7.linux-amd64.tar.gz
 
 # Build zcli
 RUN git clone https://github.com/protolambda/zcli.git && cd zcli && git fetch origin "${ZCLI_REF}" && git checkout "${ZCLI_REF}"
 RUN cd zcli && PATH="/usr/local/go/bin/:$PATH" go build
 
 # Build genesis tool
-RUN git clone https://github.com/pk910/eth2-testnet-genesis.git && cd eth2-testnet-genesis && git fetch origin "${ETH2_TESTNET_GENESIS_REF}" && git checkout "${ETH2_TESTNET_GENESIS_REF}"
+RUN git clone https://github.com/protolambda/eth2-testnet-genesis.git && cd eth2-testnet-genesis && git fetch origin "${ETH2_TESTNET_GENESIS_REF}" && git checkout "${ETH2_TESTNET_GENESIS_REF}"
 RUN cd eth2-testnet-genesis && PATH="/usr/local/go/bin/:$PATH" go build
 
 # builder and relay
 FROM bitnami/minideb:${DEBIAN_RELEASE} AS ethodbuilder
-ARG ETHDO_REF="v1.31.0"
+ARG ETHDO_REF="5895bbfbe6484505ddf666f0f4c0e25dde3cce9b"
 ENV GO_1_20_SHA256="5a9ebcc65c1cce56e0d2dc616aff4c4cedcfbda8cc6f0288cc08cda3b18dcbf1"
 
 RUN install_packages curl ca-certificates git build-essential
@@ -126,7 +126,7 @@ RUN echo "${GO_1_20_SHA256} /tmp/go1.20.linux-amd64.tar.gz" | sha256sum -c
 RUN cd /tmp && tar -C /usr/local -xvf go1.20.linux-amd64.tar.gz
 
 WORKDIR /usr/local/src/
-RUN git clone https://github.com/wealdtech/ethdo.git && cd ethdo && git fetch origin "${ETHDO_REF}" && git checkout "${ETHDO_REF}"
+RUN git clone https://github.com/ChorusOne/ethdo.git && cd ethdo && git fetch origin "${ETHDO_REF}" && git checkout "${ETHDO_REF}"
 RUN cd ethdo && PATH="/usr/local/go/bin/:$PATH" go mod download
 RUN cd ethdo && PATH="/usr/local/go/bin/:$PATH" go build
 

eth_possim/__main__.py

@@ -73,6 +73,7 @@ def deploy_batch_deposit_contract(rpc: str):
     with open("./.data/configuration.yaml", "w") as f:
         yaml.dump(cfg, f)
 
+
 @cli.command()
 @click.option("--rpc", help="RPC endpoint URL address.", default="")
 def deploy_fee_manager_contracts(rpc: str):
@@ -97,34 +98,47 @@ def deploy_fee_manager_contracts(rpc: str):
         rpc=rpc,
         foundry_json_path=f"{cfg['resources']}/ethereum_compiled_contracts/FeeRewardsManager.json",
         args=[2800],
-        libraries=[("__$c56d76a1417c078a963cba4fa22c45184c$__", fee_manager_library_address)]
+        libraries=[
+            ("__$c56d76a1417c078a963cba4fa22c45184c$__", fee_manager_library_address)
+        ],
     )
     cfg["cl"]["fee_manager_address"] = fee_manager_address
 
     with open("./.data/configuration.yaml", "w") as f:
         yaml.dump(cfg, f)
 
+
 @cli.command()
 @click.option("--rpc", help="RPC endpoint URL address.", default="")
 @click.option("--path", help="Path to the contract source.")
 @click.option("--cfg-key-address", help="Key in which to save the contract address.")
-@click.option("--library", multiple=True, type=(str, str), help="Libraries to be replaced in the bytecode contract in the format key value")
+@click.option(
+    "--library",
+    multiple=True,
+    type=(str, str),
+    help="Libraries to be replaced in the bytecode contract in the format key value",
+)
 @click.argument("args", nargs=-1)
-def deploy_contract_bytecode(rpc: str, path: str, cfg_key_address: str, args: list, library: list):
+def deploy_contract_bytecode(
+    rpc: str, path: str, cfg_key_address: str, args: list, library: list
+):
     """Deploys a contract by the compiled bytecode and abi."""
 
     with open("./.data/configuration.yaml", "r") as f:
         cfg = yaml.safe_load(f)
     if not rpc:
         rpc = f"http://localhost:{cfg['haproxy']['el']['port_geth_rpc']}"
 
-    contract_address = deploy_compiled_contract(cfg=cfg, rpc=rpc, foundry_json_path=path, args=args, libraries=library)
+    contract_address = deploy_compiled_contract(
+        cfg=cfg, rpc=rpc, foundry_json_path=path, args=args, libraries=library
+    )
 
     # Patch and write back `configuration.yaml`
     cfg["cl"][cfg_key_address] = contract_address
     with open("./.data/configuration.yaml", "w") as f:
         yaml.dump(cfg, f)
 
+
 @cli.command()
 @click.option("--rpc", help="RPC endpoint URL address.", default="")
 @click.option("--path", help="Path to the contract source.")

eth_possim/contracts.py

@@ -4,21 +4,29 @@
 import solcx
 import web3
 import re
+import time
 from typing import List, Tuple
 
 
 logger = logging.getLogger(__name__)
 
-def deploy_compiled_contract(cfg: dict, rpc: str, foundry_json_path: str, args: list = [], libraries: List[Tuple[str, str]]  = []) -> str:
+
+def deploy_compiled_contract(
+    cfg: dict,
+    rpc: str,
+    foundry_json_path: str,
+    args: list = [],
+    libraries: List[Tuple[str, str]] = [],
+) -> str:
     with open(foundry_json_path, "r") as f:
         foundry_json = json.loads(f.read())
-    
+
     bytecode_str = foundry_json["bytecode"]["object"][2:]
     for library in libraries:
         # Skip 0x from the library address.
-       bytecode_str = bytecode_str.replace(library[0], library[1][2:])
+        bytecode_str = bytecode_str.replace(library[0], library[1][2:])
     bytecode = binascii.unhexlify(bytecode_str)
-        
+
     abi = foundry_json["abi"]
 
     w3 = web3.Web3(web3.Web3.HTTPProvider(rpc))
@@ -54,6 +62,7 @@ def deploy_compiled_contract(cfg: dict, rpc: str, foundry_json_path: str, args:
 
     return tx_receipt["contractAddress"]
 
+
 def deploy_contract_onchain(
     cfg: dict, rpc: str, path: str, name: str, args: list = []
 ) -> str:
@@ -96,10 +105,21 @@ def deploy_contract_onchain(
         private_key=cfg["el"]["funder"]["private_key"],
     )
     w3.eth.send_raw_transaction(signed_txn.rawTransaction)
-    tx_receipt = w3.eth.wait_for_transaction_receipt(signed_txn.hash)
+    for _ in range(1, 10):
+        try:
+            tx_receipt = w3.eth.wait_for_transaction_receipt(signed_txn.hash)
+        except ValueError as exc:
+            if exc.args[0]["message"] == "transaction indexing is in progress":
+                logger.info(
+                    "Failed to get transaction receipt due to indexing, will retry"
+                )
+                time.sleep(5)
+                continue
+            else:
+                raise
 
-    logger.info(
-        f"Contract from '{path}' was published at address '{tx_receipt['contractAddress']}' [block: {tx_receipt['blockNumber']}]"
-    )
+        logger.info(
+            f"Contract from '{path}' was published at address '{tx_receipt['contractAddress']}' [block: {tx_receipt['blockNumber']}]"
+        )
 
-    return tx_receipt["contractAddress"]
+        return tx_receipt["contractAddress"]

eth_possim/resources/cl/minimal.yaml

@@ -11,7 +11,8 @@ PRESET_BASE: 'minimal'
 # * 'mainnet' - there can be only one
 # * 'prater' - testnet
 # Must match the regex: [a-z0-9\-]
-CONFIG_NAME: 'minimal'
+# CONFIG_NAME: 'minimal'
+# read from privatenet.yaml
 
 # Transition
 # ---------------------------------------------------------------

eth_possim/resources/shell/cl/__init/eth2-genesis.sh

@@ -2,10 +2,9 @@
 
 set -euxo pipefail
 
-# Generate post-merge (Capella-enabled) genesis
+# Generate Deneb genesis
 genesis_args=(
     deneb
-    --legacy-config {{ cfg.meta.dir.cl }}/etc/config-prysm.yaml
     --config {{ cfg.meta.dir.cl }}/etc/config-prysm.yaml
     --mnemonics {{ cfg.meta.dir.cl }}/etc/mnemonics.yaml
     --tranches-dir {{ cfg.meta.dir.cl }}/etc/tranches

requirements-dev.txt

@@ -1,4 +1,4 @@
 pytest
 pytest-ordering
 pytest-timeout
-typing-extensions
+typing-extensions

setup.py

@@ -6,15 +6,15 @@
 
 
 setup(
-    name='eth_possim',
-    version='0.1.0',
-    description='''
+    name="eth_possim",
+    version="0.1.0",
+    description="""
 Run full-featured Ethereum PoS simulator (private net) locally or in CI/CD,
 with experimental PBS (MEV) emulation support.
-    ''',
-    packages=find_packages(exclude=['ez_setup', 'tests', 'tests.*']),
+    """,
+    packages=find_packages(exclude=["ez_setup", "tests", "tests.*"]),
     include_package_data=True,
     install_requires=requires,
-    python_requires='==3.11',
+    python_requires="==3.11",
     author_email="opus@chorus.one",
 )

tests/test_pos_mev_enabled.py

@@ -20,7 +20,11 @@
 def tilt_up():
     os.system("killall -9 geth tilt bootnode haproxy java lighthouse")
 
-    p = subprocess.Popen("make freshrun CONFIG=/opt/privatenet/pbs_config.yaml", shell=True, stdout=subprocess.PIPE)
+    p = subprocess.Popen(
+        "make freshrun CONFIG=/opt/privatenet/pbs_config.yaml",
+        shell=True,
+        stdout=subprocess.PIPE,
+    )
 
     # Wait some time to delete non-PBS privatenet data.
     time.sleep(2)
@@ -38,7 +42,6 @@ def tilt_up():
     p.kill()
 
 
-
 def beacon_url(cfg):
     return f"http://127.0.0.1:{cfg['port']['beacon_api']}"
 
@@ -155,14 +158,70 @@ def test_validator_exited(tilt_up):
     with open(f"{os.path.dirname(__file__)}/deposit_data.json") as f:
         deposit_data = json.load(f)
 
-    # Send exits
-    for i, private_key in enumerate(deposit_data["private_keys"]):
+    # Offline preparation is necessary, since
+    # ethdo does not support minimal beacon spec, making it unable
+    # to read necessary beacon data due to ssz encoding mismatches
+    beacon_genesis_data = requests.get(f"{base_url}/eth/v1/beacon/genesis").json()
+    beacon_fork_data = requests.get(f"{base_url}/eth/v1/beacon/states/head/fork").json()
+    beacon_spec = requests.get(f"{base_url}/eth/v1/config/spec").json()
+
+    # Fork version: honor EIP-7044
+    current_fork_version = beacon_fork_data["data"]["current_version"]
+    if current_fork_version >= beacon_spec["data"]["DENEB_FORK_VERSION"]:
+        exit_fork_version = beacon_spec["data"]["CAPELLA_FORK_VERSION"]
+    else:
+        exit_fork_version = current_fork_version
+    voluntary_exit_domain = beacon_spec["data"]["DOMAIN_VOLUNTARY_EXIT"]
+    bls_to_execution_change_domain = beacon_spec["data"]["DOMAIN_BEACON_PROPOSER"]
+
+    chain_preparation_data = {
+        "version": "3",
+        "validators": [],
+        "genesis_validators_root": beacon_genesis_data["data"][
+            "genesis_validators_root"
+        ],
+        "genesis_fork_version": beacon_genesis_data["data"]["genesis_fork_version"],
+        "epoch": beacon_fork_data["data"]["epoch"],
+        "exit_fork_version": exit_fork_version,
+        "current_fork_version": current_fork_version,
+        "voluntary_exit_domain_type": voluntary_exit_domain,
+        "bls_to_execution_change_domain_type": bls_to_execution_change_domain,
+    }
+    for i, _ in enumerate(deposit_data["private_keys"]):
+        pubkey = deposit_data["deposit_data"][i]["pubkey"]
+        validator_data = requests.get(
+            f"{base_url}/eth/v1/beacon/states/head/validators/0x{pubkey}"
+        ).json()
+        chain_preparation_data["validators"].append(
+            {
+                "index": validator_data["data"]["index"],
+                "withdrawal_credentials": validator_data["data"]["validator"][
+                    "withdrawal_credentials"
+                ],
+                "pubkey": pubkey,
+                "state": validator_data["data"]["status"],
+            }
+        )
+    with open("offline-preparation.json", "w") as opf:
+        opf.write(json.dumps(chain_preparation_data))
+
+    mnemonic = deposit_data["mnemonic"]["seed"]
+    for i, _ in enumerate(deposit_data["private_keys"]):
+        pubkey = deposit_data["deposit_data"][i]["pubkey"]
+        index = chain_preparation_data["validators"][i]["index"]
         subprocess.check_call(
             shlex.split(
-                f"ethdo validator exit --private-key 0x{private_key}  --connection {base_url}"
-            )
+                f'ethdo validator exit --mnemonic "{mnemonic}" --validator {index} --public-key {pubkey}  --connection {base_url} --allow-insecure-connections --offline '
+            ),
+        )
+        with open("exit-operations.json", "r") as f:
+            exit_data = f.read()
+            exit = json.loads(exit_data)
+        print(f"Prepared exit for validator #{index}")
+        response = requests.post(
+            f"{base_url}/eth/v1/beacon/pool/voluntary_exits", json=exit
         )
-        print(f"Sent exit to validator #{33 + i}")
+        print(f"Exit status: {response.status_code}")
 
     # Wait until validators become exited
     while True: