Merge pull request #2999 from ethereum/dev

release v1.2.0

setup.py

@@ -586,6 +586,7 @@ def imports(cls, preset_name: str):
         return super().imports(preset_name) + f'''
 from eth2spec.utils import kzg
 from eth2spec.bellatrix import {preset_name} as bellatrix
+from eth2spec.utils.ssz.ssz_impl import serialize as ssz_serialize
 '''
 
     @classmethod

specs/altair/light-client/sync-protocol.md

@@ -47,7 +47,7 @@ Such environments include resource-constrained devices (e.g. phones for trust-mi
 and metered VMs (e.g. blockchain VMs for cross-chain bridges).
 
 This document suggests a minimal light client design for the beacon chain that
-uses sync committees introduced in [this beacon chain extension](./beacon-chain.md).
+uses sync committees introduced in [this beacon chain extension](../beacon-chain.md).
 
 Additional documents describe how the light client sync protocol can be used:
 - [Full node](./full-node.md)

specs/capella/beacon-chain.md

@@ -29,7 +29,7 @@
     - [`BeaconState`](#beaconstate)
 - [Helpers](#helpers)
   - [Beacon state mutators](#beacon-state-mutators)
-    - [`withdraw`](#withdraw)
+    - [`withdraw_balance`](#withdraw_balance)
   - [Predicates](#predicates)
     - [`has_eth1_withdrawal_credential`](#has_eth1_withdrawal_credential)
     - [`is_fully_withdrawable_validator`](#is_fully_withdrawable_validator)
@@ -52,19 +52,19 @@
 
 Capella is a consensus-layer upgrade containing a number of features related
 to validator withdrawals. Including:
-* Automatic withdrawals of `withdrawable` validators
+* Automatic withdrawals of `withdrawable` validators.
 * Partial withdrawals sweep for validators with 0x01 withdrawal
-  credentials and balances in exceess of `MAX_EFFECTIVE_BALANCE`
+  credentials and balances in excess of `MAX_EFFECTIVE_BALANCE`.
 * Operation to change from `BLS_WITHDRAWAL_PREFIX` to
-  `ETH1_ADDRESS_WITHDRAWAL_PREFIX` versioned withdrawal credentials to enable withdrawals for a validator
+  `ETH1_ADDRESS_WITHDRAWAL_PREFIX` versioned withdrawal credentials to enable withdrawals for a validator.
 
 ## Custom types
 
 We define the following Python custom types for type hinting and readability:
 
 | Name | SSZ equivalent | Description |
 | - | - | - |
-| `WithdrawalIndex` | `uint64` | an index of a `Withdrawal`|
+| `WithdrawalIndex` | `uint64` | an index of a `Withdrawal` |
 
 ## Constants
 
@@ -84,9 +84,9 @@ We define the following Python custom types for type hinting and readability:
 
 ### State list lengths
 
-| Name | Value | Unit | Duration |
-| - | - | :-: | :-: |
-| `WITHDRAWAL_QUEUE_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawals enqueued in state|
+| Name | Value | Unit |
+| - | - | :-: |
+| `WITHDRAWAL_QUEUE_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawals enqueued in state |
 
 ### Max operations per block
 
@@ -266,7 +266,7 @@ class BeaconState(Container):
 
 ### Beacon state mutators
 
-#### `withdraw`
+#### `withdraw_balance`
 
 ```python
 def withdraw_balance(state: BeaconState, validator_index: ValidatorIndex, amount: Gwei) -> None:
@@ -289,7 +289,7 @@ def withdraw_balance(state: BeaconState, validator_index: ValidatorIndex, amount
 ```python
 def has_eth1_withdrawal_credential(validator: Validator) -> bool:
     """
-    Check if ``validator`` has an 0x01 prefixed "eth1" withdrawal credential
+    Check if ``validator`` has an 0x01 prefixed "eth1" withdrawal credential.
     """
     return validator.withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX
 ```

specs/capella/fork-choice.md

@@ -58,5 +58,5 @@ class PayloadAttributes(object):
     timestamp: uint64
     prev_randao: Bytes32
     suggested_fee_recipient: ExecutionAddress
-    withdrawals: Sequence[Withdrawal]  # new in Capella
+    withdrawals: Sequence[Withdrawal]  # [New in Capella]
 ```

specs/capella/fork.md

@@ -65,7 +65,7 @@ an irregular state change is made to upgrade to Capella.
 
 The upgrade occurs after the completion of the inner loop of `process_slots` that sets `state.slot` equal to `CAPELLA_FORK_EPOCH * SLOTS_PER_EPOCH`.
 Care must be taken when transitioning through the fork boundary as implementations will need a modified [state transition function](../phase0/beacon-chain.md#beacon-chain-state-transition-function) that deviates from the Phase 0 document.
-In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead the logic must be within `process_slots`.
+In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead, the logic must be within `process_slots`.
 
 ```python
 def upgrade_to_capella(pre: bellatrix.BeaconState) -> BeaconState:

specs/eip4844/p2p-interface.md

@@ -108,7 +108,7 @@ Alias `sidecar = signed_blobs_sidecar.message`.
 - _[REJECT]_ the beacon proposer signature, `signed_blobs_sidecar.signature`, is valid -- i.e.
     - Let `domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, sidecar.beacon_block_slot // SLOTS_PER_EPOCH)`
     - Let `signing_root = compute_signing_root(sidecar, domain)`
-    - Verify `bls.Verify(proposer_pubkey, signing_root, signed_blob_header.signature) is True`,   
+    - Verify `bls.Verify(proposer_pubkey, signing_root, signed_blobs_sidecar.signature) is True`,   
       where `proposer_pubkey` is the pubkey of the beacon block proposer of `sidecar.beacon_block_slot`
 - _[IGNORE]_ The sidecar is the first sidecar with valid signature received for the `(proposer_index, sidecar.beacon_block_slot)` combination,
   where `proposer_index` is the validator index of the beacon block proposer of `sidecar.beacon_block_slot`

specs/eip4844/polynomial-commitments.md

@@ -15,8 +15,8 @@
   - [BLS12-381 helpers](#bls12-381-helpers)
     - [`bls_modular_inverse`](#bls_modular_inverse)
     - [`div`](#div)
-    - [`lincomb`](#lincomb)
-    - [`matrix_lincomb`](#matrix_lincomb)
+    - [`g1_lincomb`](#g1_lincomb)
+    - [`vector_lincomb`](#vector_lincomb)
   - [KZG](#kzg)
     - [`blob_to_kzg_commitment`](#blob_to_kzg_commitment)
     - [`verify_kzg_proof`](#verify_kzg_proof)
@@ -85,10 +85,10 @@ def div(x: BLSFieldElement, y: BLSFieldElement) -> BLSFieldElement:
     return (int(x) * int(bls_modular_inverse(y))) % BLS_MODULUS
 ```
 
-#### `lincomb`
+#### `g1_lincomb`
 
 ```python
-def lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement]) -> KZGCommitment:
+def g1_lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement]) -> KZGCommitment:
     """
     BLS multiscalar multiplication. This function can be optimized using Pippenger's algorithm and variants.
     """
@@ -99,10 +99,10 @@ def lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement])
     return KZGCommitment(bls.G1_to_bytes48(result))
 ```
 
-#### `matrix_lincomb`
+#### `vector_lincomb`
 
 ```python
-def matrix_lincomb(vectors: Sequence[Sequence[BLSFieldElement]],
+def vector_lincomb(vectors: Sequence[Sequence[BLSFieldElement]],
                    scalars: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]:
     """
     Given a list of ``vectors``, interpret it as a 2D matrix and compute the linear combination
@@ -123,7 +123,7 @@ KZG core functions. These are also defined in EIP-4844 execution specs.
 
 ```python
 def blob_to_kzg_commitment(blob: Blob) -> KZGCommitment:
-    return lincomb(KZG_SETUP_LAGRANGE, blob)
+    return g1_lincomb(KZG_SETUP_LAGRANGE, blob)
 ```
 
 #### `verify_kzg_proof`
@@ -165,7 +165,7 @@ def compute_kzg_proof(polynomial: Sequence[BLSFieldElement], z: BLSFieldElement)
 
     # Calculate quotient polynomial by doing point-by-point division
     quotient_polynomial = [div(a, b) for a, b in zip(polynomial_shifted, denominator_poly)]
-    return KZGProof(lincomb(KZG_SETUP_LAGRANGE, quotient_polynomial))
+    return KZGProof(g1_lincomb(KZG_SETUP_LAGRANGE, quotient_polynomial))
 ```
 
 ### Polynomials

specs/eip4844/validator.md

@@ -93,9 +93,10 @@ def is_data_available(slot: Slot, beacon_block_root: Root, blob_kzg_commitments:
 ```python
 def hash_to_bls_field(x: Container) -> BLSFieldElement:
     """
-    This function is used to generate Fiat-Shamir challenges. The output is not uniform over the BLS field.
+    Compute 32-byte hash of serialized container and convert it to BLS field.
+    The output is not uniform over the BLS field.
     """
-    return int.from_bytes(hash_tree_root(x), "little") % BLS_MODULUS
+    return int.from_bytes(hash(ssz_serialize(x)), "little") % BLS_MODULUS
 ```
 
 ### `compute_powers`
@@ -116,7 +117,7 @@ def compute_powers(x: BLSFieldElement, n: uint64) -> Sequence[BLSFieldElement]:
 
 ```python
 def compute_aggregated_poly_and_commitment(
-        blobs: Sequence[BLSFieldElement],
+        blobs: Sequence[Sequence[BLSFieldElement]],
         kzg_commitments: Sequence[KZGCommitment]) -> Tuple[Polynomial, KZGCommitment]:
     """
     Return the aggregated polynomial and aggregated KZG commitment.
@@ -126,10 +127,10 @@ def compute_aggregated_poly_and_commitment(
     r_powers = compute_powers(r, len(kzg_commitments))
 
     # Create aggregated polynomial in evaluation form
-    aggregated_poly = Polynomial(matrix_lincomb(blobs, r_powers))
+    aggregated_poly = Polynomial(vector_lincomb(blobs, r_powers))
 
     # Compute commitment to aggregated polynomial
-    aggregated_poly_commitment = KZGCommitment(lincomb(kzg_commitments, r_powers))
+    aggregated_poly_commitment = KZGCommitment(g1_lincomb(kzg_commitments, r_powers))
 
     return aggregated_poly, aggregated_poly_commitment
 ```

sync/optimistic.md

@@ -88,8 +88,8 @@ Let `current_slot: Slot` be `(time - genesis_time) // SECONDS_PER_SLOT` where
 class OptimisticStore(object):
     optimistic_roots: Set[Root]
     head_block_root: Root
-    blocks: Dict[Root, BeaconBlock]
-    block_states: Dict[Root, BeaconState]
+    blocks: Dict[Root, BeaconBlock] = field(default_factory=dict)
+    block_states: Dict[Root, BeaconState] = field(default_factory=dict)
 ```
 
 ```python

tests/core/pyspec/eth2spec/VERSION.txt

@@ -1 +1 @@
-1.2.0-rc.3
+1.2.0

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -9,7 +9,6 @@
 import json
 from typing import Iterable, AnyStr, Any, Callable
 import traceback
-
 from ruamel.yaml import (
     YAML,
 )
@@ -98,6 +97,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     yaml = YAML(pure=True)
     yaml.default_flow_style = None
 
+    def _represent_none(self, _):
+        return self.represent_scalar('tag:yaml.org,2002:null', 'null')
+
+    yaml.representer.add_representer(type(None), _represent_none)
+
     # Spec config is using a YAML subset
     cfg_yaml = YAML(pure=True)
     cfg_yaml.default_flow_style = False  # Emit separate line for each key

tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py

@@ -0,0 +1,99 @@
+from eth2spec.test.context import (
+    spec_state_test,
+    with_bellatrix_and_later,
+)
+from eth2spec.test.helpers.attestations import (
+    state_transition_with_full_block,
+)
+from eth2spec.test.helpers.block import (
+    build_empty_block_for_next_slot,
+)
+from eth2spec.test.helpers.fork_choice import (
+    get_genesis_forkchoice_store_and_block,
+    on_tick_and_append_step,
+)
+from eth2spec.test.helpers.optimistic_sync import (
+    PayloadStatusV1,
+    PayloadStatusV1Status,
+    MegaStore,
+    add_optimistic_block,
+    get_optimistic_store,
+)
+from eth2spec.test.helpers.state import (
+    next_epoch,
+    state_transition_and_sign_block,
+)
+
+
+@with_bellatrix_and_later
+@spec_state_test
+def test_from_syncing_to_invalid(spec, state):
+    test_steps = []
+    # Initialization
+    fc_store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    op_store = get_optimistic_store(spec, state, anchor_block)
+    mega_store = MegaStore(spec, fc_store, op_store)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+
+    next_epoch(spec, state)
+
+    current_time = (
+        (spec.SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY * 10 + state.slot) * spec.config.SECONDS_PER_SLOT
+        + fc_store.genesis_time
+    )
+    on_tick_and_append_step(spec, fc_store, current_time, test_steps)
+
+    # Block 0
+    block_0 = build_empty_block_for_next_slot(spec, state)
+    block_0.body.execution_payload.block_hash = spec.hash(bytes(f'block_0', 'UTF-8'))
+    signed_block = state_transition_and_sign_block(spec, state, block_0)
+    yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, status=PayloadStatusV1Status.VALID)
+    assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root
+
+    state_0 = state.copy()
+
+    # Create VALID chain `a`
+    signed_blocks_a = []
+    for i in range(3):
+        block = build_empty_block_for_next_slot(spec, state)
+        block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_a_{i}', 'UTF-8'))
+        block.body.execution_payload.parent_hash = (
+            spec.hash(bytes(f'chain_a_{i - 1}', 'UTF-8')) if i != 0 else block_0.body.execution_payload.block_hash
+        )
+
+        signed_block = state_transition_and_sign_block(spec, state, block)
+        yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, status=PayloadStatusV1Status.VALID)
+        assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root
+        signed_blocks_a.append(signed_block.copy())
+
+    # Create SYNCING chain `b`
+    signed_blocks_b = []
+    state = state_0.copy()
+    for i in range(3):
+        block = build_empty_block_for_next_slot(spec, state)
+        block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_b_{i}', 'UTF-8'))
+        block.body.execution_payload.parent_hash = (
+            spec.hash(bytes(f'chain_b_{i - 1}', 'UTF-8')) if i != 0 else block_0.body.execution_payload.block_hash
+        )
+        signed_block = state_transition_with_full_block(spec, state, True, True, block=block)
+        signed_blocks_b.append(signed_block.copy())
+        yield from add_optimistic_block(spec, mega_store, signed_block, test_steps,
+                                        status=PayloadStatusV1Status.SYNCING)
+        assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root
+
+    # Now add block 4 to chain `b` with INVALID
+    block = build_empty_block_for_next_slot(spec, state)
+    block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_b_3', 'UTF-8'))
+    block.body.execution_payload.parent_hash = signed_blocks_b[-1].message.body.execution_payload.block_hash
+    signed_block = state_transition_and_sign_block(spec, state, block)
+    payload_status = PayloadStatusV1(
+        status=PayloadStatusV1Status.INVALID,
+        latest_valid_hash=block_0.body.execution_payload.block_hash,
+        validation_error="invalid",
+    )
+    yield from add_optimistic_block(spec, mega_store, signed_block, test_steps,
+                                    payload_status=payload_status)
+    assert mega_store.opt_store.head_block_root == signed_blocks_a[-1].message.hash_tree_root()
+
+    yield 'steps', test_steps

tests/core/pyspec/eth2spec/test/helpers/attestations.py

@@ -251,11 +251,13 @@ def state_transition_with_full_block(spec,
                                      fill_cur_epoch,
                                      fill_prev_epoch,
                                      participation_fn=None,
-                                     sync_aggregate=None):
+                                     sync_aggregate=None,
+                                     block=None):
     """
     Build and apply a block with attestions at the calculated `slot_to_attest` of current epoch and/or previous epoch.
     """
-    block = build_empty_block_for_next_slot(spec, state)
+    if block is None:
+        block = build_empty_block_for_next_slot(spec, state)
     if fill_cur_epoch and state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY:
         slot_to_attest = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1
         if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)):

tests/core/pyspec/eth2spec/test/helpers/deposits.py

@@ -137,14 +137,21 @@ def prepare_random_genesis_deposits(spec,
     return deposits, root, deposit_data_list
 
 
-def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_credentials=None, signed=False):
+def prepare_state_and_deposit(spec, state, validator_index, amount,
+                              pubkey=None,
+                              privkey=None,
+                              withdrawal_credentials=None,
+                              signed=False):
     """
     Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount.
     """
     deposit_data_list = []
 
-    pubkey = pubkeys[validator_index]
-    privkey = privkeys[validator_index]
+    if pubkey is None:
+        pubkey = pubkeys[validator_index]
+
+    if privkey is None:
+        privkey = privkeys[validator_index]
 
     # insecurely use pubkey as withdrawal key if no credentials provided
     if withdrawal_credentials is None:
@@ -196,7 +203,7 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef
 
     yield 'post', state
 
-    if not effective:
+    if not effective or not bls.KeyValidate(deposit.data.pubkey):
         assert len(state.validators) == pre_validator_count
         assert len(state.balances) == pre_validator_count
         if validator_index < pre_validator_count: