Merge branch 'dev' into consolidation-switch-to-comp

.github/workflows/generate_vectors.yml

@@ -12,17 +12,19 @@ on:
         default: dev
         type: string
         required: true
+  schedule:
+    - cron:  '0 2 * * *'
 
 jobs:
   generate-tests:
-    runs-on: [self-hosted-ghr-custom, size-chungus-x64, profile-consensusSpecs]
+    runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           repository: 'ethereum/consensus-specs'
           path: 'consensus-specs'
-          ref: ${{ inputs.source_ref }}
+          ref: ${{ inputs.ref || 'dev' }}
       - name: Checkout consensus-spec-tests repository
         uses: actions/checkout@v4
         with:
@@ -50,6 +52,16 @@ jobs:
           cp -r presets/ ../consensus-spec-tests/presets
           cp -r configs/ ../consensus-spec-tests/configs
           find . -type d -empty -delete
+      - name: Check for errors
+        run: |
+          if grep -q "\[ERROR\]" consensustestgen.log; then
+            echo "There is an error in the log"
+            exit 1
+          fi
+          if find . -type f -name "INCOMPLETE" | grep -q "INCOMPLETE"; then
+            echo "There is an INCOMPLETE file"
+            exit 1
+          fi
       - name: Archive configurations
         run: |
           cd consensus-spec-tests

Makefile

@@ -34,7 +34,7 @@ MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
-ALL_EXECUTABLE_SPEC_NAMES = phase0 altair bellatrix capella deneb electra whisk eip6800 eip7732
+ALL_EXECUTABLE_SPEC_NAMES = phase0 altair bellatrix capella deneb electra whisk eip6800 eip7594 eip7732
 # The parameters for commands. Use `foreach` to avoid listing specs again.
 COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth2spec.$S.$(TEST_PRESET_TYPE))
 PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), ./eth2spec/$S)
@@ -96,13 +96,16 @@ dist_check:
 dist_upload:
 	python3 -m twine upload dist/*
 
+build_wheel: install_test pyspec
+	. venv/bin/activate && \
+	python3 -m build --no-isolation --outdir ./dist ./
 
 # "make generate_tests" to run all generators
 generate_tests: $(GENERATOR_TARGETS)
 
 # "make pyspec" to create the pyspec for all phases.
 pyspec:
-	python3 -m venv venv; . venv/bin/activate; python3 setup.py pyspecdev
+	@python3 -m venv venv; . venv/bin/activate; python3 setup.py pyspecdev
 
 # check the setup tool requirements
 preinstallation:
@@ -138,13 +141,21 @@ endif
 open_cov:
 	((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &
 
+# Check all files and error if any ToC were modified.
 check_toc: $(MARKDOWN_FILES:=.toc)
+	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
 
+# Generate ToC sections & save copy of original if modified.
 %.toc:
-	cp $* $*.tmp && \
-	doctoc $* && \
-	diff -q $* $*.tmp && \
-	rm $*.tmp
+	@cp $* $*.tmp; \
+	doctoc $* > /dev/null; \
+	if diff -q $* $*.tmp > /dev/null; then \
+		echo "Good $*"; \
+		rm $*.tmp; \
+	else \
+		echo "\033[1;33m Bad $*\033[0m"; \
+		echo "\033[1;34m See $*.tmp\033[0m"; \
+	fi
 
 codespell:
 	codespell . --skip "./.git,./venv,$(PY_SPEC_DIR)/.mypy_cache" -I .codespell-whitelist
@@ -195,7 +206,8 @@ define run_generator
 	cd $(GENERATOR_DIR)/$(1); \
 	if ! test -d venv; then python3 -m venv venv; fi; \
 	. venv/bin/activate; \
-	pip3 install -r requirements.txt; \
+	pip3 install ../../../dist/eth2spec-*.whl; \
+	pip3 install 'eth2spec[generator]'; \
 	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \
 	echo "generator $(1) finished"
 endef
@@ -217,7 +229,7 @@ gen_kzg_setups:
 
 # For any generator, build it using the run_generator function.
 # (creation of output dir is a dependency)
-gen_%: $(TEST_VECTOR_DIR)
+gen_%: build_wheel $(TEST_VECTOR_DIR)
 	$(call run_generator,$*)
 
 detect_generator_incomplete: $(TEST_VECTOR_DIR)

README.md

@@ -8,7 +8,7 @@ This repository hosts the current Ethereum proof-of-stake specifications. Discus
 
 ## Specs
 
-[![GitHub release](https://img.shields.io/github/v/release/ethereum/eth2.0-specs)](https://github.com/ethereum/eth2.0-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec)
+[![GitHub release](https://img.shields.io/github/v/release/ethereum/consensus-specs)](https://github.com/ethereum/consensus-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) [![testgen](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml/badge.svg?branch=dev&event=schedule)](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml)
 
 Core specifications for Ethereum proof-of-stake clients can be found in [specs](specs/). These are divided into features.
 Features are researched and developed in parallel, and then consolidated into sequential upgrades when ready.

presets/minimal/electra.yaml

@@ -41,5 +41,5 @@ MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 2
 
 # Withdrawals processing
 # ---------------------------------------------------------------
-# 2**0 ( = 1) pending withdrawals
-MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 1
+# 2**1 ( = 2) pending withdrawals
+MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 2

pysetup/helpers.py

@@ -119,6 +119,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
     hardcoded_func_dep_presets = reduce(lambda obj, builder: {**obj, **builder.hardcoded_func_dep_presets(spec_object)}, builders, {})
     # Concatenate all strings
     imports =              reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name)  ).strip("\n"), builders, "")
+    classes =              reduce(lambda txt, builder: (txt + "\n\n" + builder.classes()             ).strip("\n"), builders, "")
     preparations =         reduce(lambda txt, builder: (txt + "\n\n" + builder.preparations()        ).strip("\n"), builders, "")
     sundry_functions =     reduce(lambda txt, builder: (txt + "\n\n" + builder.sundry_functions()    ).strip("\n"), builders, "")
     # Keep engine from the most recent fork
@@ -154,6 +155,8 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
         constant_vars_spec,
         preset_vars_spec,
         config_spec,
+        # Custom classes which are not required to be SSZ containers.
+        classes,
         ordered_class_objects_spec,
         protocols_spec,
         functions_spec,

pysetup/spec_builders/base.py

@@ -15,6 +15,13 @@ def imports(cls, preset_name: str) -> str:
         """
         return ""
 
+    @classmethod
+    def classes(cls) -> str:
+        """
+        Define special classes.
+        """
+        return ""
+
     @classmethod
     def preparations(cls) -> str:
         """

pysetup/spec_builders/deneb.py

@@ -12,6 +12,21 @@ def imports(cls, preset_name: str):
 from eth2spec.capella import {preset_name} as capella
 '''
 
+    @classmethod
+    def classes(cls):
+        return f'''
+class BLSFieldElement(bls.Scalar):
+    pass
+
+
+class Polynomial(list):
+    def __init__(self, evals: Optional[Sequence[BLSFieldElement]] = None):
+        if evals is None:
+            evals = [BLSFieldElement(0)] * FIELD_ELEMENTS_PER_BLOB
+        if len(evals) != FIELD_ELEMENTS_PER_BLOB:
+            raise ValueError("expected FIELD_ELEMENTS_PER_BLOB evals")
+        super().__init__(evals)
+'''
 
     @classmethod
     def preparations(cls):

pysetup/spec_builders/eip7594.py

@@ -12,12 +12,41 @@ def imports(cls, preset_name: str):
         return f'''
 from eth2spec.deneb import {preset_name} as deneb
 '''
-
+
+
+    @classmethod
+    def classes(cls):
+        return f'''
+class PolynomialCoeff(list):
+    def __init__(self, coeffs: Sequence[BLSFieldElement]):
+        if len(coeffs) > FIELD_ELEMENTS_PER_EXT_BLOB:
+            raise ValueError("expected <= FIELD_ELEMENTS_PER_EXT_BLOB coeffs")
+        super().__init__(coeffs)
+
+
+class Coset(list):
+    def __init__(self, coeffs: Optional[Sequence[BLSFieldElement]] = None):
+        if coeffs is None:
+            coeffs = [BLSFieldElement(0)] * FIELD_ELEMENTS_PER_CELL
+        if len(coeffs) != FIELD_ELEMENTS_PER_CELL:
+            raise ValueError("expected FIELD_ELEMENTS_PER_CELL coeffs")
+        super().__init__(coeffs)
+
+
+class CosetEvals(list):
+    def __init__(self, evals: Optional[Sequence[BLSFieldElement]] = None):
+        if evals is None:
+            evals = [BLSFieldElement(0)] * FIELD_ELEMENTS_PER_CELL
+        if len(evals) != FIELD_ELEMENTS_PER_CELL:
+            raise ValueError("expected FIELD_ELEMENTS_PER_CELL coeffs")
+        super().__init__(evals)
+'''
 
     @classmethod
     def sundry_functions(cls) -> str:
         return """
 def retrieve_column_sidecars(beacon_block_root: Root) -> Sequence[DataColumnSidecar]:
+    # pylint: disable=unused-argument
     return []
 """
 

pysetup/spec_builders/electra.py

@@ -19,3 +19,41 @@ def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
             'CURRENT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(86)',
             'NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(87)',
         }
+
+
+    @classmethod
+    def execution_engine_cls(cls) -> str:
+        return """
+class NoopExecutionEngine(ExecutionEngine):
+
+    def notify_new_payload(self: ExecutionEngine,
+                           execution_payload: ExecutionPayload,
+                           execution_requests: ExecutionRequests,
+                           parent_beacon_block_root: Root) -> bool:
+        return True
+
+    def notify_forkchoice_updated(self: ExecutionEngine,
+                                  head_block_hash: Hash32,
+                                  safe_block_hash: Hash32,
+                                  finalized_block_hash: Hash32,
+                                  payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]:
+        pass
+
+    def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse:
+        # pylint: disable=unused-argument
+        raise NotImplementedError("no default block production")
+
+    def is_valid_block_hash(self: ExecutionEngine,
+                            execution_payload: ExecutionPayload,
+                            parent_beacon_block_root: Root) -> bool:
+        return True
+
+    def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool:
+        return True
+
+    def verify_and_notify_new_payload(self: ExecutionEngine,
+                                      new_payload_request: NewPayloadRequest) -> bool:
+        return True
+
+
+EXECUTION_ENGINE = NoopExecutionEngine()"""

requirements_preinstallation.txt

@@ -1,4 +1,5 @@
 pip>=24.0.0
 wheel>=0.44.0
 setuptools>=72.0.0
-pylint>=3.2.0
+pylint>=3.2.0
+build>=1.2.2

setup.py

@@ -35,6 +35,16 @@
 )
 from pysetup.md_doc_paths import get_md_doc_paths
 
+# Ignore '1.5.0-alpha.*' to '1.5.0a*' messages.
+import warnings
+warnings.filterwarnings('ignore', message='Normalizing .* to .*')
+
+# Ignore 'running' and 'creating' messages
+import logging
+class PyspecFilter(logging.Filter):
+    def filter(self, record):
+        return not record.getMessage().startswith(('running ', 'creating '))
+logging.getLogger().addFilter(PyspecFilter())
 
 # NOTE: have to programmatically include third-party dependencies in `setup.py`.
 def installPackage(package: str):
@@ -173,7 +183,7 @@ def _update_constant_vars_with_kzg_setups(constant_vars, preset_name):
     constant_vars['KZG_SETUP_G1_MONOMIAL'] = VariableDefinition(constant_vars['KZG_SETUP_G1_MONOMIAL'].value, str(kzg_setups[0]), comment, None)
     constant_vars['KZG_SETUP_G1_LAGRANGE'] = VariableDefinition(constant_vars['KZG_SETUP_G1_LAGRANGE'].value, str(kzg_setups[1]), comment, None)
     constant_vars['KZG_SETUP_G2_MONOMIAL'] = VariableDefinition(constant_vars['KZG_SETUP_G2_MONOMIAL'].value, str(kzg_setups[2]), comment, None)
-    
+
 
 def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], preset_name=str) -> SpecObject:
     functions: Dict[str, str] = {}
@@ -251,10 +261,17 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
                                 # marko parses `**X**` as a list containing a X
                                 description = description[0].children
 
+                    if isinstance(name, list):
+                        # marko parses `[X]()` as a list containing a X
+                        name = name[0].children
                     if isinstance(value, list):
                         # marko parses `**X**` as a list containing a X
                         value = value[0].children
 
+                    # Skip types that have been defined elsewhere
+                    if description is not None and description.startswith("<!-- predefined-type -->"):
+                        continue
+
                     if not _is_constant_id(name):
                         # Check for short type declarations
                         if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List", "ByteVector")):
@@ -394,8 +411,6 @@ def initialize_options(self):
     def finalize_options(self):
         """Post-process options."""
         if len(self.md_doc_paths) == 0:
-            print("no paths were specified, using default markdown file paths for pyspec"
-                  " build (spec fork: %s)" % self.spec_fork)
             self.md_doc_paths = get_md_doc_paths(self.spec_fork)
             if len(self.md_doc_paths) == 0:
                 raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork)
@@ -428,6 +443,7 @@ def run(self):
         if not self.dry_run:
             dir_util.mkpath(self.out_dir)
 
+        print(f'Building pyspec: {self.spec_fork}')
         for (name, preset_paths, config_path) in self.parsed_build_targets:
             spec_str = build_spec(
                 spec_builders[self.spec_fork].fork,
@@ -492,7 +508,6 @@ def run_pyspec_cmd(self, spec_fork: str, **opts):
         self.run_command('pyspec')
 
     def run(self):
-        print("running build_py command")
         for spec_fork in spec_builders:
             self.run_pyspec_cmd(spec_fork=spec_fork)
 
@@ -524,26 +539,30 @@ def run(self):
     long_description=readme,
     long_description_content_type="text/markdown",
     author="ethereum",
-    url="https://github.com/ethereum/eth2.0-specs",
+    url="https://github.com/ethereum/consensus-specs",
     include_package_data=False,
-    package_data={'configs': ['*.yaml'],
-                  'presets': ['*.yaml'],
-                  'specs': ['**/*.md'],
-                  'eth2spec': ['VERSION.txt']},
+    package_data={
+        'configs': ['*.yaml'],
+        'eth2spec': ['VERSION.txt'],
+        'presets': ['**/*.yaml', '**/*.json'],
+        'specs': ['**/*.md'],
+        'sync': ['optimistic.md'],
+    },
     package_dir={
-        "eth2spec": "tests/core/pyspec/eth2spec",
         "configs": "configs",
+        "eth2spec": "tests/core/pyspec/eth2spec",
         "presets": "presets",
         "specs": "specs",
+        "sync": "sync",
     },
-    packages=find_packages(where='tests/core/pyspec') + ['configs', 'specs'],
+    packages=find_packages(where='tests/core/pyspec') + ['configs', 'presets', 'specs', 'presets', 'sync'],
     py_modules=["eth2spec"],
     cmdclass=commands,
     python_requires=">=3.9, <4",
     extras_require={
         "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"],
         "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==2.15.3"],
-        "generator": ["python-snappy==0.6.1", "filelock", "pathos==0.3.0"],
+        "generator": ["setuptools>=72.0.0", "pytest>4.4", "python-snappy==0.7.3", "filelock", "pathos==0.3.0"],
         "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3",  "mkdocs-awesome-pages-plugin==2.8.0"]
     },
     install_requires=[
@@ -557,7 +576,7 @@ def run(self):
         RUAMEL_YAML_VERSION,
         "lru-dict==1.2.0",
         MARKO_VERSION,
-        "py_arkworks_bls12381==0.3.4",
-        "curdleproofs==0.1.1",
+        "py_arkworks_bls12381==0.3.8",
+        "curdleproofs==0.1.2",
     ]
 )