diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..e90b2a63 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,37 @@ +# Python +*.pyc +*.pyo +*.pyd +__pycache__ +*.swp +*.swo +.Python +env/ +venv/ +ENV/ +*.egg-info +.eggs/ +.dist-info +*.egg +*.whl +.build/ +.tox/ +.coverage +coverage.* +.cache +.pytest_cache/ + +# Rust +target/ +Cargo.lock + +# Docker files +Dockerfile +docker-compose.yml + +# Version control +.git/ +.gitignore + +hdp-test +cairo-vm \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index b73fd8e6..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Main - -on: - pull_request: - -jobs: - main: - runs-on: ubuntu-latest - - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Setup Scarb - uses: software-mansion/setup-scarb@v1 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Cache Python environment - uses: actions/cache@v3 - with: - path: | - ~/.cache/pip - venv - key: ${{ runner.os }}-python-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-python- - - - name: Install Dependencies - run: make setup - - - name: Check Python formatting - run: | - source venv/bin/activate - ./tools/make/python_format_check.sh - - - name: Check Cairo formatting - run: | - source venv/bin/activate - ./tools/make/cairo_format_check.sh - - - name: Compile Cairo files - run: | - source venv/bin/activate - make build - - - name: Run Unit Cairo tests - env: - RPC_URL_MAINNET: ${{ secrets.RPC_URL_MAINNET }} - run: | - source venv/bin/activate - ./tools/make/cairo_tests.sh - - - name: Run Full Flow tests - run: | - source venv/bin/activate - ./tools/make/full_flow_test.sh diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml new file mode 100644 index 00000000..cb3a3ccb --- /dev/null +++ b/.github/workflows/merge.yml @@ -0,0 +1,32 @@ +name: Build and Push Docker Image + +on: + push: + branches: + - main + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: "lab:latest" + driver: cloud + endpoint: "dataprocessor/github" + cleanup: true + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + file: environment.dockerfile + tags: "dataprocessor/hdp-cairo:latest" + platforms: linux/amd64,linux/arm64 + push: true diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 00000000..4b733d40 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,87 @@ +name: CI Build and Test Workflow + +on: + pull_request: + branches: + - main +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: "lab:latest" + driver: cloud + endpoint: "dataprocessor/github" + cleanup: true + - name: Build and cache Docker image + uses: docker/build-push-action@v5 + with: + file: environment.dockerfile + tags: "dataprocessor/hdp-cairo:latest" +# + - name: Setup Scarb + uses: software-mansion/setup-scarb@v1 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Cache Python environment + uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + venv + key: ${{ runner.os }}-python-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('tools/make/setup.sh') }} + restore-keys: | + ${{ runner.os }}-python-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('tools/make/setup.sh') }} + ${{ runner.os }}-python- + +# - name: Cache Cargo directories +# uses: actions/cache@v3 +# with: +# path: | +# ~/.cargo/bin/ +# ~/.cargo/registry/index/ +# ~/.cargo/registry/cache/ +# ~/.cargo/git/db/ +# cairo-vm/cargo-run/target +# key: ${{ runner.os }}-cargo-${{ hashFiles('tools/make/install_cairo1_run.sh') }} +# restore-keys: ${{ runner.os }}-cargo- + + - name: Install Dependencies + run: make setup + + - name: Check Python formatting + run: | + source venv/bin/activate + ./tools/make/python_format_check.sh + + - name: Check Cairo formatting + run: | + source venv/bin/activate + ./tools/make/cairo_format_check.sh + - name: Compile Cairo files + run: | + source venv/bin/activate + make build + - name: Run Unit Cairo tests + env: + RPC_URL_MAINNET: ${{ secrets.RPC_URL_MAINNET }} + run: | + source venv/bin/activate + ./tools/make/cairo_tests.sh + - name: Run Full Flow tests + run: | + source venv/bin/activate + ./tools/make/full_flow_test.sh \ No newline at end of file diff --git a/.gitignore b/.gitignore index 11b384b7..fd52d97d 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ hdp/ # Whitelist - these files should not be ignored !tests/cairo_programs/fixtures/*.json !tools/js/package.json +!packages/cairo-lang-0.13.1.zip diff --git a/README.md b/README.md index 4840bb19..2e536e22 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,11 @@ Cairo HDP is a collection of Cairo0 programs designed to verify inclusion proofs and perform computations on the data. These computations can be verified on-chain, enabling trustless operations on any historical data from Ethereum or integrated EVM chains. +## Run docker test run +```bash +docker build -t hdp-cairo . && docker run hdp-cairo +``` + ## Installation and Setup To install the required dependencies and set up the Python virtual environment, run: diff --git a/environment.dockerfile b/environment.dockerfile new file mode 100644 index 00000000..1a0123e6 --- /dev/null +++ b/environment.dockerfile @@ -0,0 +1,33 @@ +# Use the official Python 3.9 image from the Docker Hub +FROM python:3.9.0 + +# Set the default shell to bash and the working directory in the container +SHELL ["/bin/bash", "-ci"] +WORKDIR /hdp + +# Install Rust using Rustup +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ + echo 'export PATH="/root/.cargo/bin:$PATH"' >> /root/.bashrc + +# Add Cargo executables to PATH +RUN mkdir -p /root/.local/bin && \ + echo 'export PATH="/root/.local/bin:$PATH"' >> /root/.bashrc + +# Install cairo1-run into PATH +RUN git clone https://github.com/HerodotusDev/cairo-vm.git && \ + cd cairo-vm && git checkout aecbb3f01dacb6d3f90256c808466c2c37606252 && \ + cd cairo1-run && cargo install --path . + +# Copy project requirements and install them +COPY tools/make/requirements.txt tools/make/requirements.txt +RUN python -m pip install --upgrade pip && pip install -r tools/make/requirements.txt + +# Copy and install Contract bootloader SNOS dependencies +COPY packages/cairo-lang-0.13.1.zip packages/cairo-lang-0.13.1.zip +RUN pip install packages/cairo-lang-0.13.1.zip + +# Copy the entire project into the image +COPY . . + +# Install Python package modules +RUN pip install . diff --git a/packages/cairo-lang-0.13.1.zip b/packages/cairo-lang-0.13.1.zip new file mode 100644 index 00000000..02c43b3d Binary files /dev/null and b/packages/cairo-lang-0.13.1.zip differ diff --git a/tests/cairo_programs/rlp.cairo b/tests/cairo_programs/rlp.cairo index a99afb08..f89da575 100644 --- a/tests/cairo_programs/rlp.cairo +++ b/tests/cairo_programs/rlp.cairo @@ -25,7 +25,6 @@ func main{range_check_ptr, bitwise_ptr: BitwiseBuiltin*, keccak_ptr: KeccakBuilt uint256_reverse_endian, split_128, reverse_endian, - bytes_to_8_bytes_chunks, ) import rlp %} diff --git a/tools/make/build.sh b/tools/make/build.sh index a12d7ae3..ddee277d 100755 --- a/tools/make/build.sh +++ b/tools/make/build.sh @@ -23,7 +23,7 @@ export -f process_cairo_file mkdir -p build/compiled_cairo_files # Find Cairo files and process them in parallel -find ./src ./tests/cairo_programs ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name "*.cairo" ! -path "./src/cairo1/*" | parallel --halt now,fail=1 process_cairo_file +find ./src ./tests/cairo_programs ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name "*.cairo" ! -path "./src/cairo1/*" | parallel --halt now,fail=1 process_cairo_file {} # Capture the exit status of parallel exit_status=$? diff --git a/tools/make/build_cairo1.sh b/tools/make/build_cairo1.sh index d565b77c..8875efa0 100755 --- a/tools/make/build_cairo1.sh +++ b/tools/make/build_cairo1.sh @@ -9,16 +9,15 @@ process_scarb_project() { # Change to the project directory and build the project if (cd "$project_dir" && scarb build); then echo "Copying built files from $project_dir/target/dev to $build_dir" - cp "$project_dir/target/dev/"* "$build_dir/" - - local status=$? - if [ $status -eq 0 ]; then + if cp "$project_dir/target/dev/"* "$build_dir/"; then echo "$(date '+%Y-%m-%d %H:%M:%S') - Successfully built and copied files for Scarb project in $project_dir" else echo "$(date '+%Y-%m-%d %H:%M:%S') - Built Scarb project in $project_dir, but failed to copy files" + return 1 fi else echo "$(date '+%Y-%m-%d %H:%M:%S') - Failed to build Scarb project in $project_dir" + return 1 fi } @@ -29,7 +28,7 @@ export -f process_scarb_project mkdir -p build/compiled_cairo_files # Find Scarb projects and execute process_scarb_project in each -find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 process_scarb_project +find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 process_scarb_project {} # Capture the exit status of parallel exit_status=$? diff --git a/tools/make/cairo_format_check.sh b/tools/make/cairo_format_check.sh index 6f802613..dcf8af9f 100755 --- a/tools/make/cairo_format_check.sh +++ b/tools/make/cairo_format_check.sh @@ -30,15 +30,25 @@ export -f format_scarb_project # Find all .cairo files and format them in parallel echo "Finding and formatting .cairo files..." -find ./src ./tests ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name '*.cairo' ! -path "./src/cairo1/*" | parallel --halt soon,fail=1 format_file +find ./src ./tests ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name '*.cairo' ! -path "./src/cairo1/*" | parallel --halt soon,fail=1 format_file {} + +# Capture the exit status of parallel for .cairo files +exit_status_cairo_files=$? # Find Scarb projects and format them in parallel echo "Finding and formatting Scarb projects..." -find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 format_scarb_project +find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 format_scarb_project {} + +# Capture the exit status of parallel for Scarb projects +exit_status_scarb_projects=$? -# Capture the exit status of parallel -exit_status=$? +# Determine the final exit status +if [ $exit_status_cairo_files -ne 0 ] || [ $exit_status_scarb_projects -ne 0 ]; then + final_exit_status=1 +else + final_exit_status=0 +fi -# Exit with the captured status -echo "Parallel execution exited with status: $exit_status" -exit $exit_status +# Exit with the determined status +echo "Parallel execution exited with status: $final_exit_status" +exit $final_exit_status diff --git a/tools/make/cairo_tests.sh b/tools/make/cairo_tests.sh index 468727d6..ed831bf5 100755 --- a/tools/make/cairo_tests.sh +++ b/tools/make/cairo_tests.sh @@ -7,33 +7,39 @@ run_tests() { echo "Running tests for $cairo_file..." - # Redirecting output to temp file for potential error capture + # Compile the Cairo file cairo-compile --cairo_path="packages/eth_essentials" "$cairo_file" --output "build/compiled_cairo_files/$filename.json" > "$temp_output" 2>&1 + local compile_status=$? + + if [ $compile_status -ne 0 ]; then + echo "$(date '+%Y-%m-%d %H:%M:%S') - Compilation Failed: $cairo_file" + cat "$temp_output" # Display the captured output on failure + rm -f "$temp_output" + return $compile_status + fi + + # Run the compiled program cairo-run --program="build/compiled_cairo_files/$filename.json" --layout=starknet_with_keccak >> "$temp_output" 2>&1 - local status=$? + local run_status=$? - if [ $status -eq 0 ]; then + if [ $run_status -eq 0 ]; then echo "$(date '+%Y-%m-%d %H:%M:%S') - Test Successful: $cairo_file" else echo "$(date '+%Y-%m-%d %H:%M:%S') - Test Failed: $cairo_file" cat "$temp_output" # Display the captured output on failure - rm -f "$temp_output" - return $status fi rm -f "$temp_output" + return $run_status } -# Activate the virtual environment -source venv/bin/activate - # Export the function so it's available in subshells export -f run_tests # Find all .cairo files under tests/cairo_programs directory and run tests in parallel -# Excluding 'test_vectors.cairo' and './src/cairo1/*' files +# Exclude 'test_vectors.cairo' and './src/cairo1/*' files echo "Finding and running tests..." -find ./tests/cairo_programs -name '*.cairo' ! -name 'test_vectors.cairo' ! -path "./src/cairo1/*" | parallel --halt soon,fail=1 run_tests +find ./tests/cairo_programs -name '*.cairo' ! -name 'test_vectors.cairo' ! -path "./src/cairo1/*" | parallel --halt soon,fail=1 run_tests {} # Capture the exit status of parallel exit_status=$? diff --git a/tools/make/ci_local.sh b/tools/make/ci_local.sh index a9b692c0..ea531ce8 100755 --- a/tools/make/ci_local.sh +++ b/tools/make/ci_local.sh @@ -5,10 +5,26 @@ run_check() { local script_path="$2" echo "Running $check_name..." - (source "$script_path") - echo "$check_name completed successfully." + echo "Script path: $script_path" + + # Attempt to execute the script + if [ -x "$script_path" ]; then + "$script_path" + local check_exit_code=$? + + if [ $check_exit_code -ne 0 ]; then + echo "$check_name failed with exit code $check_exit_code." + exit $check_exit_code + else + echo "$check_name completed successfully." + fi + else + echo "Error: Script $script_path is not executable or does not exist." + exit 1 + fi } + # Start time start_time=$SECONDS @@ -27,3 +43,5 @@ end_time=$SECONDS # Calculate and print the total runtime runtime=$((end_time - start_time)) echo "Total local CI Runtime: $runtime seconds." + +echo "All checks passed successfully." diff --git a/tools/make/format_cairo_files.sh b/tools/make/format_cairo_files.sh index ac40acd3..325e80cf 100755 --- a/tools/make/format_cairo_files.sh +++ b/tools/make/format_cairo_files.sh @@ -34,15 +34,25 @@ export -f format_scarb_project # Find all .cairo files under src/ and tests/ directories and format them in parallel echo "Formatting .cairo files..." -find ./src ./tests ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name '*.cairo' ! -path "./src/cairo1/*" | parallel --halt soon,fail=1 format_file +find ./src ./tests ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name '*.cairo' ! -path "./src/cairo1/*" | parallel --halt soon,fail=1 format_file {} + +# Capture the exit status of parallel for .cairo files +exit_status_cairo_files=$? # Find Scarb projects and execute format_scarb_project in each echo "Formatting Scarb projects..." -find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 format_scarb_project +find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 format_scarb_project {} + +# Capture the exit status of parallel for Scarb projects +exit_status_scarb_projects=$? -# Capture the exit status of parallel -exit_status=$? +# Determine the final exit status +if [ $exit_status_cairo_files -ne 0 ] || [ $exit_status_scarb_projects -ne 0 ]; then + final_exit_status=1 +else + final_exit_status=0 +fi -# Exit with the captured status -echo "Parallel execution exited with status: $exit_status" -exit $exit_status +# Exit with the determined status +echo "Parallel execution exited with status: $final_exit_status" +exit $final_exit_status diff --git a/tools/make/full_flow_test.sh b/tools/make/full_flow_test.sh index 6cca08f1..4d204e34 100755 --- a/tools/make/full_flow_test.sh +++ b/tools/make/full_flow_test.sh @@ -45,11 +45,17 @@ if [ ! -d "hdp-test" ]; then fi echo "Starting tests..." -find ./hdp-test/fixtures -name "input.json" | parallel --halt soon,fail=1 run_tests $filename +# Use find to locate all input.json files in hdp-test/fixtures directory and run them in parallel +find ./hdp-test/fixtures -name "input.json" | parallel --halt soon,fail=1 run_tests {} # Capture the exit status of parallel exit_status=$? # Exit with the captured status -echo "Parallel execution exited with status: $exit_status" +if [ $exit_status -ne 0 ]; then + echo "Parallel execution exited with status: $exit_status. Some tests failed." +else + echo "Parallel execution exited successfully. All tests passed." +fi + exit $exit_status diff --git a/tools/make/python_format_check.sh b/tools/make/python_format_check.sh index 1d0ad50c..6b27cca4 100755 --- a/tools/make/python_format_check.sh +++ b/tools/make/python_format_check.sh @@ -1,4 +1,9 @@ #!/bin/bash -echo "Checking code formatting..." -black --check . +echo "Checking code formatting with black..." +if black --check src/ tests/ tools/ packages/ setup.py; then + echo "Code formatting check passed." +else + echo "Code formatting check failed." + exit 1 +fi