Merge pull request #36 from HerodotusDev/feat/dockerize

Feat/dockerize

.dockerignore

@@ -0,0 +1,37 @@
+# Python
+*.pyc
+*.pyo
+*.pyd
+__pycache__
+*.swp
+*.swo
+.Python
+env/
+venv/
+ENV/
+*.egg-info
+.eggs/
+.dist-info
+*.egg
+*.whl
+.build/
+.tox/
+.coverage
+coverage.*
+.cache
+.pytest_cache/
+
+# Rust
+target/
+Cargo.lock
+
+# Docker files
+Dockerfile
+docker-compose.yml
+
+# Version control
+.git/
+.gitignore
+
+hdp-test
+cairo-vm

.github/workflows/merge.yml

@@ -0,0 +1,32 @@
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: "lab:latest"
+          driver: cloud
+          endpoint: "dataprocessor/github"
+          cleanup: true
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          file: environment.dockerfile
+          tags: "dataprocessor/hdp-cairo:latest"
+          platforms: linux/amd64,linux/arm64
+          push: true

.github/workflows/pull-request.yml

@@ -0,0 +1,87 @@
+name: CI Build and Test Workflow
+
+on:
+  pull_request:
+    branches:
+      - main
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: "lab:latest"
+          driver: cloud
+          endpoint: "dataprocessor/github"
+          cleanup: true
+      - name: Build and cache Docker image
+        uses: docker/build-push-action@v5
+        with:
+          file: environment.dockerfile
+          tags: "dataprocessor/hdp-cairo:latest"
+#
+      - name: Setup Scarb
+        uses: software-mansion/setup-scarb@v1
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Cache Python environment
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pip
+            venv
+          key: ${{ runner.os }}-python-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('tools/make/setup.sh') }}
+          restore-keys: |
+              ${{ runner.os }}-python-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('tools/make/setup.sh') }}
+              ${{ runner.os }}-python-
+
+#      - name: Cache Cargo directories
+#        uses: actions/cache@v3
+#        with:
+#          path: |
+#            ~/.cargo/bin/
+#            ~/.cargo/registry/index/
+#            ~/.cargo/registry/cache/
+#            ~/.cargo/git/db/
+#            cairo-vm/cargo-run/target
+#          key: ${{ runner.os }}-cargo-${{ hashFiles('tools/make/install_cairo1_run.sh') }}
+#          restore-keys: ${{ runner.os }}-cargo-
+
+      - name: Install Dependencies
+        run: make setup
+
+      - name: Check Python formatting
+        run: |
+          source venv/bin/activate
+          ./tools/make/python_format_check.sh
+
+      - name: Check Cairo formatting
+        run: |
+          source venv/bin/activate
+          ./tools/make/cairo_format_check.sh
+      - name: Compile Cairo files
+        run: |
+          source venv/bin/activate
+          make build
+      - name: Run Unit Cairo tests
+        env:
+          RPC_URL_MAINNET: ${{ secrets.RPC_URL_MAINNET }}
+        run: |
+          source venv/bin/activate
+          ./tools/make/cairo_tests.sh
+      - name: Run Full Flow tests
+        run: |
+          source venv/bin/activate
+          ./tools/make/full_flow_test.sh

.gitignore

@@ -52,3 +52,4 @@ hdp/
 # Whitelist - these files should not be ignored
 !tests/cairo_programs/fixtures/*.json
 !tools/js/package.json
+!packages/cairo-lang-0.13.1.zip

README.md

@@ -2,6 +2,11 @@
 
 Cairo HDP is a collection of Cairo0 programs designed to verify inclusion proofs and perform computations on the data. These computations can be verified on-chain, enabling trustless operations on any historical data from Ethereum or integrated EVM chains.
 
+## Run docker test run
+```bash
+docker build -t hdp-cairo . && docker run hdp-cairo
+```
+
 ## Installation and Setup
 
 To install the required dependencies and set up the Python virtual environment, run:

environment.dockerfile

@@ -0,0 +1,33 @@
+# Use the official Python 3.9 image from the Docker Hub
+FROM python:3.9.0
+
+# Set the default shell to bash and the working directory in the container
+SHELL ["/bin/bash", "-ci"]
+WORKDIR /hdp
+
+# Install Rust using Rustup
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    echo 'export PATH="/root/.cargo/bin:$PATH"' >> /root/.bashrc
+
+# Add Cargo executables to PATH
+RUN mkdir -p /root/.local/bin && \
+    echo 'export PATH="/root/.local/bin:$PATH"' >> /root/.bashrc
+
+# Install cairo1-run into PATH
+RUN git clone https://github.com/HerodotusDev/cairo-vm.git && \
+    cd cairo-vm && git checkout aecbb3f01dacb6d3f90256c808466c2c37606252 && \
+    cd cairo1-run && cargo install --path .
+
+# Copy project requirements and install them
+COPY tools/make/requirements.txt tools/make/requirements.txt
+RUN python -m pip install --upgrade pip && pip install -r tools/make/requirements.txt
+
+# Copy and install Contract bootloader SNOS dependencies
+COPY packages/cairo-lang-0.13.1.zip packages/cairo-lang-0.13.1.zip
+RUN pip install packages/cairo-lang-0.13.1.zip
+
+# Copy the entire project into the image
+COPY . .
+
+# Install Python package modules
+RUN pip install .

tests/cairo_programs/rlp.cairo

@@ -25,7 +25,6 @@ func main{range_check_ptr, bitwise_ptr: BitwiseBuiltin*, keccak_ptr: KeccakBuilt
             uint256_reverse_endian,
             split_128,
             reverse_endian,
-            bytes_to_8_bytes_chunks,
         )
         import rlp
     %}

tools/make/build.sh

@@ -23,7 +23,7 @@ export -f process_cairo_file
 mkdir -p build/compiled_cairo_files
 
 # Find Cairo files and process them in parallel
-find ./src ./tests/cairo_programs ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name "*.cairo" ! -path "./src/cairo1/*" | parallel --halt now,fail=1 process_cairo_file
+find ./src ./tests/cairo_programs ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name "*.cairo" ! -path "./src/cairo1/*" | parallel --halt now,fail=1 process_cairo_file {}
 
 # Capture the exit status of parallel
 exit_status=$?

tools/make/build_cairo1.sh

@@ -9,16 +9,15 @@ process_scarb_project() {
     # Change to the project directory and build the project
     if (cd "$project_dir" && scarb build); then
         echo "Copying built files from $project_dir/target/dev to $build_dir"
-        cp "$project_dir/target/dev/"* "$build_dir/"
-
-        local status=$?
-        if [ $status -eq 0 ]; then
+        if cp "$project_dir/target/dev/"* "$build_dir/"; then
             echo "$(date '+%Y-%m-%d %H:%M:%S') - Successfully built and copied files for Scarb project in $project_dir"
         else
             echo "$(date '+%Y-%m-%d %H:%M:%S') - Built Scarb project in $project_dir, but failed to copy files"
+            return 1
         fi
     else
         echo "$(date '+%Y-%m-%d %H:%M:%S') - Failed to build Scarb project in $project_dir"
+        return 1
     fi
 }
 
@@ -29,7 +28,7 @@ export -f process_scarb_project
 mkdir -p build/compiled_cairo_files
 
 # Find Scarb projects and execute process_scarb_project in each
-find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 process_scarb_project
+find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 process_scarb_project {}
 
 # Capture the exit status of parallel
 exit_status=$?

tools/make/cairo_format_check.sh

@@ -30,15 +30,25 @@ export -f format_scarb_project
 
 # Find all .cairo files and format them in parallel
 echo "Finding and formatting .cairo files..."
-find ./src ./tests ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name '*.cairo' ! -path "./src/cairo1/*" | parallel --halt soon,fail=1 format_file
+find ./src ./tests ./packages/hdp_bootloader/bootloader ./packages/hdp_bootloader/builtin_selection -name '*.cairo' ! -path "./src/cairo1/*" | parallel --halt soon,fail=1 format_file {}
+
+# Capture the exit status of parallel for .cairo files
+exit_status_cairo_files=$?
 
 # Find Scarb projects and format them in parallel
 echo "Finding and formatting Scarb projects..."
-find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 format_scarb_project
+find ./src/cairo1 -mindepth 1 -maxdepth 1 -type d | parallel --halt now,fail=1 format_scarb_project {}
+
+# Capture the exit status of parallel for Scarb projects
+exit_status_scarb_projects=$?
 
-# Capture the exit status of parallel
-exit_status=$?
+# Determine the final exit status
+if [ $exit_status_cairo_files -ne 0 ] || [ $exit_status_scarb_projects -ne 0 ]; then
+    final_exit_status=1
+else
+    final_exit_status=0
+fi
 
-# Exit with the captured status
-echo "Parallel execution exited with status: $exit_status"
-exit $exit_status
+# Exit with the determined status
+echo "Parallel execution exited with status: $final_exit_status"
+exit $final_exit_status