diff --git a/.github/scripts/install-platformio-esp32.sh b/.github/scripts/install-platformio-esp32.sh deleted file mode 100755 index 8cd6552eb80..00000000000 --- a/.github/scripts/install-platformio-esp32.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/bin/bash - -export PLATFORMIO_ESP32_PATH="$HOME/.platformio/packages/framework-arduinoespressif32" -PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git" - -TOOLCHAIN_VERSION="12.2.0+20230208" -ESPTOOLPY_VERSION="~1.40501.0" -ESPRESSIF_ORGANIZATION_NAME="espressif" -SDKCONFIG_DIR="$PLATFORMIO_ESP32_PATH/tools/esp32-arduino-libs" -SCRIPTS_DIR="./.github/scripts" -COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count" -CHECK_REQUIREMENTS="${SCRIPTS_DIR}/sketch_utils.sh check_requirements" - -echo "Installing Python Wheel ..." -pip install wheel > /dev/null 2>&1 - -echo "Installing PlatformIO ..." -pip install -U https://github.com/platformio/platformio/archive/master.zip > /dev/null 2>&1 - -echo "Installing Platform ESP32 ..." -python -m platformio platform install $PLATFORMIO_ESP32_URL > /dev/null 2>&1 - -echo "Replacing the package versions ..." -replace_script="import json; import os;" -replace_script+="fp=open(os.path.expanduser('~/.platformio/platforms/espressif32/platform.json'), 'r+');" -replace_script+="data=json.load(fp);" -# Use framework sources from the repository -replace_script+="data['packages']['framework-arduinoespressif32']['version'] = '*';" -replace_script+="del data['packages']['framework-arduinoespressif32']['owner'];" -# Use toolchain packages from the "espressif" organization -replace_script+="data['packages']['toolchain-xtensa-esp32']['owner']='$ESPRESSIF_ORGANIZATION_NAME';" -replace_script+="data['packages']['toolchain-xtensa-esp32s2']['owner']='$ESPRESSIF_ORGANIZATION_NAME';" -replace_script+="data['packages']['toolchain-riscv32-esp']['owner']='$ESPRESSIF_ORGANIZATION_NAME';" -# Update versions to use the upstream -replace_script+="data['packages']['toolchain-xtensa-esp32']['version']='$TOOLCHAIN_VERSION';" -replace_script+="data['packages']['toolchain-xtensa-esp32s2']['version']='$TOOLCHAIN_VERSION';" -replace_script+="data['packages']['toolchain-xtensa-esp32s3']['version']='$TOOLCHAIN_VERSION';" -replace_script+="data['packages']['toolchain-riscv32-esp']['version']='$TOOLCHAIN_VERSION';" -# Add new "framework-arduinoespressif32-libs" package -# Read "package_esp32_index.template.json" to extract a url to a zip package for "esp32-arduino-libs" -replace_script+="fpackage=open(os.path.join('package', 'package_esp32_index.template.json'), 'r+');" -replace_script+="package_data=json.load(fpackage);" -replace_script+="fpackage.close();" -replace_script+="libs_package_archive_url=next(next(system['url'] for system in tool['systems'] if system['host'] == 'x86_64-pc-linux-gnu') for tool in package_data['packages'][0]['tools'] if tool['name'] == 'esp32-arduino-libs');" -replace_script+="data['packages'].update({'framework-arduinoespressif32-libs':{'type':'framework','optional':False,'version':libs_package_archive_url}});" -replace_script+="data['packages']['toolchain-xtensa-esp32'].update({'optional':False});" -# esptool.py may require an upstream version (for now platformio is the owner) -replace_script+="data['packages']['tool-esptoolpy']['version']='$ESPTOOLPY_VERSION';" -# Save results -replace_script+="fp.seek(0);fp.truncate();json.dump(data, fp, indent=2);fp.close()" -python -c "$replace_script" - -if [ "$GITHUB_REPOSITORY" == "espressif/arduino-esp32" ]; then - echo "Linking Core..." - ln -s "$GITHUB_WORKSPACE" "$PLATFORMIO_ESP32_PATH" -else - echo "Cloning Core Repository ..." - git clone --recursive https://github.com/espressif/arduino-esp32.git "$PLATFORMIO_ESP32_PATH" > /dev/null 2>&1 -fi - -echo "PlatformIO for ESP32 has been installed" -echo "" - -function build_pio_sketch { # build_pio_sketch - if [ "$#" -lt 3 ]; then - echo "ERROR: Illegal number of parameters" - echo "USAGE: build_pio_sketch " - return 1 - fi - - local board="$1" - local options="$2" - local sketch="$3" - local sketch_dir - - sketch_dir=$(dirname "$sketch") - echo "" - echo "Compiling '$(basename "$sketch")' ..." - python -m platformio ci --board "$board" "$sketch_dir" --project-option="$options" -} - -function build_pio_sketches { # build_pio_sketches - if [ "$#" -lt 3 ]; then - echo "ERROR: Illegal number of parameters" - echo "USAGE: build_pio_sketches [ ]" - return 1 - fi - - local board=$1 - local options="$2" - local examples=$3 - local chunk_idex=$4 - local chunks_num=$5 - - if [ "$#" -lt 5 ]; then - chunk_idex="0" - chunks_num="1" - fi - - if [ "$chunks_num" -le 0 ]; then - echo "ERROR: Chunks count must be positive number" - return 1 - fi - if [ "$chunk_idex" -ge "$chunks_num" ]; then - echo "ERROR: Chunk index must be less than chunks count" - return 1 - fi - - set +e - ${COUNT_SKETCHES} "$examples" "esp32" - local sketchcount=$? - set -e - local sketches - sketches=$(cat sketches.txt) - rm -rf sketches.txt - - local chunk_size - local all_chunks - local start_index - local end_index - local start_num - - chunk_size=$(( sketchcount / chunks_num )) - all_chunks=$(( chunks_num * chunk_size )) - if [ "$all_chunks" -lt "$sketchcount" ]; then - chunk_size=$(( chunk_size + 1 )) - fi - - start_index=$(( chunk_idex * chunk_size )) - if [ "$sketchcount" -le "$start_index" ]; then - echo "Skipping job" - return 0 - fi - - end_index=$(( $(( chunk_idex + 1 )) * chunk_size )) - if [ "$end_index" -gt "$sketchcount" ]; then - end_index=$sketchcount - fi - - start_num=$(( start_index + 1 )) - echo "Found $sketchcount Sketches"; - echo "Chunk Count : $chunks_num" - echo "Chunk Size : $chunk_size" - echo "Start Sketch: $start_num" - echo "End Sketch : $end_index" - - local sketchnum=0 - for sketch in $sketches; do - local sketchdir - local sketchdirname - local sketchname - local is_target - local has_requirements - - sketchdir=$(dirname "$sketch") - sketchdirname=$(basename "$sketchdir") - sketchname=$(basename "$sketch") - - if [[ "$sketchdirname.ino" != "$sketchname" ]]; then - continue - elif [ -f "$sketchdir"/ci.json ]; then - # If the target is listed as false, skip the sketch. Otherwise, include it. - is_target=$(jq -r '.targets[esp32]' "$sketchdir"/ci.json) - if [[ "$is_target" == "false" ]]; then - continue - fi - - has_requirements=$(${CHECK_REQUIREMENTS} "$sketchdir" "$SDKCONFIG_DIR/esp32/sdkconfig") - if [ "$has_requirements" == "0" ]; then - continue - fi - fi - - sketchnum=$((sketchnum + 1)) - if [ "$sketchnum" -le "$start_index" ] \ - || [ "$sketchnum" -gt "$end_index" ]; then - continue - fi - build_pio_sketch "$board" "$options" "$sketch" - local result=$? - if [ $result -ne 0 ]; then - return $result - fi - done - return 0 -} diff --git a/.github/scripts/on-push.sh b/.github/scripts/on-push.sh index 5158df3cc9b..aafc9de0596 100755 --- a/.github/scripts/on-push.sh +++ b/.github/scripts/on-push.sh @@ -55,14 +55,11 @@ CHUNK_INDEX=$1 CHUNKS_CNT=$2 BUILD_LOG=$3 SKETCHES_FILE=$4 -BUILD_PIO=0 if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then CHUNK_INDEX=0 CHUNKS_CNT=1 elif [ "$CHUNK_INDEX" -gt "$CHUNKS_CNT" ] && [ "$CHUNKS_CNT" -ge 2 ]; then CHUNK_INDEX=$CHUNKS_CNT -elif [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then - BUILD_PIO=1 fi if [ -z "$BUILD_LOG" ] || [ "$BUILD_LOG" -le 0 ]; then @@ -73,54 +70,35 @@ fi #git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1 SCRIPTS_DIR="./.github/scripts" -if [ "$BUILD_PIO" -eq 0 ]; then - source "${SCRIPTS_DIR}/install-arduino-cli.sh" - source "${SCRIPTS_DIR}/install-arduino-core-esp32.sh" - - SKETCHES_ESP32=( - "$ARDUINO_ESP32_PATH/libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" - "$ARDUINO_ESP32_PATH/libraries/BLE/examples/Server/Server.ino" - "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino" - "$ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino" - ) - #create sizes_file - sizes_file="$GITHUB_WORKSPACE/cli_compile_$CHUNK_INDEX.json" - - if [ "$BUILD_LOG" -eq 1 ]; then - #create sizes_file and echo start of JSON array with "boards" key - echo "{\"boards\": [" > "$sizes_file" - fi - - #build sketches for different targets - build "esp32p4" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" - build "esp32s3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" - build "esp32s2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" - build "esp32c3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" - build "esp32c6" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" - build "esp32h2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" - build "esp32" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" - - if [ "$BUILD_LOG" -eq 1 ]; then - #remove last comma from the last JSON object - sed -i '$ s/,$//' "$sizes_file" - #echo end of JSON array - echo "]}" >> "$sizes_file" - fi -else - source "${SCRIPTS_DIR}/install-platformio-esp32.sh" - # PlatformIO ESP32 Test - BOARD="esp32dev" - OPTIONS="board_build.partitions = huge_app.csv" - build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \ - build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \ - build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \ - build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/BLE/examples/Server/Server.ino" && \ - build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino" - - # Basic sanity testing for other series - for board in "esp32-c3-devkitm-1" "esp32-s2-saola-1" "esp32-s3-devkitc-1"; do - python -m platformio ci --board "$board" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.partitions = huge_app.csv" - done +source "${SCRIPTS_DIR}/install-arduino-cli.sh" +source "${SCRIPTS_DIR}/install-arduino-core-esp32.sh" + +SKETCHES_ESP32=( + "$ARDUINO_ESP32_PATH/libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" + "$ARDUINO_ESP32_PATH/libraries/BLE/examples/Server/Server.ino" + "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino" + "$ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino" +) +#create sizes_file +sizes_file="$GITHUB_WORKSPACE/cli_compile_$CHUNK_INDEX.json" + +if [ "$BUILD_LOG" -eq 1 ]; then + #create sizes_file and echo start of JSON array with "boards" key + echo "{\"boards\": [" > "$sizes_file" +fi - #build_pio_sketches "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries" +#build sketches for different targets +build "esp32p4" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" +build "esp32s3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" +build "esp32s2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" +build "esp32c3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" +build "esp32c6" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" +build "esp32h2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" +build "esp32" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" + +if [ "$BUILD_LOG" -eq 1 ]; then + #remove last comma from the last JSON object + sed -i '$ s/,$//' "$sizes_file" + #echo end of JSON array + echo "]}" >> "$sizes_file" fi diff --git a/.github/scripts/on-release.sh b/.github/scripts/on-release.sh index 119c66e1eab..242cee93d0b 100755 --- a/.github/scripts/on-release.sh +++ b/.github/scripts/on-release.sh @@ -239,7 +239,7 @@ cp -f "$GITHUB_WORKSPACE/tools/gen_insights_package.py" "$PKG_DIR/tools/" cp -f "$GITHUB_WORKSPACE/tools/gen_insights_package.exe" "$PKG_DIR/tools/" cp -Rf "$GITHUB_WORKSPACE/tools/partitions" "$PKG_DIR/tools/" cp -Rf "$GITHUB_WORKSPACE/tools/ide-debug" "$PKG_DIR/tools/" -cp -f "$GITHUB_WORKSPACE/tools/platformio-build.py" "$PKG_DIR/tools/" +cp -f "$GITHUB_WORKSPACE/tools/pioarduino-build.py" "$PKG_DIR/tools/" # Remove unnecessary files in the package folder echo "Cleaning up folders ..." diff --git a/.github/scripts/set_push_chunks.sh b/.github/scripts/set_push_chunks.sh index 047cae6efd8..ff0af7da6e8 100644 --- a/.github/scripts/set_push_chunks.sh +++ b/.github/scripts/set_push_chunks.sh @@ -79,7 +79,6 @@ chunks+="]" echo "build_libraries=$BUILD_LIBRARIES" echo "build_static_sketches=$BUILD_STATIC_SKETCHES" echo "build_idf=$BUILD_IDF" - echo "build_platformio=$BUILD_PLATFORMIO" echo "chunk_count=$chunks_count" echo "chunks=$chunks" } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4f30bdbb844..e6f497998b0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -61,7 +61,6 @@ jobs: build_libraries: ${{ steps.set-chunks.outputs.build_libraries }} build_static_sketches: ${{ steps.set-chunks.outputs.build_static_sketches }} build_idf: ${{ steps.set-chunks.outputs.build_idf }} - build_platformio: ${{ steps.set-chunks.outputs.build_platformio }} chunk_count: ${{ steps.set-chunks.outputs.chunk_count }} chunks: ${{ steps.set-chunks.outputs.chunks }} steps: @@ -77,11 +76,9 @@ jobs: files_yaml: | core: - '.github/**' - - '!.github/scripts/install-platformio-esp32.sh' - 'cores/**' - 'package/**' - 'tools/**' - - '!tools/platformio-build.py' - 'platform.txt' - 'programmers.txt' - "variants/esp32/**/*" @@ -110,10 +107,6 @@ jobs: - 'Kconfig.projbuild' - 'CMakeLists.txt' - "variants/esp32c2/**/*" - platformio: - - 'package.json' - - '.github/scripts/install-platformio-esp32.sh' - - 'tools/platformio-build.py' - name: Set chunks id: set-chunks @@ -121,7 +114,6 @@ jobs: LIB_FILES: ${{ steps.changed-files.outputs.libraries_all_changed_files }} IS_PR: ${{ github.event_name == 'pull_request' }} MAX_CHUNKS: ${{ env.MAX_CHUNKS }} - BUILD_PLATFORMIO: ${{ steps.changed-files.outputs.platformio_any_changed == 'true' }} BUILD_IDF: ${{ steps.changed-files.outputs.idf_any_changed == 'true' }} BUILD_LIBRARIES: ${{ steps.changed-files.outputs.libraries_any_changed == 'true' }} BUILD_STATIC_SKETCHES: ${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }} @@ -212,28 +204,6 @@ jobs: - name: Build Sketches run: bash ./.github/scripts/on-push.sh - # # PlatformIO on Windows, Ubuntu and Mac - # build-platformio: - # name: PlatformIO on ${{ matrix.os }} - # needs: gen-chunks - # if: | - # needs.gen-chunks.outputs.build_all == 'true' || - # needs.gen-chunks.outputs.build_static_sketches == 'true' || - # needs.gen-chunks.outputs.build_platformio == 'true' - # runs-on: ${{ matrix.os }} - # strategy: - # fail-fast: false - # matrix: - # os: [ubuntu-latest, windows-latest, macOS-latest] - # steps: - # - uses: actions/checkout@v4 - # - uses: actions/setup-python@v5 - # with: - # python-version: "3.x" - # - name: Build Sketches - # run: bash ./.github/scripts/on-push.sh 1 1 #equal and non-zero to trigger PIO - - # ESP-IDF component build build-esp-idf-component: name: Build with ESP-IDF ${{ matrix.idf_ver }} for ${{ matrix.idf_target }} needs: gen-chunks diff --git a/platform.txt b/platform.txt index b075e539866..44353ab3bf4 100644 --- a/platform.txt +++ b/platform.txt @@ -44,7 +44,7 @@ compiler.warning_flags.more=-Wall compiler.warning_flags.all=-Wall -Wextra # Additional flags specific to Arduino (not based on IDF flags). -# Update tools/platformio-build.py when changing these flags. +# Update tools/pioarduino-build.py when changing these flags. compiler.common_werror_flags=-Werror=return-type # Compile Flags diff --git a/tools/platformio-build.py b/tools/pioarduino-build.py similarity index 98% rename from tools/platformio-build.py rename to tools/pioarduino-build.py index 485879944eb..3335a716888 100644 --- a/tools/platformio-build.py +++ b/tools/pioarduino-build.py @@ -22,7 +22,7 @@ http://arduino.cc/en/Reference/HomePage """ -# Extends: https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py +# Extends: https://github.com/pioarduino/platform-espressif32/blob/develop/builder/main.py from os.path import abspath, basename, isdir, isfile, join from copy import deepcopy @@ -160,7 +160,7 @@ def add_tinyuf2_extra_image(): join( FRAMEWORK_LIBS_DIR, build_mcu, - "platformio-build.py", + "pioarduino-build.py", ) )