From eae300496fa971dff341c202338de0d3d9283635 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 30 Dec 2024 12:29:30 -0800 Subject: [PATCH 1/5] Handle illegal character in benchmark configs filename --- .github/workflows/android-perf.yml | 2 +- .github/workflows/apple-perf.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index 48625898ae..bfbd028241 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -111,7 +111,7 @@ jobs: # so let's just sed it sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2 - BENCHMARK_CONFIG_ID="${{ matrix.model }}_${{ matrix.config }}" + BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # The config for this benchmark runs, we save it in the test spec so that it can be fetched # later by the upload script sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' android-llm-device-farm-test-spec.yml.j2 diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index b35daebaf5..8cd1a972be 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -113,7 +113,7 @@ jobs: # so let's just sed it sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' default-ios-device-farm-appium-test-spec.yml.j2 - BENCHMARK_CONFIG_ID="${{ matrix.model }}_${{ matrix.config }}" + BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # The config for this benchmark runs, we save it in the test spec so that it can be fetched # later by the upload script sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' default-ios-device-farm-appium-test-spec.yml.j2 From 9b5876c3b89f257a68a531b7c2bdb61ad9c9e55f Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 30 Dec 2024 12:34:50 -0800 Subject: [PATCH 2/5] Only change the filename --- .github/workflows/android-perf.yml | 5 +++-- .github/workflows/apple-perf.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index bfbd028241..dd20480997 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -111,7 +111,7 @@ jobs: # so let's just sed it sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2 - BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') + BENCHMARK_CONFIG_ID="${{ matrix.model }}_${{ matrix.config }}" # The config for this benchmark runs, we save it in the test spec so that it can be fetched # later by the upload script sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' android-llm-device-farm-test-spec.yml.j2 @@ -120,8 +120,9 @@ jobs: # Just print the test spec for debugging cat android-llm-device-farm-test-spec.yml + BENCHMARK_CONFIG_FILENAME=$(echo "${BENCHMARK_CONFIG_ID}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # Save the benchmark configs so that we can use it later in the dashboard - echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" + echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_FILENAME}.json" - name: Upload the spec uses: seemethere/upload-artifact-s3@v5 diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index 8cd1a972be..54954dbc2c 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -113,7 +113,7 @@ jobs: # so let's just sed it sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' default-ios-device-farm-appium-test-spec.yml.j2 - BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') + BENCHMARK_CONFIG_ID="${{ matrix.model }}_${{ matrix.config }}" # The config for this benchmark runs, we save it in the test spec so that it can be fetched # later by the upload script sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' default-ios-device-farm-appium-test-spec.yml.j2 @@ -122,8 +122,9 @@ jobs: # Just print the test spec for debugging cat default-ios-device-farm-appium-test-spec.yml + BENCHMARK_CONFIG_FILENAME=$(echo "${BENCHMARK_CONFIG_ID}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # Save the benchmark configs so that we can use it later in the dashboard - echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" + echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_FILENAME}.json" - name: Upload the spec uses: seemethere/upload-artifact-s3@v5 From 966785014c50858549c77da3b9e636bebbd72181 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 30 Dec 2024 12:48:59 -0800 Subject: [PATCH 3/5] Save benchmark config filename as GH output --- .github/workflows/android-perf.yml | 4 +++- .github/workflows/apple-perf.yml | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index dd20480997..fbc34142c0 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -98,6 +98,7 @@ jobs: - uses: actions/checkout@v3 - name: Prepare the spec + id: prepare shell: bash env: BENCHMARK_CONFIG: ${{ toJSON(matrix) }} @@ -123,6 +124,7 @@ jobs: BENCHMARK_CONFIG_FILENAME=$(echo "${BENCHMARK_CONFIG_ID}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # Save the benchmark configs so that we can use it later in the dashboard echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_FILENAME}.json" + echo "benchmark-config-filename=${BENCHMARK_CONFIG_FILENAME}" >> $GITHUB_OUTPUT - name: Upload the spec uses: seemethere/upload-artifact-s3@v5 @@ -142,7 +144,7 @@ jobs: ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ retention-days: 1 if-no-files-found: error - path: extension/benchmark/android/benchmark/${{ matrix.model }}_${{ matrix.config }}.json + path: extension/benchmark/android/benchmark/${{ steps.prepare.outputs.benchmark-config-filename }}.json export-models: name: export-models diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index 54954dbc2c..eb9027e248 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -125,6 +125,7 @@ jobs: BENCHMARK_CONFIG_FILENAME=$(echo "${BENCHMARK_CONFIG_ID}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # Save the benchmark configs so that we can use it later in the dashboard echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_FILENAME}.json" + echo "benchmark-config-filename=${BENCHMARK_CONFIG_FILENAME}" >> $GITHUB_OUTPUT - name: Upload the spec uses: seemethere/upload-artifact-s3@v5 @@ -144,7 +145,7 @@ jobs: ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ retention-days: 1 if-no-files-found: error - path: extension/benchmark/apple/Benchmark/${{ matrix.model }}_${{ matrix.config }}.json + path: extension/benchmark/apple/Benchmark/${{ steps.prepare.outputs.benchmark-config-filename }}.json export-models: name: export-models From be7ba65eb09805624907d4b7dd3c7cdcfebccf3a Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 30 Dec 2024 12:53:06 -0800 Subject: [PATCH 4/5] Typo --- .github/workflows/apple-perf.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index eb9027e248..564ff80590 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -100,6 +100,7 @@ jobs: - uses: actions/checkout@v3 - name: Prepare the spec + id: prepare shell: bash env: BENCHMARK_CONFIG: ${{ toJSON(matrix) }} From 2f18c0bb25c5e3cad73b8a2de5af5180937daf04 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 30 Dec 2024 16:59:25 -0800 Subject: [PATCH 5/5] Also update the spec --- .github/workflows/android-perf.yml | 9 ++++----- .github/workflows/apple-perf.yml | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index fbc34142c0..1874a4fd6e 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -112,7 +112,7 @@ jobs: # so let's just sed it sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2 - BENCHMARK_CONFIG_ID="${{ matrix.model }}_${{ matrix.config }}" + BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # The config for this benchmark runs, we save it in the test spec so that it can be fetched # later by the upload script sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' android-llm-device-farm-test-spec.yml.j2 @@ -121,10 +121,9 @@ jobs: # Just print the test spec for debugging cat android-llm-device-farm-test-spec.yml - BENCHMARK_CONFIG_FILENAME=$(echo "${BENCHMARK_CONFIG_ID}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # Save the benchmark configs so that we can use it later in the dashboard - echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_FILENAME}.json" - echo "benchmark-config-filename=${BENCHMARK_CONFIG_FILENAME}" >> $GITHUB_OUTPUT + echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" + echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT - name: Upload the spec uses: seemethere/upload-artifact-s3@v5 @@ -144,7 +143,7 @@ jobs: ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ retention-days: 1 if-no-files-found: error - path: extension/benchmark/android/benchmark/${{ steps.prepare.outputs.benchmark-config-filename }}.json + path: extension/benchmark/android/benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json export-models: name: export-models diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index 564ff80590..f4fbc0128b 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -114,7 +114,7 @@ jobs: # so let's just sed it sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' default-ios-device-farm-appium-test-spec.yml.j2 - BENCHMARK_CONFIG_ID="${{ matrix.model }}_${{ matrix.config }}" + BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # The config for this benchmark runs, we save it in the test spec so that it can be fetched # later by the upload script sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' default-ios-device-farm-appium-test-spec.yml.j2 @@ -123,10 +123,9 @@ jobs: # Just print the test spec for debugging cat default-ios-device-farm-appium-test-spec.yml - BENCHMARK_CONFIG_FILENAME=$(echo "${BENCHMARK_CONFIG_ID}" | sed -e 's/[^A-Za-z0-9._-]/_/g') # Save the benchmark configs so that we can use it later in the dashboard - echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_FILENAME}.json" - echo "benchmark-config-filename=${BENCHMARK_CONFIG_FILENAME}" >> $GITHUB_OUTPUT + echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json" + echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT - name: Upload the spec uses: seemethere/upload-artifact-s3@v5 @@ -146,7 +145,7 @@ jobs: ${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/ retention-days: 1 if-no-files-found: error - path: extension/benchmark/apple/Benchmark/${{ steps.prepare.outputs.benchmark-config-filename }}.json + path: extension/benchmark/apple/Benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json export-models: name: export-models