From a0593230665759542719632b81b60f0bb57dbcb7 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Mon, 29 Jan 2024 14:13:02 -0500 Subject: [PATCH 1/7] Add gh issue creation test --- .github/workflows/pr-test-ghissue.yml | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/pr-test-ghissue.yml diff --git a/.github/workflows/pr-test-ghissue.yml b/.github/workflows/pr-test-ghissue.yml new file mode 100644 index 0000000000..39ea0ff995 --- /dev/null +++ b/.github/workflows/pr-test-ghissue.yml @@ -0,0 +1,43 @@ +name: TorchBench PR Test on A10G +on: + pull_request: + workflow_dispatch: + push: + branches: + - main + +env: + CONDA_ENV: "torchbench" + DOCKER_IMAGE: "ghcr.io/pytorch/torchbench:latest" + SETUP_SCRIPT: "/workspace/setup_instance.sh" + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + + +jobs: + pr-test: + runs-on: [self-hosted, linux.4xlarge] + timeout-minutes: 1440 # 24 hours + environment: docker-s3-upload + steps: + - name: Checkout TorchBench + uses: actions/checkout@v3 + with: + path: benchmark + - name: Generate the GH Issue file + run: | + echo "Test performance file" > ./benchmark/gh-issue.md + - name: Create the github issue + continue-on-error: true + uses: peter-evans/create-issue-from-file@v5 + with: + title: Test Performance Issue creation + content-filepath: ./benchmark/gh-issue.md + labels: | + torchbench-perf-report + - name: Teardown Linux + uses: pytorch/test-infra/.github/actions/teardown-linux@main + if: always() + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true From ac6d5f26e54289217830c993abf93f7338489062 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Mon, 29 Jan 2024 14:15:42 -0500 Subject: [PATCH 2/7] Testing to create the GitHub issue --- .github/workflows/build-gcp-docker.yml | 6 ++---- .github/workflows/pr-test-ghissue.yml | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-gcp-docker.yml b/.github/workflows/build-gcp-docker.yml index 874216def6..624fc5e9c4 100644 --- a/.github/workflows/build-gcp-docker.yml +++ b/.github/workflows/build-gcp-docker.yml @@ -1,12 +1,10 @@ name: TorchBench Nightly GCP Base Docker Build on: workflow_dispatch: - inputs: - nightly_date: - description: "PyTorch nightly version" - required: false + env: WITH_PUSH: "true" + jobs: build-push-docker: if: ${{ github.repository_owner == 'pytorch' }} diff --git a/.github/workflows/pr-test-ghissue.yml b/.github/workflows/pr-test-ghissue.yml index 39ea0ff995..dace5c39a7 100644 --- a/.github/workflows/pr-test-ghissue.yml +++ b/.github/workflows/pr-test-ghissue.yml @@ -1,4 +1,4 @@ -name: TorchBench PR Test on A10G +name: TorchBench PR Test for GitHub issue creation on: pull_request: workflow_dispatch: From 55a64bf530b9fdc235927eaa63f97fbdb901466d Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Mon, 29 Jan 2024 14:18:19 -0500 Subject: [PATCH 3/7] Assign token to ghissue --- .github/workflows/pr-test-ghissue.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-test-ghissue.yml b/.github/workflows/pr-test-ghissue.yml index dace5c39a7..e9a88f37d8 100644 --- a/.github/workflows/pr-test-ghissue.yml +++ b/.github/workflows/pr-test-ghissue.yml @@ -30,7 +30,8 @@ jobs: continue-on-error: true uses: peter-evans/create-issue-from-file@v5 with: - title: Test Performance Issue creation + title: Test Performance Issue Creation + token: ${{ secrets.TORCHBENCH_ACCESS_TOKEN }} content-filepath: ./benchmark/gh-issue.md labels: | torchbench-perf-report From e8d456bf58c07171398ece30c151fec61e87b031 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Mon, 29 Jan 2024 14:19:29 -0500 Subject: [PATCH 4/7] Another change --- .github/workflows/pr-test-ghissue.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/pr-test-ghissue.yml b/.github/workflows/pr-test-ghissue.yml index e9a88f37d8..a9b292b5e2 100644 --- a/.github/workflows/pr-test-ghissue.yml +++ b/.github/workflows/pr-test-ghissue.yml @@ -2,9 +2,6 @@ name: TorchBench PR Test for GitHub issue creation on: pull_request: workflow_dispatch: - push: - branches: - - main env: CONDA_ENV: "torchbench" @@ -12,9 +9,9 @@ env: SETUP_SCRIPT: "/workspace/setup_instance.sh" HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} - jobs: pr-test: + if: ${{ github.repository_owner == 'pytorch' }} runs-on: [self-hosted, linux.4xlarge] timeout-minutes: 1440 # 24 hours environment: docker-s3-upload From 3e7bd893758c0c1842146805245be51d3638bc26 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Mon, 29 Jan 2024 14:27:07 -0500 Subject: [PATCH 5/7] Revert another time --- .github/workflows/pr-test-ghissue.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-test-ghissue.yml b/.github/workflows/pr-test-ghissue.yml index a9b292b5e2..af1bade5b2 100644 --- a/.github/workflows/pr-test-ghissue.yml +++ b/.github/workflows/pr-test-ghissue.yml @@ -25,7 +25,7 @@ jobs: echo "Test performance file" > ./benchmark/gh-issue.md - name: Create the github issue continue-on-error: true - uses: peter-evans/create-issue-from-file@v5 + uses: peter-evans/create-issue-from-file@v4 with: title: Test Performance Issue Creation token: ${{ secrets.TORCHBENCH_ACCESS_TOKEN }} From 4fc139b0b7cca1d1f251e59d41ef4aa97077e5fb Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Mon, 29 Jan 2024 14:29:43 -0500 Subject: [PATCH 6/7] Use admin's secret for GitHub issue creation --- .github/workflows/pr-test-ghissue.yml | 41 ------------------- .../userbenchmark-regression-detector.yml | 1 + .github/workflows/v2-bisection.yml | 1 + .github/workflows/v3-bisection.yml | 1 + 4 files changed, 3 insertions(+), 41 deletions(-) delete mode 100644 .github/workflows/pr-test-ghissue.yml diff --git a/.github/workflows/pr-test-ghissue.yml b/.github/workflows/pr-test-ghissue.yml deleted file mode 100644 index af1bade5b2..0000000000 --- a/.github/workflows/pr-test-ghissue.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: TorchBench PR Test for GitHub issue creation -on: - pull_request: - workflow_dispatch: - -env: - CONDA_ENV: "torchbench" - DOCKER_IMAGE: "ghcr.io/pytorch/torchbench:latest" - SETUP_SCRIPT: "/workspace/setup_instance.sh" - HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} - -jobs: - pr-test: - if: ${{ github.repository_owner == 'pytorch' }} - runs-on: [self-hosted, linux.4xlarge] - timeout-minutes: 1440 # 24 hours - environment: docker-s3-upload - steps: - - name: Checkout TorchBench - uses: actions/checkout@v3 - with: - path: benchmark - - name: Generate the GH Issue file - run: | - echo "Test performance file" > ./benchmark/gh-issue.md - - name: Create the github issue - continue-on-error: true - uses: peter-evans/create-issue-from-file@v4 - with: - title: Test Performance Issue Creation - token: ${{ secrets.TORCHBENCH_ACCESS_TOKEN }} - content-filepath: ./benchmark/gh-issue.md - labels: | - torchbench-perf-report - - name: Teardown Linux - uses: pytorch/test-infra/.github/actions/teardown-linux@main - if: always() - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true diff --git a/.github/workflows/userbenchmark-regression-detector.yml b/.github/workflows/userbenchmark-regression-detector.yml index c919321160..abb4834829 100644 --- a/.github/workflows/userbenchmark-regression-detector.yml +++ b/.github/workflows/userbenchmark-regression-detector.yml @@ -78,6 +78,7 @@ jobs: uses: peter-evans/create-issue-from-file@v4 with: title: Optim Perf Signal Detected by TorchBench CI on ${{ env.TORCHBENCH_REGRESSION_DETECTED }} + token: ${{ secrets.TORCHBENCH_ACCESS_TOKEN }} content-filepath: ./benchmark/gh-issue.md labels: | torchbench-perf-report diff --git a/.github/workflows/v2-bisection.yml b/.github/workflows/v2-bisection.yml index 120150dd9e..1c05cf0840 100644 --- a/.github/workflows/v2-bisection.yml +++ b/.github/workflows/v2-bisection.yml @@ -62,6 +62,7 @@ jobs: - name: Create the github issue if: env.TORCHBENCH_PERF_BISECTION_NONEMPTY_SIGNAL uses: peter-evans/create-issue-from-file@v4 + token: ${{ secrets.TORCHBENCH_ACCESS_TOKEN }} with: title: V2 Performance Signal Detected by TorchBench CI on ${{ env.TORCHBENCH_PERF_BISECTION_NONEMPTY_SIGNAL }} content-filepath: ./bisection-result/gh-issue.md diff --git a/.github/workflows/v3-bisection.yml b/.github/workflows/v3-bisection.yml index 26fb442cd6..e07f2360d2 100644 --- a/.github/workflows/v3-bisection.yml +++ b/.github/workflows/v3-bisection.yml @@ -85,6 +85,7 @@ jobs: continue-on-error: true if: env.TORCHBENCH_BISECTION_COMMIT_FOUND_OR_FAILED uses: peter-evans/create-issue-from-file@v4 + token: ${{ secrets.TORCHBENCH_ACCESS_TOKEN }} with: title: V3 Performance Signal Detected by TorchBench Userbenchmark "torch-nightly" on ${{ env.TORCHBENCH_BISECTION_COMMIT_FOUND_OR_FAILED }} content-filepath: ./benchmark/gh-issue.md From 2cd1146f29aa0da5f3069857334e8d37ae0ac31c Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Mon, 29 Jan 2024 14:41:02 -0500 Subject: [PATCH 7/7] Renew README --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 25665edfd2..af2394251e 100644 --- a/README.md +++ b/README.md @@ -11,29 +11,29 @@ except for the torch products which are intended to be installed separately so different torch versions can be benchmarked. ### Using Pre-built Packages -We support Python 3.8+, and 3.10 is recommended. Conda is optional but suggested. To start with Python 3.10 in conda: +We support Python 3.8+, and 3.11 is recommended. Conda is optional but suggested. To start with Python 3.11 in conda: ``` # Using your current conda environment: -conda install -y python=3.10 +conda install -y python=3.11 # Or, using a new conda environment: -conda create -n torchbenchmark python=3.10 +conda create -n torchbenchmark python=3.11 conda activate torchbenchmark ``` -If you are running NVIDIA GPU tests, we support CUDA 11.8+, and use CUDA 11.8 as default: +If you are running NVIDIA GPU tests, we support both CUDA 11.8 and 12.1, and use CUDA 12.1 as default: ``` -conda install -y -c pytorch magma-cuda118 +conda install -y -c pytorch magma-cuda121 ``` Then install pytorch, torchvision, and torchaudio using conda: ``` -conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch-nightly -c nvidia +conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch-nightly -c nvidia ``` Or use pip: (but don't mix and match pip and conda for the torch family of libs! - [see notes below](#notes)) ``` -pip install --pre torch torchvision torchaudio -i https://download.pytorch.org/whl/nightly/cu118 +pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121 ``` Install the benchmark suite, which will recursively install dependencies for all the models. Currently, the repo is intended to be installed from the source tree.