Refactor test configs and CI (#170)

huggingface · Mar 30, 2024 · 39ccbf3 · 39ccbf3
1 parent 1ec7544
commit 39ccbf3
Show file tree

Hide file tree

Showing 125 changed files with 937 additions and 1,223 deletions.
diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml
diff --git a/.github/workflows/test_api_cpu.yaml b/.github/workflows/test_api_cpu.yaml
@@ -29,4 +29,4 @@ jobs:
           pip install -e .[testing,timm,diffusers]
 
       - name: Run tests
-        run: pytest -k "api and cpu"
+        run: pytest -s -k "api and cpu"
diff --git a/.github/workflows/test_api_cuda.yaml b/.github/workflows/test_api_cuda.yaml
@@ -12,41 +12,28 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_api_cuda_tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        image:
-          [
-            { torch_cuda: cu118, torch_pre_release: 0, cuda_version: 11.8.0 },
-            { torch_cuda: cu121, torch_pre_release: 1, cuda_version: 12.1.1 },
-          ]
-
-    runs-on: [multi-gpu, nvidia-gpu, 4-a10, ci]
+  run_api_cuda_tests:
+    runs-on: [single-gpu, nvidia-gpu, a10, ci]
 
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Build image
-        run: docker build
-          --file docker/cuda.dockerfile
-          --build-arg USER_ID=$(id -u)
-          --build-arg GROUP_ID=$(id -g)
-          --build-arg TORCH_CUDA=${{ matrix.image.torch_cuda }}
-          --build-arg CUDA_VERSION=${{ matrix.image.cuda_version }}
-          --tag opt-bench-cuda:${{ matrix.image.cuda_version }}
-          .
+      - name: Pull image
+        run: docker pull pytorch/pytorch:2.2.2-cuda12.1-cudnn8-runtime
 
       - name: Run tests
-        run: docker run
-          --rm
-          --gpus all
-          --shm-size 64G
-          --env USE_CUDA="1"
-          --env PROCESS_SPECIFIC_VRAM="0"
-          --volume $(pwd):/workspace/optimum-benchmark
-          --workdir /workspace/optimum-benchmark
-          --entrypoint /bin/bash
-          opt-bench-cuda:${{ matrix.image.cuda_version }}
-          -c "pip install -e .[testing,timm,diffusers] && pytest -k 'api and cuda' -x"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: pytorch/pytorch:2.2.2-cuda12.1-cudnn8-runtime
+          options: |
+            --rm 
+            --gpus all 
+            --shm-size 64G 
+            --env USE_CUDA="1" 
+            --env PROCESS_SPECIFIC_VRAM="0" 
+            --volume ${{ github.workspace }}:/workspace
+            --workdir /workspace
+          run: |
+            pip install -e .[testing,timm,diffusers]
+            pytest -s -x -k "api and cuda"
diff --git a/.github/workflows/test_api_misc.yaml b/.github/workflows/test_api_misc.yaml
@@ -29,4 +29,4 @@ jobs:
           pip install -e .[testing,timm,diffusers]
 
       - name: Run tests
-        run: pytest -k "api and not (cpu or cuda or rocm or tensorrt)"
+        run: pytest -s -k "api and not (cpu or cuda)"
diff --git a/.github/workflows/test_api_rocm.yaml b/.github/workflows/test_api_rocm.yaml
@@ -16,39 +16,43 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        image:
-          [
-            { torch_rocm: rocm5.6, torch_pre_release: 0, rocm_version: 5.6.1 },
-            { torch_rocm: rocm5.7, torch_pre_release: 1, rocm_version: 5.7.1 },
-          ]
+        image: [{ rocm_version: 5.7.1, torch_rocm: rocm5.7 }]
+
+    runs-on: [single-gpu, amd-gpu, mi250, ci]
 
-    runs-on: amd-gpu
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Target devices
+        run: |
+          echo "DEVICE: $DEVICE"
+          echo "DEVICE=$DEVICE" >> $GITHUB_ENV
+
       - name: Build image
         run: docker build
-          --file docker/rocm.dockerfile
           --build-arg USER_ID=$(id -u)
           --build-arg GROUP_ID=$(id -g)
           --build-arg TORCH_ROCM=${{ matrix.image.torch_rocm }}
-          --build-arg TORCH_PRE_RELEASE=${{ matrix.image.torch_pre_release }}
           --build-arg ROCM_VERSION=${{ matrix.image.rocm_version }}
           --tag opt-bench-rocm:${{ matrix.image.rocm_version }}
-          .
+          docker/rocm
 
       - name: Run tests
-        run: docker run
-          --rm
-          --pid host
-          --shm-size 64G
-          --env USE_ROCM="1"
-          --volume $(pwd):/workspace/optimum-benchmark
-          --workdir /workspace/optimum-benchmark
-          --device /dev/kfd
-          --device /dev/dri/renderD128
-          --device /dev/dri/renderD129
-          --entrypoint /bin/bash
-          opt-bench-rocm:${{ matrix.image.rocm_version }}
-          -c "pip install -e .[testing,timm,diffusers] && pytest -k 'api and cuda' -x"
+        uses: addnab/docker-run-action@v3
+        env:
+          DEVICE: ${{ env.DEVICE }}
+        with:
+          image: opt-bench-rocm:${{ matrix.image.rocm_version }}
+          options: |
+            --rm
+            --pid host
+            --shm-size 64G
+            --env USE_ROCM="1"
+            --device /dev/kfd
+            --device /dev/dri/${{ env.DEVICE }}
+            --volume ${{ github.workspace }}:/workspace
+            --workdir /workspace
+          run: |
+            pip install -e .[testing,timm,diffusers]
+            pytest -s -x -k "api and cuda"
diff --git a/.github/workflows/test_cli_cpu_neural_compressor.yaml b/.github/workflows/test_cli_cpu_neural_compressor.yaml
@@ -28,5 +28,5 @@ jobs:
           pip install --upgrade pip
           pip install -e .[testing,neural-compressor,diffusers,timm]
 
-      - name: Run CPU tests
-        run: pytest -k "cli and cpu and neural_compressor"
+      - name: Run tests
+        run: pytest -s -k "cli and cpu and neural_compressor"
diff --git a/.github/workflows/test_cli_cpu_onnxruntime.yaml b/.github/workflows/test_cli_cpu_onnxruntime.yaml
@@ -29,4 +29,4 @@ jobs:
           pip install -e .[testing,onnxruntime,diffusers,timm]
 
       - name: Run tests
-        run: pytest -k "cli and cpu and onnxruntime"
+        run: pytest -s -k "cli and cpu and onnxruntime"
diff --git a/.github/workflows/test_cli_cpu_openvino.yaml b/.github/workflows/test_cli_cpu_openvino.yaml
@@ -29,4 +29,4 @@ jobs:
           pip install -e .[testing,openvino,diffusers,timm]
 
       - name: Run tests
-        run: pytest -k "cli and cpu and openvino"
+        run: pytest -s -k "cli and cpu and openvino"
diff --git a/.github/workflows/test_cli_cpu_py_txi.yaml b/.github/workflows/test_cli_cpu_py_txi.yaml
@@ -44,4 +44,4 @@ jobs:
         run: docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-latest
 
       - name: Run tests
-        run: pytest -k "cli and cpu and py_txi"
+        run: pytest -s -k "cli and cpu and py_txi"
diff --git a/.github/workflows/test_cli_cpu_pytorch.yaml b/.github/workflows/test_cli_cpu_pytorch.yaml
@@ -29,4 +29,4 @@ jobs:
           pip install -e .[testing,diffusers,timm,peft]
 
       - name: Run tests
-        run: pytest -k "cli and cpu and pytorch"
+        run: pytest -s -k "cli and cpu and pytorch"
diff --git a/.github/workflows/test_cli_cuda_onnxruntime.yaml b/.github/workflows/test_cli_cuda_onnxruntime.yaml
@@ -19,25 +19,21 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Build image
-        run: docker build
-          --file docker/cuda.dockerfile
-          --build-arg USER_ID=$(id -u)
-          --build-arg GROUP_ID=$(id -g)
-          --build-arg CUDA_VERSION=11.8.0
-          --build-arg TORCH_CUDA=cu118
-          --tag opt-bench-cuda:11.8.0
-          .
+      - name: Pull image
+        run: docker pull pytorch/pytorch:2.2.2-cuda11.8-cudnn8-runtime
 
       - name: Run tests
-        run: docker run
-          --rm
-          --gpus all
-          --shm-size 64G
-          --env USE_CUDA="1"
-          --env PROCESS_SPECIFIC_VRAM="0"
-          --volume $(pwd):/workspace/optimum-benchmark
-          --workdir /workspace/optimum-benchmark
-          --entrypoint /bin/bash
-          opt-bench-cuda:11.8.0
-          -c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and cuda and onnxruntime' -x"
+        uses: addnab/docker-run-action@v3
+        with:
+          image: pytorch/pytorch:2.2.2-cuda11.8-cudnn8-runtime
+          options: |
+            --rm 
+            --gpus all 
+            --shm-size 64G 
+            --env USE_CUDA="1" 
+            --env PROCESS_SPECIFIC_VRAM="0" 
+            --volume ${{ github.workspace }}:/workspace
+            --workdir /workspace
+          run: |
+            pip install -e .[testing,onnxruntime-gpu,diffusers,timm] &&
+            pytest -x -s -k "cli and cuda and onnxruntime"
diff --git a/.github/workflows/test_cli_cuda_pytorch.yaml b/.github/workflows/test_cli_cuda_pytorch.yaml
diff --git a/.github/workflows/test_cli_cuda_pytorch_multi_gpu.yaml b/.github/workflows/test_cli_cuda_pytorch_multi_gpu.yaml
@@ -0,0 +1,39 @@
+name: CLI CUDA Pytorch Multi-GPU Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  cli_cuda_pytorch_multi_gpu_tests:
+    runs-on: [multi-gpu, nvidia-gpu, 4-a10, ci]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Pull image
+        run: docker pull pytorch/pytorch:2.2.2-cuda12.1-cudnn8-devel
+
+      - name: Run tests
+        uses: addnab/docker-run-action@v3
+        with:
+          image: pytorch/pytorch:2.2.2-cuda12.1-cudnn8-devel
+          options: |
+            --rm
+            --gpus all
+            --shm-size 64G
+            --env USE_CUDA="1"
+            --env PROCESS_SPECIFIC_VRAM="0"
+            --volume ${{ github.workspace }}:/workspace
+            --workdir /workspace
+          run: |
+            pip install -e .[testing,diffusers,timm,peft,deepspeed]
+            pytest -x -s -k "cli and cuda and pytorch and (dp or ddp or device_map or deepspeed)"