From e2526714e23284c9a722fe9b714ef66c0e39c7f5 Mon Sep 17 00:00:00 2001 From: PeixuanZuo <94887879+PeixuanZuo@users.noreply.github.com> Date: Wed, 5 Jul 2023 13:55:28 +0800 Subject: [PATCH] [ROCm] Move MIGraphX build step on CPU only machine (#16582) - Move MIGraphX build step on CPU only machine - Use ccache on build step - Not pass host uid into docker build process. --- .../linux-migraphx-ci-pipeline.yml | 115 ++++++++++++++---- .../migraphx-ci-pipeline-env.Dockerfile | 11 +- 2 files changed, 99 insertions(+), 27 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml index a3e2e4813697..f934eba5cbc5 100644 --- a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml @@ -1,23 +1,32 @@ trigger: none name: 'linux_ci_$(Date:yyyyMMdd)_$(Rev:r)' + +# gid of video and render group on gcramdrr1-mi100-085 and -86 +variables: + - name: video + value: 44 + - name: render + value: 109 + - name: RocmVersion + value: 5.5 + jobs: -- job: AMDMIGraphX_CI +- job: Linux_Build + variables: + skipComponentGovernanceDetection: true + CCACHE_DIR: $(Pipeline.Workspace)/ccache + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] workspace: clean: all - pool: 'AMD-GPU' - timeoutInMinutes: 180 - - # gid of video and render group on gcramdrr1-mi100-085 and -86 - variables: - - name: video - value: 44 - - name: render - value: 109 - - name: RocmVersion - value: 5.5 + pool: onnxruntime-Ubuntu2004-AMD-CPU + timeoutInMinutes: 120 steps: + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + - checkout: self clean: true submodules: recursive @@ -26,24 +35,39 @@ jobs: parameters: Dockerfile: tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" Repository: onnxruntimetrainingmigraphx-cibuild-rocm$(RocmVersion) + - task: Cache@2 + inputs: + key: '"$(TODAY)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"' + path: $(CCACHE_DIR) + cacheHitVar: CACHE_RESTORED + restoreKeys: | + "$(TODAY)" | "$(Build.SourceBranch)" + "$(TODAY)" | + displayName: Cache Task + + - script: mkdir -p $(CCACHE_DIR) + condition: ne(variables.CACHE_RESTORED, 'true') + displayName: Create Cache Dir + - task: CmdLine@2 inputs: script: | docker run --rm \ --security-opt seccomp=unconfined \ --shm-size=1024m \ - --device=/dev/kfd \ - --device=/dev/dri/renderD$DRIVER_RENDER \ - --group-add $(video) \ - --group-add $(render) \ - --user onnxruntimedev \ + --user $UID:$(id -g $USER) \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ + --volume $(CCACHE_DIR):/cache \ + -e CCACHE_DIR=/cache \ --workdir /onnxruntime_src \ onnxruntimetrainingmigraphx-cibuild-rocm$(RocmVersion) \ + /bin/bash -c " + set -ex; \ + env; \ + ccache -s; \ python tools/ci_build/build.py \ --config Release \ --cmake_extra_defines \ @@ -58,13 +82,57 @@ jobs: --update \ --build_dir /build \ --build \ - --parallel 32 \ + --parallel \ --build_wheel \ --skip_submodule_sync \ - --skip_tests --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest + --use_cache \ + --skip_tests --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest; \ + ccache -sv; \ + ccache -z" workingDirectory: $(Build.SourcesDirectory) displayName: 'Build onnxruntime' + - task: CmdLine@2 + inputs: + script: | + cd $(Build.BinariesDirectory)/Release + find -executable -type f > $(Build.BinariesDirectory)/Release/perms.txt + displayName: 'Find Executable Files' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact' + inputs: + artifactName: 'drop-linux' + targetPath: '$(Build.BinariesDirectory)/Release' + + - template: templates/explicitly-defined-final-tasks.yml + +- job: Linux_Test + workspace: + clean: all + pool: AMD-GPU + dependsOn: + - Linux_Build + timeoutInMinutes: 120 + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Pipeline Artifact' + inputs: + buildType: 'current' + artifactName: 'drop-linux' + targetPath: '$(Build.BinariesDirectory)/Release' + + - checkout: self + clean: true + submodules: recursive + + - template: templates/get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile + Context: tools/ci_build/github/linux/docker + Repository: onnxruntimetrainingmigraphx-cibuild-rocm$(RocmVersion) + - task: CmdLine@2 inputs: script: | @@ -75,12 +143,15 @@ jobs: --device=/dev/dri/renderD$DRIVER_RENDER \ --group-add $(video) \ --group-add $(render) \ - --user onnxruntimedev \ + --user $UID:$(id -g $USER) \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --workdir /build/Release \ onnxruntimetrainingmigraphx-cibuild-rocm$(RocmVersion) \ - /onnxruntime_src/tools/ci_build/github/pai/migraphx_test_launcher.sh + /bin/bash -c " + set -ex; \ + cd /build/Release && xargs -a /build/Release/perms.txt chmod a+x; \ + bash /onnxruntime_src/tools/ci_build/github/pai/migraphx_test_launcher.sh" workingDirectory: $(Build.SourcesDirectory) displayName: 'Run onnxruntime unit tests' diff --git a/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile b/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile index c36e0047b95f..1eb505b8d414 100644 --- a/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile +++ b/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile @@ -37,8 +37,9 @@ RUN cd /migraphx && rbuild package --cxx /opt/rocm/llvm/bin/clang++ -d /migraphx RUN dpkg -i /migraphx/build/*.deb RUN rm -rf /migraphx -ARG BUILD_UID=1001 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER +# ccache +RUN mkdir -p /tmp/ccache && \ + cd /tmp/ccache && \ + wget -q -O - https://github.com/ccache/ccache/releases/download/v4.7.4/ccache-4.7.4-linux-x86_64.tar.xz | tar --strip 1 -J -xf - && \ + cp /tmp/ccache/ccache /usr/bin && \ + rm -rf /tmp/ccache