.gitlab-ci.yml

# MIT License
#
# Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

stages:
  - build
  - test          # Tests if unit tests are passing (CTest)
  - benchmark

variables:
  SUDO_CMD: "" # Must be "sudo" on images which don't use root user
  DEPS_DIR: "$CI_PROJECT_DIR/__dependencies"
  CMAKE_ROCM_MINIMUM_URL: "https://cmake.org/files/v3.10/cmake-3.10.2-Linux-x86_64.tar.gz"
  CMAKE_NVIDIA_MINIMUM_URL: "https://cmake.org/files/v3.15/cmake-3.15.7-Linux-x86_64.tar.gz"
  # General build flags
  CXXFLAGS: ""
  CMAKE_OPTIONS: ""
  # Local build options
  LOCAL_CXXFLAGS: ""
  LOCAL_CMAKE_OPTIONS: ""
  ROCM_LATEST_PATH: "/opt/rocm-4.3.0/"
  ROCPRIM_GIT_BRANCH: "develop_stream"

# hipCUB with rocPRIM backend
.rocm:
  variables:
    SUDO_CMD: "sudo -E"
  tags:
    - rocm
  image: rocm/rocm-terminal:latest
  before_script:
    - $SUDO_CMD rm /etc/apt/sources.list.d/rocm.list
    - $SUDO_CMD apt-get update -qq
    - $SUDO_CMD apt-get install -y -qq wget libidn11 libnuma-dev libunwind-dev git tar xz-utils bzip2 build-essential pkg-config  ninja-build ca-certificates
    - $SUDO_CMD wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | $SUDO_CMD apt-key add -
    - $SUDO_CMD echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' | sudo tee /etc/apt/sources.list.d/rocm.list
    - $SUDO_CMD apt-get update -qq
    - hipconfig
    # cmake
    - mkdir -p $DEPS_DIR/cmake
    - wget --no-check-certificate --quiet -O - $CMAKE_ROCM_MINIMUM_URL | tar --strip-components=1 -xz -C $DEPS_DIR/cmake
    - export PATH=$DEPS_DIR/cmake/bin:$PATH
    # Combine global build options with local options
    - export CXXFLAGS=$CXXFLAGS" "$LOCAL_CXXFLAGS
    - export CMAKE_OPTIONS=$CXXFLAGS" "$LOCAL_CMAKE_OPTIONS
.rocm:build:
  extends: .rocm
  before_script:
    # We need to duplicate before script
    - $SUDO_CMD rm /etc/apt/sources.list.d/rocm.list
    - $SUDO_CMD apt-get update -qq
    - $SUDO_CMD apt-get install -y -qq wget libidn11 libnuma-dev libunwind-dev git tar xz-utils bzip2 build-essential pkg-config  ninja-build ca-certificates
    - $SUDO_CMD wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | $SUDO_CMD apt-key add -
    - $SUDO_CMD echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' | sudo tee /etc/apt/sources.list.d/rocm.list
    - $SUDO_CMD apt-get update -qq
    - hipconfig
    # cmake
    - mkdir -p $DEPS_DIR/cmake
    - wget --no-check-certificate --quiet -O - $CMAKE_ROCM_MINIMUM_URL | tar --strip-components=1 -xz -C $DEPS_DIR/cmake
    - export PATH=$DEPS_DIR/cmake/bin:$PATH
    # Combine global build options with local options
    - export CXXFLAGS=$CXXFLAGS" "$LOCAL_CXXFLAGS
    - export CMAKE_OPTIONS=$CXXFLAGS" "$LOCAL_CMAKE_OPTIONS
    # Install rocPRIM package
    - branch_name="$ROCPRIM_GIT_BRANCH"
    - if [ $CI_COMMIT_BRANCH == develop ] || [ $CI_COMMIT_BRANCH == master ]; then branch_name=$CI_COMMIT_BRANCH;
    - fi;
    - git clone -b $branch_name https://gitlab-ci-token:${CI_JOB_TOKEN}@${ROCPRIM_GIT_URL}
    - cd rocPRIM
    - mkdir build
    - cd build
    - cmake
      -G Ninja
      -D CMAKE_CXX_COMPILER=hipcc
      -D CMAKE_BUILD_TYPE=Release
      -D BUILD_TEST=OFF
      -D BUILD_EXAMPLE=OFF
      -D ROCM_DEP_ROCMCORE=OFF
      -B build
      ../.
    - $SUDO_CMD cpack
      -G "DEB"
    - $SUDO_CMD dpkg -i rocprim*.deb
    - cd ../..

build:rocm:
  extends: .rocm:build
  stage: build
  script:
    - if [ ! -d "build" ] ; then mkdir build;
    - fi;
    - cd build
    - cmake
      -G Ninja
      -D CMAKE_CXX_COMPILER=hipcc
      -D CMAKE_BUILD_TYPE=Release
      -D BUILD_TEST=ON
      -D BUILD_EXAMPLE=ON
      -D AMDGPU_TARGETS="gfx803;gfx900;gfx906"
      -B build
      ../.
    - cmake
      --build ./
    - $SUDO_CMD cpack
      -G "DEB;ZIP"
  artifacts:
    paths:
      - build/test/hipcub/test_*
      - build/test/CTestTestfile.cmake
      - build/test/hipcub/CTestTestfile.cmake
      - build/gtest/
      - build/CMakeCache.txt
      - build/CTestTestfile.cmake
      - build/hipcub*.deb
      - build/hipcub*.zip
    expire_in: 2 weeks

build:rocm-benchmark:
  extends: .rocm:build
  stage: build
  only:
    - add_bf16
    - develop_stream
    - develop
    - master
  script:
    - if [ ! -d "build" ] ; then mkdir build;
    - fi;
    - cd build
    # Build hipCUB benchmark
    - cmake
      -G Ninja
      -D CMAKE_CXX_COMPILER=hipcc
      -D CMAKE_BUILD_TYPE=Release
      -D BUILD_BENCHMARK=ON
      -B build
      ../.
    - cmake
      --build ./
  artifacts:
    paths:
      - build/benchmark/*
      - build/googlebenchmark/
    expire_in: 2 weeks

test:rocm_vega20:
  extends: .rocm
  stage: test
  needs:
    - build:rocm
  tags:
    - vega20
    - rocm
  script:
    - cd build
    - $SUDO_CMD ctest --output-on-failure --repeat-until-fail 2
  allow_failure: true

benchmark:rocm_vega20:
  extends: .rocm
  stage: benchmark
  when: manual
  only:
    - add_bf16
    - develop_stream
    - develop
    - master
  needs:
    - build:rocm-benchmark
  tags:
    - vega20
    - rocm
  script:
    - cmake
      -D BENCHMARK_BINARY_DIR=build/benchmark
      -P ${CI_PROJECT_DIR}/.gitlab/cmake-run-benchmarks.txt
  allow_failure: true

test:rocm_s9300:
  extends: .rocm
  stage: test
  needs:
    - build:rocm
  tags:
    - s9300
    - rocm
  script:
    - cd build
    - $SUDO_CMD ctest --output-on-failure --repeat-until-fail 2
  allow_failure: true

benchmark:rocm_s9300:
  extends: .rocm
  stage: benchmark
  when: manual
  only:
    - add_bf16
    - develop_stream
    - develop
    - master
  needs:
    - build:rocm-benchmark
  tags:
    - s9300
    - rocm
  script:
    - cmake
      -D BENCHMARK_BINARY_DIR=build/benchmark
      -P ${CI_PROJECT_DIR}/.gitlab/cmake-run-benchmarks.txt

test:rocm_mi25:
  extends: .rocm
  stage: test
  needs:
    - build:rocm
  tags:
    - mi25
    - rocm
  script:
    - cd build
    - $SUDO_CMD ctest --output-on-failure --repeat-until-fail 2

benchmark:rocm_mi25:
  extends: .rocm
  stage: benchmark
  when: manual
  only:
    - add_bf16
    - develop_stream
    - develop
    - master
  needs:
    - build:rocm-benchmark
  tags:
    - mi25
    - rocm
  script:
    - cmake
      -D BENCHMARK_BINARY_DIR=build/benchmark
      -P ${CI_PROJECT_DIR}/.gitlab/cmake-run-benchmarks.txt

test:rocm_package:
  extends: .rocm
  stage: test
  tags:
    - rocm-stable
  needs:
    - build:rocm
  script:
    # Install rocPRIM package
    - echo ${ROCPRIM_GIT_URL}
    - echo $ROCPRIM_GIT_URL
    - git clone -b $ROCPRIM_GIT_BRANCH https://gitlab-ci-token:${CI_JOB_TOKEN}@${ROCPRIM_GIT_URL}
    - cd rocPRIM
    - mkdir build
    - cd build
    - CXX=hipcc cmake -DBUILD_TEST=OFF ../.
    - make package
    - $SUDO_CMD dpkg -i rocprim*.deb
    - cd ../..
    # hipCUB
    - cd build
    - $SUDO_CMD dpkg -i hipcub*.deb
    - mkdir ../package_test && cd ../package_test
    - CXX=hipcc cmake ../test/extra/. -Drocprim_DIR="/opt/rocm/rocprim"
    - make VERBOSE=1
    - $SUDO_CMD ctest --output-on-failure --repeat-until-fail 2
    - $SUDO_CMD dpkg -r hipcub
    - $SUDO_CMD dpkg -r rocprim
  allow_failure: true

test:rocm_install:
  extends: .rocm
  stage: test
  tags:
    - rocm-stable
  needs: []
  script:
    # Install rocPRIM package
    - echo ${ROCPRIM_GIT_URL}
    - echo $ROCPRIM_GIT_URL
    - git clone -b $ROCPRIM_GIT_BRANCH https://gitlab-ci-token:${CI_JOB_TOKEN}@${ROCPRIM_GIT_URL}
    - cd rocPRIM
    - mkdir build
    - cd build
    - CXX=hipcc cmake -DBUILD_TEST=OFF ../.
    - make package
    - $SUDO_CMD dpkg -i rocprim*.deb
    - cd ../..
    # hipCUB
    - mkdir build_only_install
    - cd build_only_install
    - CXX=hipcc cmake -DBUILD_TEST=OFF ../.
    - $SUDO_CMD make install
    - mkdir ../install_test && cd ../install_test
    - CXX=hipcc cmake ../test/extra/.
    - make VERBOSE=1
    - $SUDO_CMD ctest --output-on-failure --repeat-until-fail 2
  allow_failure: true

# hipCUB with CUB backend

.nvcc:
  tags:
    - nvcc
  image: nvidia/cuda:11.3.1-devel-ubuntu18.04
  before_script:
    - nvidia-smi
    # HIP-nvcc
    - $SUDO_CMD apt-get update -qq
    - $SUDO_CMD apt-get install -y -qq libnuma-dev libunwind-dev git wget tar xz-utils bzip2 build-essential pkg-config ninja-build protobuf-compiler gcc-8 g++-8 libidn11 ca-certificates
    - $SUDO_CMD wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | $SUDO_CMD apt-key add -
    - $SUDO_CMD echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' | $SUDO_CMD tee /etc/apt/sources.list.d/rocm.list
    - $SUDO_CMD apt-get update -qq
    - $SUDO_CMD apt-get install -y hip-base
    # Install hip-nvcc ignoring dependencies because it depends on cuda metapackage
    # (with heavy libraries, tools etc. that also require GUI and other packages)
    - apt-get download hip-nvcc rocm-cmake
    - $SUDO_CMD dpkg -i --ignore-depends=cuda hip*.deb
    - $SUDO_CMD ls -d /opt/*
    - $SUDO_CMD ln -s $ROCM_LATEST_PATH /opt/rocm
    - export PATH=$PATH:/opt/rocm/bin
    - hipconfig
    # cmake
    - mkdir -p $DEPS_DIR/cmake
    - wget --no-check-certificate --quiet -O - $CMAKE_NVIDIA_MINIMUM_URL | tar --strip-components=1 -xz -C $DEPS_DIR/cmake
    - export PATH=$DEPS_DIR/cmake/bin:$PATH
    # Combine global build options with local options
    - export CXXFLAGS=$CXXFLAGS" "$LOCAL_CXXFLAGS
    - export CMAKE_OPTIONS=$CXXFLAGS" "$LOCAL_CMAKE_OPTIONS

build:nvcc:
  extends: .nvcc
  stage: build
  script:
    - mkdir build
    - cmake
      -G Ninja
      -D CMAKE_BUILD_TYPE=Release
      -D BUILD_TEST=ON
      -D BUILD_EXAMPLE=ON
      -B build
      .
    - cmake
      --build build
    - cd build
    - $SUDO_CMD cpack
      -G "DEB;ZIP"
  artifacts:
    paths:
      - build/test/hipcub/test_*
      - build/test/CTestTestfile.cmake
      - build/test/hipcub/CTestTestfile.cmake
      - build/gtest/
      - build/CMakeCache.txt
      - build/CTestTestfile.cmake
      - build/hipcub*.deb
      - build/hipcub*.zip
    expire_in: 2 weeks

build:nvcc-benchmark:
  extends: .nvcc
  stage: build
  only:
    - add_bf16
    - develop_stream
    - develop
    - master
  script:
    - mkdir build
    - cmake
      -G Ninja
      -D CMAKE_BUILD_TYPE=Release
      -D BUILD_BENCHMARK=ON
      -D CMAKE_CXX_COMPILER=g++-8
      -D CMAKE_C_COMPILER=g++-8
      -B build
      .
    - cmake
      --build build
  artifacts:
    paths:
      - build/benchmark/*
      - build/googlebenchmark/
    expire_in: 2 weeks

test:nvcc_titanv:
  extends: .nvcc
  stage: test
  needs:
    - build:nvcc
  script:
    - cd build
    - CUDA_VISIBLE_DEVICES=0 ctest --output-on-failure --repeat-until-fail 2

benchmark:nvcc_titanv:
  extends: .nvcc
  stage: benchmark
  when: manual
  only:
    - add_bf16
    - develop_stream
    - develop
    - master
  needs:
    - build:nvcc-benchmark
  script:
    - CUDA_VISIBLE_DEVICES=0
    - cmake
      -D BENCHMARK_BINARY_DIR=build/benchmark
      -P ${CI_PROJECT_DIR}/.gitlab/cmake-run-benchmarks.txt
  allow_failure: true

test:nvcc_980:
  extends: .nvcc
  stage: test
  needs:
    - build:nvcc
  script:
    - cd build
    - CUDA_VISIBLE_DEVICES=1 ctest --output-on-failure --repeat-until-fail 2

benchmark:nvcc_980:
  extends: .nvcc
  stage: benchmark
  when: manual
  only:
    - fix_benchmarks
    - develop_stream
    - develop
    - master
  needs:
    - build:nvcc-benchmark
  script:
    - CUDA_VISIBLE_DEVICES=1
    - cmake
      -D BENCHMARK_BINARY_DIR=build/benchmark
      -P ${CI_PROJECT_DIR}/.gitlab/cmake-run-benchmarks.txt
  allow_failure: true

test:nvcc_package:
  extends: .nvcc
  stage: test
  needs:
    - build:nvcc
  script:
    - cd build
    - $SUDO_CMD dpkg -i hipcub_nvcc*.deb
    - mkdir ../package_test && cd ../package_test
    - cmake ../test/extra/.
    - make VERBOSE=1
    - ctest --output-on-failure --repeat-until-fail 2
    - $SUDO_CMD dpkg -r hipcub_nvcc
  allow_failure: true

test:nvcc_install:
  extends: .nvcc
  stage: test
  needs: []
  script:
    - mkdir build_only_install
    - cd build_only_install
    - cmake -DBUILD_TEST=OFF ../.
    - $SUDO_CMD make install
    - mkdir ../install_test && cd ../install_test
    - cmake ../test/extra/.
    - make VERBOSE=1
    - ctest --output-on-failure --repeat-until-fail 2

test:doc:
  variables:
    SUDO_CMD: "sudo -E"
  tags:
    - rocm # Do it on rocm machine
  image: rocm/rocm-terminal:latest
  stage: test
  needs: []
  before_script:
    - $SUDO_CMD rm /etc/apt/sources.list.d/rocm.list
    - $SUDO_CMD apt-get update -qq
    # Do we need all of this?
    - $SUDO_CMD apt-get install -y -qq wget git tar xz-utils bzip2 texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended latexmk python-pip python3-sphinx doxygen
    - $SUDO_CMD -H pip install sphinx_rtd_theme breathe==4.12.0 exhale
    - $SUDO_CMD wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | $SUDO_CMD apt-key add -
    - $SUDO_CMD echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' | sudo tee /etc/apt/sources.list.d/rocm.list
    - $SUDO_CMD apt-get update -qq
  script:
    - bash -x ./docs/run_doc.sh