build.yml

name: "build"
on:
  pull_request:
    paths:
      - "include/**"
      - "inference/**"
      - "cmake/**"
      - "config/**"
      - "python/**"
      - "src/**"
      - ".github/workflows/helpers/install_dependencies.sh"
      - ".github/workflows/build.yml"
  push:
    branches:
      - "master"
    paths:
      - "include/**"
      - "inference/**"
      - "cmake/**"
      - "config/**"
      - "python/**"
      - "src/**"
      - ".github/workflows/helpers/install_dependencies.sh"
      - ".github/workflows/build.yml"
  workflow_dispatch:
concurrency:
  group: build-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  cmake-build:
    name: Build FlexFlow with CMake
    runs-on: ubuntu-20.04
    defaults:
      run:
        shell: bash -l {0} # required to use an activated conda environment
    strategy:
      matrix:
        gpu_backend: ["cuda", "hip_rocm"]
      fail-fast: false
    steps:
      - name: Checkout Git Repository
        uses: actions/checkout@v3
        with:
          submodules: recursive

      - name: Free additional space on runner
        run: .github/workflows/helpers/free_space_on_runner.sh

      - name: Install CUDA
        uses: Jimver/cuda-toolkit@v0.2.8
        id: cuda-toolkit
        with:
          cuda: "11.1.1"
          # Disable caching of the CUDA binaries, since it does not give us any significant performance improvement
          use-github-cache: "false"

      - name: Install system dependencies
        run: FF_GPU_BACKEND=${{ matrix.gpu_backend }} .github/workflows/helpers/install_dependencies.sh

      - name: Install conda and FlexFlow dependencies
        uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: flexflow
          environment-file: conda/environment.yml
          auto-activate-base: false

      - name: Build FlexFlow
        run: |
          export CUDNN_DIR=/usr/local/cuda
          export CUDA_DIR=/usr/local/cuda
          export FF_HOME=$(pwd)
          export FF_GPU_BACKEND=${{ matrix.gpu_backend }}
          export FF_CUDA_ARCH=70
          cores_available=$(nproc --all)
          n_build_cores=$(( cores_available -1 ))
          if (( $n_build_cores < 1 )) ; then n_build_cores=1 ; fi
          mkdir build
          cd build
          if [[ "${FF_GPU_BACKEND}" == "cuda" ]]; then
            export FF_BUILD_ALL_EXAMPLES=ON
            export FF_BUILD_ALL_INFERENCE_EXAMPLES=ON
            export FF_BUILD_UNIT_TESTS=ON
          else 
            export FF_BUILD_ALL_EXAMPLES=OFF
            export FF_BUILD_ALL_INFERENCE_EXAMPLES=OFF
            export FF_BUILD_UNIT_TESTS=OFF
          fi
          ../config/config.linux
          make -j $n_build_cores

      - name: Install FlexFlow
        run: |
          export CUDNN_DIR=/usr/local/cuda
          export CUDA_DIR=/usr/local/cuda
          export FF_HOME=$(pwd)
          export FF_GPU_BACKEND=${{ matrix.gpu_backend }}
          export FF_CUDA_ARCH=70
          cd build
          if [[ "${FF_GPU_BACKEND}" == "cuda" ]]; then
            export FF_BUILD_ALL_EXAMPLES=ON
            export FF_BUILD_ALL_INFERENCE_EXAMPLES=ON
            export FF_BUILD_UNIT_TESTS=ON
          else 
            export FF_BUILD_ALL_EXAMPLES=OFF
            export FF_BUILD_ALL_INFERENCE_EXAMPLES=OFF
            export FF_BUILD_UNIT_TESTS=OFF
          fi
          ../config/config.linux
          sudo make install
          sudo ldconfig

      - name: Check availability of Python flexflow.core module
        if: ${{ matrix.gpu_backend == 'cuda' }}
        run: |
          python -c "import flexflow.core; exit()"

      - name: Run C++ unit tests
        if: ${{ matrix.gpu_backend == 'cuda' }}
        run: |
          export CUDNN_DIR=/usr/local/cuda
          export CUDA_DIR=/usr/local/cuda
          export FF_HOME=$(pwd)
          export LD_LIBRARY_PATH=$CUDA_DIR/lib64/stubs:$LD_LIBRARY_PATH
          sudo ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
          cd build
          ./tests/unit/unit-test

  makefile-build:
    name: Build FlexFlow with the Makefile
    runs-on: ubuntu-20.04
    defaults:
      run:
        shell: bash -l {0} # required to use an activated conda environment
    steps:
      - name: Checkout Git Repository
        uses: actions/checkout@v3
        with:
          submodules: recursive

      - name: Free additional space on runner
        run: .github/workflows/helpers/free_space_on_runner.sh

      - name: Install CUDA
        uses: Jimver/cuda-toolkit@v0.2.8
        id: cuda-toolkit
        with:
          cuda: "11.1.1"
          use-github-cache: "false"

      - name: Install system dependencies
        run: .github/workflows/helpers/install_dependencies.sh

      - name: Install conda and FlexFlow dependencies
        uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: flexflow
          environment-file: conda/environment.yml
          auto-activate-base: false

      - name: Build FlexFlow
        run: |
          export CUDNN_DIR=/usr/local/cuda
          export CUDA_DIR=/usr/local/cuda
          export FF_HOME=$(pwd)
          cores_available=$(nproc --all)
          n_build_cores=$(( cores_available -1 ))
          if (( $n_build_cores < 1 )) ; then n_build_cores=1 ; fi

          cd python
          make -j $n_build_cores
          python -c 'import flexflow.core'