feat: support cann

Signed-off-by: thxCode <thxcode0824@gmail.com>
gpustack · Jul 20, 2024 · d1964a7 · d1964a7
1 parent 1ab2df1
commit d1964a7
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 0 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -396,6 +396,94 @@ jobs:
           path: ${{ github.workspace }}/out/*.zip
           name: llama-box-linux-${{ matrix.arch }}-oneapi-${{ matrix.version }}
 
+  linux-cann:
+    strategy:
+      # cache-able building with ccache.
+      fail-fast: false
+      matrix:
+        # see https://hub.docker.com/r/ascendai/cann/tags?page=&page_size=&ordering=&name=.
+        #     8.0 ==> 8.0.RC1
+        arch: [ amd64, arm64 ]
+        version: [ '8.0' ]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Maximize Space
+        # see https://github.com/easimon/maximize-build-space/blob/master/action.yml.
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+      - name: Clone
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+      - name: Setup Cache
+        timeout-minutes: 5
+        uses: actions/cache@v3
+        with:
+          key: cache-linux-cann-${{ matrix.arch }}-${{ matrix.version }}
+          path: |
+            ${{ github.workspace }}/.cache
+      - name: Setup QEMU
+        if: ${{ matrix.arch == 'arm64' }}
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:qemu-v7.0.0
+          platforms: "arm64"
+      - name: Build
+        # disable OpenMP,
+        # see https://github.com/ggerganov/llama.cpp/issues/7743#issuecomment-2148342691,
+        #     https://github.com/ggerganov/llama.cpp/issues/7719#issuecomment-2147631216.
+        env:
+          CCACHE_DIR: "${{ github.workspace }}/.cache/ccache"
+        run: |
+          echo "===== SCRIPT ====="
+          cat <<EOF > /tmp/entrypoint.sh
+          #!/bin/bash
+          source /usr/local/Ascend/ascend-toolkit/set_env.sh
+          apt-get update && apt-get install -y build-essential git cmake ccache
+          git config --system --add safe.directory '*'
+          mkdir -p ${{ github.workspace }}/.cache
+          cmake -S ${{ github.workspace }} -B ${{ github.workspace }}/build -DCMAKE_BUILD_TYPE=Release \
+            -DGGML_CANN=on \
+            ${{ matrix.arch == 'amd64' && '-DGGML_NATIVE=off' || '-DGGML_NATIVE=on' }} \
+            -DGGML_OPENMP=off
+          cmake --build ${{ github.workspace }}/build --target llama-box --config Release -- -j $(nproc)
+          EOF
+          chmod +x /tmp/entrypoint.sh
+          cat /tmp/entrypoint.sh
+
+          echo "===== BUILD ====="
+          docker run \
+            --rm \
+            --privileged \
+            --platform linux/${{ matrix.arch }} \
+            --volume ${{ github.workspace }}:${{ github.workspace }} \
+            --workdir ${{ github.workspace }} \
+            --env CCACHE_DIR \
+            --volume /tmp/entrypoint.sh:/entrypoint.sh \
+            --entrypoint /entrypoint.sh \
+            ascendai/cann:${{ matrix.version }}
+
+          echo "===== RESULT ====="
+          if [ -f ${{ github.workspace }}/build/bin/llama-box ]; then
+            ldd ${{ github.workspace }}/build/bin/llama-box
+          else
+            exit 1
+          fi
+
+          echo "===== PACKAGE ====="
+          mkdir -p ${{ github.workspace }}/out
+          zip -j ${{ github.workspace }}/out/llama-box-linux-${{ matrix.arch }}-cann-${{ matrix.version }}.zip ${{ github.workspace }}/build/bin/*
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          path: ${{ github.workspace }}/out/*.zip
+          name: llama-box-linux-${{ matrix.arch }}-cann-${{ matrix.version }}
+
   windows-hip:
     continue-on-error: ${{ !startsWith(github.ref, 'refs/tags/') }}
     strategy:
@@ -653,6 +741,7 @@ jobs:
       - linux-hip
       - linux-cuda
       - linux-oneapi
+      - linux-cann
       - windows-hip
       - windows-cuda
       - windows-oneapi

diff --git a/llama-box/CMakeLists.txt b/llama-box/CMakeLists.txt
@@ -62,6 +62,18 @@ add_dependencies(patch version)
 #
 set(TARGET llama-box)
 include_directories(${CMAKE_SOURCE_DIR})
+if (GGML_CANN)
+    if ("cann${CANN_INSTALL_DIR}" STREQUAL "cann" AND DEFINED ENV{ASCEND_TOOLKIT_HOME})
+        set(CANN_INSTALL_DIR $ENV{ASCEND_TOOLKIT_HOME})
+        message(STATUS "CANN: updated CANN_INSTALL_DIR from ASCEND_TOOLKIT_HOME=$ENV{ASCEND_TOOLKIT_HOME}")
+    endif ()
+    if (CANN_INSTALL_DIR)
+        message(STATUS "CANN: updated link directoreis to ${CANN_INSTALL_DIR}/lib64")
+        link_directories(
+                ${CANN_INSTALL_DIR}/lib64
+                ${CANN_INSTALL_DIR}/runtime/lib64/stub)
+    endif ()
+endif ()
 add_executable(${TARGET} main.cpp param.hpp ratelimiter.hpp utils.hpp)
 add_dependencies(${TARGET} patch)
 target_link_libraries(${TARGET} PRIVATE version common llava ${CMAKE_THREAD_LIBS_INIT})