From 0481293488bf31bad89ebd02e868bd7315bce634 Mon Sep 17 00:00:00 2001 From: one-lithe-rune Date: Thu, 18 Jan 2024 23:43:30 +0000 Subject: [PATCH 1/3] Update cmake-rocm-windows workflow for HIP sdk 5.7.1 * Bump the rocm/hip version for the windows build to 5.7.1 --- .github/workflows/cmake-rocm-windows.yml | 68 ++++++++++++------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/cmake-rocm-windows.yml b/.github/workflows/cmake-rocm-windows.yml index bc603d726c967..caef454dbfc40 100644 --- a/.github/workflows/cmake-rocm-windows.yml +++ b/.github/workflows/cmake-rocm-windows.yml @@ -31,9 +31,9 @@ jobs: - uses: actions/checkout@v3 with: repository: 'YellowRoseCx/koboldcpp-rocm' - ref: main + ref: main submodules: 'recursive' - + - name: Setup MSYS2 uses: msys2/setup-msys2@v2 with: @@ -41,62 +41,62 @@ jobs: install: >- base-devel mingw-w64-x86_64-toolchain - + - name: Build CLBLAST & Others with make run: | - Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' + Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' Get-Location ls make CC=gcc LLAMA_CLBLAST=1 LLAMA_OPENBLAS=1 -j2 - + - name: Install ROCm SDK run: | - curl -LO https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-23.Q3-Win10-Win11-For-HIP.exe - Start-Process 'AMD-Software-PRO-Edition-23.Q3-Win10-Win11-For-HIP.exe' -ArgumentList '-install' -NoNewWindow -Wait - echo "C:\Program Files\AMD\ROCm\5.5\bin" >> $env:GITHUB_PATH - echo 'ROCM_PATH=C:\Program Files\AMD\ROCm\5.5' >> $env:GITHUB_ENV - echo 'HIP_PATH=C:\Program Files\AMD\ROCm\5.5' >> $env:GITHUB_ENV - echo "ROCM_VERSION=5.5.1" >> $env:GITHUB_ENV - + curl -LO https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-23.Q4-Win10-Win11-For-HIP.exe + Start-Process 'AMD-Software-PRO-Edition-23.Q4-Win10-Win11-For-HIP.exe' -ArgumentList '-install' -NoNewWindow -Wait + echo "C:\Program Files\AMD\ROCm\5.7\bin" >> $env:GITHUB_PATH + echo 'ROCM_PATH=C:\Program Files\AMD\ROCm\5.7' >> $env:GITHUB_ENV + echo 'HIP_PATH=C:\Program Files\AMD\ROCm\5.7' >> $env:GITHUB_ENV + echo "ROCM_VERSION=5.7.1" >> $env:GITHUB_ENV + - uses: actions/setup-python@v3 with: python-version: "3.10" - + - name: Install Dependencies run: | python -m pip install cmake ninja - + - name: Install PyInstaller run: pip install pyinstaller psutil - + - name: Build Lib run: | - $env:CC = 'C:\Program Files\AMD\ROCm\5.5\bin\clang.exe' - $env:CXX = 'C:\Program Files\AMD\ROCm\5.5\bin\clang++.exe' - $env:CMAKE_PREFIX_PATH = 'C:\Program Files\AMD\ROCm\5.5' + $env:CC = 'C:\Program Files\AMD\ROCm\5.7\bin\clang.exe' + $env:CXX = 'C:\Program Files\AMD\ROCm\5.7\bin\clang++.exe' + $env:CMAKE_PREFIX_PATH = 'C:\Program Files\AMD\ROCm\5.7' $env:VERBOSE = '1' Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' mkdir 'build' Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm\build' Get-Location - cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DLLAMA_HIPBLAS=ON -DCMAKE_C_COMPILER="C:/Program Files/AMD/ROCm/5.5/bin/clang.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/AMD/ROCm/5.5/bin/clang++.exe" -DAMDGPU_TARGETS="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1010;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102" + cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DLLAMA_HIPBLAS=ON -DCMAKE_C_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang++.exe" -DAMDGPU_TARGETS="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1010;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102" cmake --build . -j2 - name: Copy ROCm DLLs and Run PyInstaller run: | Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' Get-Location - copy "C:\Program Files\AMD\ROCm\5.5\bin\hipblas.dll" .\ - copy "C:\Program Files\AMD\ROCm\5.5\bin\rocblas.dll" .\ - robocopy "C:\Program Files\AMD\ROCm\5.5\bin\rocblas" .\rocblas /MIR + copy "C:\Program Files\AMD\ROCm\5.7\bin\hipblas.dll" .\ + copy "C:\Program Files\AMD\ROCm\5.7\bin\rocblas.dll" .\ + robocopy "C:\Program Files\AMD\ROCm\5.7\bin\rocblas" .\rocblas /MIR pip install customtkinter PyInstaller --noconfirm --onefile --collect-all customtkinter --clean --console --icon ".\niko.ico" --add-data "./klite.embd;." --add-data "./winclinfo.exe;." --add-data "./OpenCL.dll;." --add-data "./kcpp_docs.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_openblas.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./libopenblas.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./koboldcpp_clblast_noavx2.dll;." --add-data "./clblast.dll;." --add-data "D:\a\koboldcpp-rocm\koboldcpp-rocm\build\bin\koboldcpp_hipblas.dll;." --add-data "./hipblas.dll;." --add-data "./rocblas.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." --add-data "./rocblas;." --add-data "C:/Windows/System32/msvcp140.dll;." --add-data "C:/Windows/System32/vcruntime140_1.dll;." "./koboldcpp.py" -n "koboldcpp_rocm.exe" # curl -LO https://github.com/YellowRoseCx/koboldcpp-rocm/releases/download/v1.43.2-ROCm/gfx103132rocblasfiles.7z -# 7z x gfx103132rocblasfiles.7z +# 7z x gfx103132rocblasfiles.7z - name: Create ZIP archive run: | - Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' + Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' Compress-Archive -Path D:\a\koboldcpp-rocm\koboldcpp-rocm\klite.embd, D:\a\koboldcpp-rocm\koboldcpp-rocm\winclinfo.exe, D:\a\koboldcpp-rocm\koboldcpp-rocm\OpenCL.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\kcpp_docs.embd, D:\a\koboldcpp-rocm\koboldcpp-rocm\koboldcpp_default.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\koboldcpp_openblas.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\koboldcpp_failsafe.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\koboldcpp_noavx2.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\libopenblas.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\koboldcpp_clblast.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\koboldcpp_clblast_noavx2.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\clblast.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\build\bin\koboldcpp_hipblas.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\hipblas.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\rocblas.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\rwkv_vocab.embd, D:\a\koboldcpp-rocm\koboldcpp-rocm\rwkv_world_vocab.embd, D:\a\koboldcpp-rocm\koboldcpp-rocm\rocblas, D:\a\koboldcpp-rocm\koboldcpp-rocm\msvcp140.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\vcruntime140_1.dll, D:\a\koboldcpp-rocm\koboldcpp-rocm\koboldcpp.py -DestinationPath koboldcpp_rocm_files.zip copy "koboldcpp_rocm_files.zip" D:\a\koboldcpp-rocm\koboldcpp-rocm\dist @@ -105,12 +105,12 @@ jobs: with: name: koboldcpp_hipblas-${{ inputs.version }} path: D:\a\koboldcpp-rocm\koboldcpp-rocm\build\bin\koboldcpp_hipblas.dll - + - uses: actions/upload-artifact@v3 with: name: koboldcpp_rocm_only-${{ inputs.version }} path: D:\a\koboldcpp-rocm\koboldcpp-rocm\dist\koboldcpp_rocm.exe - + - uses: actions/upload-artifact@v3 with: name: koboldcpp_rocm_zip-${{ inputs.version }} @@ -130,10 +130,10 @@ jobs: - + # Copy-Item 'D:\a\koboldcpp-rocm\koboldcpp-rocm\build\bin\koboldcpp_hipblas.dll' '..\..\build' # build_wheel: -# name: Build ROCm +# name: Build ROCm # runs-on: windows-latest # needs: build_libs # strategy: @@ -150,20 +150,20 @@ jobs: # with: # repository: 'YellowRoseCx/koboldcpp-rocm' # ref: ${{ inputs.version }} - + # - uses: actions/download-artifact@v3 # with: # name: 'win-rocm-lib' # path: ./koboldcpp-rocm - + # - uses: actions/setup-python@v3 # with: # python-version: ${{ matrix.pyver }} - + # - name: Install Dependencies # run: | # python -m pip install build wheel cmake scikit-build ninja - + # - name: Build Wheel # run: | # $packageVersion = [version]$env:PCKGVER.TrimStart('v') @@ -172,7 +172,7 @@ jobs: # if ($packageVersion -gt [version]'0.1.77') {$newsetup = $setup.Replace('package_data={"llama_cpp": ["py.typed"]},','package_data={"llama_cpp": ["py.typed", "llama.dll"]},')} # New-Item 'setup.py' -itemType File -value $newsetup -force # python setup.py --skip-cmake bdist_wheel egg_info --tag-build=+rocm5.5.1 - + # - name: Upload files to a GitHub release # id: upload-release # uses: svenstaro/upload-release-action@2.6.1 @@ -183,7 +183,7 @@ jobs: # file_glob: true # make_latest: false # overwrite: true - + # - uses: actions/upload-artifact@v3 # if: steps.upload-release.outcome == 'failure' # with: From 13868bf674edfe6ecbc9147bddd887fc65f67b8c Mon Sep 17 00:00:00 2001 From: one-lithe-rune Date: Fri, 19 Jan 2024 09:27:58 +0000 Subject: [PATCH 2/3] Update cmake-rocm-windows to set HIP_PLATFORM * Explicitly set HIP_PLATFORM=amd in the cmake defines, to avoid the code path in the Hip SDK 5.7.1 where it tries to use the broken hipconfig.bat supplied with the SDK. --- .github/workflows/cmake-rocm-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake-rocm-windows.yml b/.github/workflows/cmake-rocm-windows.yml index caef454dbfc40..a98ca13f4b35a 100644 --- a/.github/workflows/cmake-rocm-windows.yml +++ b/.github/workflows/cmake-rocm-windows.yml @@ -79,7 +79,7 @@ jobs: mkdir 'build' Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm\build' Get-Location - cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DLLAMA_HIPBLAS=ON -DCMAKE_C_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang++.exe" -DAMDGPU_TARGETS="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1010;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102" + cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DLLAMA_HIPBLAS=ON -DHIP_PLATFORM=amd -DCMAKE_C_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang++.exe" -DAMDGPU_TARGETS="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1010;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102" cmake --build . -j2 - name: Copy ROCm DLLs and Run PyInstaller From dc7ca93f11be126fa9acd3611bb1aa690e3d7d03 Mon Sep 17 00:00:00 2001 From: one-lithe-rune Date: Fri, 19 Jan 2024 09:36:12 +0000 Subject: [PATCH 3/3] Update tests-rocm-windows & readme for HIP SDK 5.7.1 * Update README with cmake commandline on windows for HIP SDK 5.7.1 * Update Gibhub action tests-rocm-windows.yml for HIP SDK 5.7.1 --- .github/workflows/tests-rocm-windows.yml | 50 ++++++++++++------------ README.md | 37 +++++++++--------- 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/.github/workflows/tests-rocm-windows.yml b/.github/workflows/tests-rocm-windows.yml index cbfeba43b7017..9ab59e32b8beb 100644 --- a/.github/workflows/tests-rocm-windows.yml +++ b/.github/workflows/tests-rocm-windows.yml @@ -33,7 +33,7 @@ jobs: repository: 'YellowRoseCx/koboldcpp-rocm' ref: main submodules: 'recursive' - + - name: Setup MSYS2 uses: msys2/setup-msys2@v2 with: @@ -41,54 +41,54 @@ jobs: install: >- base-devel mingw-w64-x86_64-toolchain - + - name: Build CLBLAST & Others with make run: | - Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' + Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' Get-Location ls make CC=gcc LLAMA_CLBLAST=1 LLAMA_OPENBLAS=1 -j2 - + - name: Install ROCm SDK run: | - curl -LO https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-23.Q3-Win10-Win11-For-HIP.exe - Start-Process 'AMD-Software-PRO-Edition-23.Q3-Win10-Win11-For-HIP.exe' -ArgumentList '-install' -NoNewWindow -Wait - echo "C:\Program Files\AMD\ROCm\5.5\bin" >> $env:GITHUB_PATH - echo 'ROCM_PATH=C:\Program Files\AMD\ROCm\5.5' >> $env:GITHUB_ENV - echo 'HIP_PATH=C:\Program Files\AMD\ROCm\5.5' >> $env:GITHUB_ENV - echo "ROCM_VERSION=5.5.1" >> $env:GITHUB_ENV - + curl -LO https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-23.Q4-Win10-Win11-For-HIP.exe + Start-Process 'AMD-Software-PRO-Edition-23.Q4-Win10-Win11-For-HIP.exe' -ArgumentList '-install' -NoNewWindow -Wait + echo "C:\Program Files\AMD\ROCm\5.7\bin" >> $env:GITHUB_PATH + echo 'ROCM_PATH=C:\Program Files\AMD\ROCm\5.7' >> $env:GITHUB_ENV + echo 'HIP_PATH=C:\Program Files\AMD\ROCm\5.7' >> $env:GITHUB_ENV + echo "ROCM_VERSION=5.7.1" >> $env:GITHUB_ENV + - uses: actions/setup-python@v3 with: python-version: "3.10" - + - name: Install Dependencies run: | python -m pip install cmake ninja - + - name: Install PyInstaller run: pip install pyinstaller psutil - + - name: Test Build Lib run: | - $env:CC = 'C:\Program Files\AMD\ROCm\5.5\bin\clang.exe' - $env:CXX = 'C:\Program Files\AMD\ROCm\5.5\bin\clang++.exe' - $env:CMAKE_PREFIX_PATH = 'C:\Program Files\AMD\ROCm\5.5' + $env:CC = 'C:\Program Files\AMD\ROCm\5.7\bin\clang.exe' + $env:CXX = 'C:\Program Files\AMD\ROCm\5.7\bin\clang++.exe' + $env:CMAKE_PREFIX_PATH = 'C:\Program Files\AMD\ROCm\5.7' $env:VERBOSE = '1' Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' mkdir 'build' Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm\build' Get-Location - cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DLLAMA_HIPBLAS=ON -DCMAKE_C_COMPILER="C:/Program Files/AMD/ROCm/5.5/bin/clang.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/AMD/ROCm/5.5/bin/clang++.exe" -DAMDGPU_TARGETS="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1010;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102" - cmake --build . -j2 + cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DLLAMA_HIPBLAS=ON -DHIP_PLATFORM=amd -DCMAKE_C_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang++.exe" -DAMDGPU_TARGETS="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1010;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102" + cmake --build . -j2 - name: Copy ROCm DLLs and Run PyInstaller run: | Set-Location 'D:\a\koboldcpp-rocm\koboldcpp-rocm' Get-Location - copy "C:\Program Files\AMD\ROCm\5.5\bin\hipblas.dll" .\ - copy "C:\Program Files\AMD\ROCm\5.5\bin\rocblas.dll" .\ - robocopy "C:\Program Files\AMD\ROCm\5.5\bin\rocblas" .\rocblas /MIR + copy "C:\Program Files\AMD\ROCm\5.7\bin\hipblas.dll" .\ + copy "C:\Program Files\AMD\ROCm\5.7\bin\rocblas.dll" .\ + robocopy "C:\Program Files\AMD\ROCm\5.7\bin\rocblas" .\rocblas /MIR pip install customtkinter PyInstaller --noconfirm \ --onefile \ @@ -119,13 +119,13 @@ jobs: "./koboldcpp.py" \ -n "koboldcpp_rocm.exe" # curl -LO https://github.com/YellowRoseCx/koboldcpp-rocm/releases/download/v1.43.2-ROCm/gfx103132rocblasfiles.7z -# 7z x gfx103132rocblasfiles.7z +# 7z x gfx103132rocblasfiles.7z - uses: actions/upload-artifact@v3 with: name: koboldcpp_hipblas-${{ inputs.version }} path: D:\a\koboldcpp-rocm\koboldcpp-rocm\build\bin\koboldcpp_hipblas.dll - + - uses: actions/upload-artifact@v3 with: name: koboldcpp_rocm-${{ inputs.version }} @@ -142,5 +142,5 @@ jobs: # file_glob: false # make_latest: true # overwrite: false - + # Copy-Item 'D:\a\koboldcpp-rocm\koboldcpp-rocm\build\bin\koboldcpp_hipblas.dll' '..\..\build' diff --git a/README.md b/README.md index 6eee3894d6d59..e4ecdc8508020 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ -##### Original ([llama.cpp rocm port](https://github.com/ggerganov/llama.cpp/pull/1087), [llama.cpp commit](https://github.com/ggerganov/llama.cpp/commit/6bbc598a632560cb45dd2c51ad403bda8723b629)) by SlyEcho, YellowRoseCx, ardfork, funnbot, Engininja2, Kerfuffle, jammm, and jdecourval. +##### Original ([llama.cpp rocm port](https://github.com/ggerganov/llama.cpp/pull/1087), [llama.cpp commit](https://github.com/ggerganov/llama.cpp/commit/6bbc598a632560cb45dd2c51ad403bda8723b629)) by SlyEcho, YellowRoseCx, ardfork, funnbot, Engininja2, Kerfuffle, jammm, and jdecourval. ##### Further modified and ported to KoboldCpp by YellowRoseCx. # koboldcpp-ROCM for AMD -### Quick Linux install: +### Quick Linux install: To install, either use the file "[easy_KCPP-ROCm_install.sh](https://github.com/YellowRoseCx/koboldcpp-rocm/blob/main/easy_KCPP-ROCm_install.sh)" or navigate to the folder you want to download to in Terminal then run -``` +``` git clone https://github.com/YellowRoseCx/koboldcpp-rocm.git -b main --depth 1 && \ cd koboldcpp-rocm && \ make LLAMA_HIPBLAS=1 -j4 && \ ./koboldcpp.py ``` -When the KoboldCPP GUI appears, make sure to select "Use hipBLAS (ROCm)" and set GPU layers +When the KoboldCPP GUI appears, make sure to select "Use hipBLAS (ROCm)" and set GPU layers -------- ### Quick Summary @@ -45,27 +45,27 @@ My typical start command looks like this: ``python koboldcpp.py --threads 6 --bl - You're encouraged to use the .exe released, but if you want to compile your binaries from source at Windows, the easiest way is: - Use the latest release of w64devkit (https://github.com/skeeto/w64devkit). Be sure to use the "vanilla one", not i686 or other different stuff. If you try they will conflit with the precompiled libs! - Make sure you are using the w64devkit integrated terminal, (powershell should work for the cmake hipblas part) - - *This site may be useful, it has some patches for Windows ROCm to help it with compilation that I used, but I'm not sure if it's necessary.* https://streamhpc.com/blog/2023-08-01/how-to-get-full-cmake-support-for-amd-hip-sdk-on-windows-including-patches/ + - *This site may be useful, it has some patches for Windows ROCm to help it with compilation that I used, but I'm not sure if it's necessary.* https://streamhpc.com/blog/2023-08-01/how-to-get-full-cmake-support-for-amd-hip-sdk-on-windows-including-patches/ - (ROCm Required): https://rocm.docs.amd.com/en/latest/deploy/windows/quick_start.html - - Build command used: - ``cd koboldcpp-rocm`` + + Build command used: + ``cd koboldcpp-rocm`` ``mkdir build && cd build`` - ```cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DLLAMA_HIPBLAS=ON -DCMAKE_C_COMPILER="C:/Program Files/AMD/ROCm/5.5/bin/clang.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/AMD/ROCm/5.5/bin/clang++.exe" -DAMDGPU_TARGETS="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1010;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102"``` + ```cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DLLAMA_HIPBLAS=ON -DHIP_PLATFORM=amd -DCMAKE_C_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang.exe" -DCMAKE_CXX_COMPILER="C:/Program Files/AMD/ROCm/5.7/bin/clang++.exe" -DAMDGPU_TARGETS="gfx803;gfx900;gfx906;gfx908;gfx90a;gfx1010;gfx1030;gfx1031;gfx1032;gfx1100;gfx1101;gfx1102"``` ``cmake --build . -j 6`` (-j 6 means use 6 CPU cores, if you have more or less, feel free to change it to speed things up) - That puts koboldcpp_hipblas.dll inside of .\koboldcpp-rocm\build\bin - copy koboldcpp_hipblas.dll to the main koboldcpp-rocm folder - (You can run koboldcpp.py like this right away) like this: - + That puts koboldcpp_hipblas.dll inside of .\koboldcpp-rocm\build\bin + copy koboldcpp_hipblas.dll to the main koboldcpp-rocm folder + (You can run koboldcpp.py like this right away) like this: + ``python koboldcpp.py --usecublas mmq --threads 1 --contextsize 4096 --gpulayers 45 C:\Users\YellowRose\llama-2-7b-chat.Q8_0.gguf`` - To make it into an exe, we use ``make_pyinstaller_exe_rocm_only.bat`` - which will attempt to build the exe for you and place it in /koboldcpp-rocm/dists/ + To make it into an exe, we use ``make_pyinstaller_exe_rocm_only.bat`` + which will attempt to build the exe for you and place it in /koboldcpp-rocm/dists/ **kobold_rocm_only.exe is built!** - + ----- If you'd like to do a full feature build with OPENBLAS and CLBLAST backends, you'll need [w64devkit](https://github.com/skeeto/w64devkit). Once downloaded, open w64devkit.exe and ``cd`` into the folder then run ``make LLAMA_OPENBLAS=1 LLAMA_CLBLAST=1 -j4`` then it will build the rest of the backend files. @@ -78,12 +78,12 @@ My typical start command looks like this: ``python koboldcpp.py --threads 6 --bl - Also, replace the existing versions of the corresponding .dll files located in the project directory root (e.g. libopenblas.dll). - You can attempt a CuBLAS build with using the provided CMake file with visual studio. If you use the CMake file to build, copy the `koboldcpp_cublas.dll` generated into the same directory as the `koboldcpp.py` file. If you are bundling executables, you may need to include CUDA dynamic libraries (such as `cublasLt64_11.dll` and `cublas64_11.dll`) in order for the executable to work correctly on a different PC. - Make the KoboldCPP project using the instructions above. - - + - ## Docker - KoboldCpp has a few unofficial third-party community created docker images. Feel free to try them out, but do not expect up-to-date support: - https://github.com/korewaChino/koboldCppDocker - https://github.com/noneabove1182/koboldcpp-docker - + ## Arch Linux Packages There are 4 AUR packages available: [CPU-only](https://aur.archlinux.org/packages/koboldcpp-cpu), [CLBlast](https://aur.archlinux.org/packages/koboldcpp-clblast), [CUBLAS](https://aur.archlinux.org/packages/koboldcpp-cuda), and [HIPBLAS](https://aur.archlinux.org/packages/koboldcpp-hipblas). They are, respectively, for users with no GPU, users with a GPU (vendor-agnostic), users with NVIDIA GPUs, and users with a supported AMD GPU. @@ -165,4 +165,3 @@ Comparison with OpenCL using 6800xt (old measurement) - GPT-NeoX / Pythia / StableLM / Dolly / RedPajama - MPT models - Falcon (GGUF only) -