python3Packages.llama-cpp-python: init at 0.2.18

pkgs/by-name/ll/llama-cpp/package.nix

@@ -88,6 +88,7 @@ stdenv.mkDerivation (finalAttrs: {
   cmakeFlags = [
     "-DLLAMA_NATIVE=OFF"
     "-DLLAMA_BUILD_SERVER=ON"
+    "-DBUILD_SHARED_LIBS=ON"
   ]
   ++ lib.optionals metalSupport [
     "-DCMAKE_C_FLAGS=-D__ARM_FEATURE_DOTPROD=1"
@@ -113,7 +114,7 @@ stdenv.mkDerivation (finalAttrs: {
   installPhase = ''
     runHook preInstall
 
-    mkdir -p $out/bin
+    mkdir -p $out/{bin,lib,share/models}
 
     for f in bin/*; do
       test -x "$f" || continue
@@ -122,6 +123,11 @@ stdenv.mkDerivation (finalAttrs: {
 
     ${lib.optionalString metalSupport "cp ./bin/ggml-metal.metal $out/bin/ggml-metal.metal"}
 
+    cp libggml_shared${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/libggml_shared${stdenv.hostPlatform.extensions.sharedLibrary}
+    cp libllama${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/libllama${stdenv.hostPlatform.extensions.sharedLibrary}
+
+    cp $src/models/* $out/share/models
+
     runHook postInstall
   '';
 

pkgs/development/python-modules/llama-cpp-python/default.nix

@@ -0,0 +1,69 @@
+{ buildPythonPackage
+, diskcache
+, fetchFromGitHub
+, lib
+, llama-cpp
+, numpy
+, pytestCheckHook
+, setuptools
+, stdenv
+, typing-extensions
+}:
+
+buildPythonPackage rec {
+  pname = "llama-cpp-python";
+  version = "0.2.18";
+
+  src = fetchFromGitHub {
+    owner = "abetlen";
+    repo = "llama-cpp-python";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-kUoc61tSS7ohAl7vIN6Yt/TV1RLQg45QZXpff/URImA=";
+  };
+
+  patches = [
+    ./disable-llama-cpp-build.patch
+    ./set-llamacpp-path.patch
+  ];
+
+  postPatch = ''
+    substituteInPlace llama_cpp/llama_cpp.py \
+      --subst-var-by llamaCppSharedLibrary "${llama-cpp}/lib/libllama${stdenv.hostPlatform.extensions.sharedLibrary}"
+
+    substituteInPlace tests/test_llama.py \
+      --subst-var-by llamaCppModels "${llama-cpp}/share/models"
+  '';
+
+  format = "pyproject";
+
+  nativeBuildInputs = [
+    setuptools
+  ];
+
+  propagatedBuildInputs = [
+    diskcache
+    numpy
+    typing-extensions
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  disabledTests = [
+    # requires pydantic-settings which is broken
+    "test_llama_server"
+
+    # failing test due to regression in llama-cpp, see
+    # https://github.com/abetlen/llama-cpp-python/commit/55efc9e6b2a07bffa5250903de79e5f4873ba73d
+    "test_llama_cpp_tokenization"
+    "test_llama_patch"
+  ];
+
+  meta = with lib; {
+    description = "Python bindings for llama.cpp";
+    homepage = "https://github.com/abetlen/llama-cpp-python";
+    license = licenses.mit;
+    maintainers = with maintainers; [ elohmeier ];
+  };
+}

pkgs/development/python-modules/llama-cpp-python/disable-llama-cpp-build.patch

@@ -0,0 +1,13 @@
+diff --git a/pyproject.toml b/pyproject.toml
+index 6c10225..eaf2b6e 100644
+--- a/pyproject.toml
++++ b/pyproject.toml
+@@ -1,6 +1,5 @@
+-[build-system]
+-requires = ["scikit-build-core[pyproject]>=0.5.1"]
+-build-backend = "scikit_build_core.build"
++[tool.setuptools]
++packages = ["llama_cpp"]
+
+ [project]
+ name = "llama_cpp_python"

pkgs/development/python-modules/llama-cpp-python/set-llamacpp-path.patch

@@ -0,0 +1,26 @@
+diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py
+index a4d2100..8209415 100644
+--- a/llama_cpp/llama_cpp.py
++++ b/llama_cpp/llama_cpp.py
+@@ -62,6 +62,8 @@ def _load_shared_library(lib_base_name: str):
+             os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib"))
+         cdll_args["winmode"] = ctypes.RTLD_GLOBAL
+
++    _lib_paths = [pathlib.Path("@llamaCppSharedLibrary@")]
++
+     # Try to load the shared library, handling potential errors
+     for _lib_path in _lib_paths:
+         if _lib_path.exists():
+diff --git a/tests/test_llama.py b/tests/test_llama.py
+index 23c7e86..691a8ea 100644
+--- a/tests/test_llama.py
++++ b/tests/test_llama.py
+@@ -4,7 +4,7 @@ import pytest
+
+ import llama_cpp
+
+-MODEL = "./vendor/llama.cpp/models/ggml-vocab-llama.gguf"
++MODEL = "@llamaCppModels@/ggml-vocab-llama.gguf"
+
+
+ def test_llama_cpp_tokenization():

pkgs/top-level/python-packages.nix

@@ -6398,6 +6398,8 @@ self: super: with self; {
 
   lizard = callPackage ../development/python-modules/lizard { };
 
+  llama-cpp-python = callPackage ../development/python-modules/llama-cpp-python { };
+
   llfuse = callPackage ../development/python-modules/llfuse {
     inherit (pkgs) fuse;
   };