Add llama-cpp to main flake

flake.lock

@@ -18,6 +18,24 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1725234343,
+        "narHash": "sha256-+ebgonl3NbiKD2UD0x4BszCZQ6sTfL4xioaM49o5B3Y=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "567b938d64d4b4112ee253b9274472dc3a346eb6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "inputs": {
         "systems": "systems"
@@ -36,6 +54,30 @@
         "type": "github"
       }
     },
+    "llama-cpp": {
+      "inputs": {
+        "flake-parts": [
+          "flake-parts"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1725824628,
+        "narHash": "sha256-oiVEb+PMKumREdoV1vEzxfSWFlHNNMgxADgfpFsR8pE=",
+        "owner": "ggerganov",
+        "repo": "llama.cpp",
+        "rev": "daa9623ab051a8162ae750b150b9522571b55f21",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ggerganov",
+        "ref": "b3707",
+        "repo": "llama.cpp",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1722073938,
@@ -52,6 +94,18 @@
         "type": "github"
       }
     },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1725233747,
+        "narHash": "sha256-Ss8QWLXdr2JCBPcYChJhz4xJm+h/xjl4G0c0XlP6a74=",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/356624c12086a18f2ea2825fed34523d60ccc4e3.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/356624c12086a18f2ea2825fed34523d60ccc4e3.tar.gz"
+      }
+    },
     "nixpkgs_2": {
       "locked": {
         "lastModified": 1724208548,
@@ -103,7 +157,9 @@
     "root": {
       "inputs": {
         "devshell": "devshell",
+        "flake-parts": "flake-parts",
         "flake-utils": "flake-utils",
+        "llama-cpp": "llama-cpp",
         "nixpkgs": "nixpkgs_2",
         "npmlock2nix": "npmlock2nix",
         "rust-overlay": "rust-overlay"

flake.nix

@@ -1,18 +1,32 @@
 {
   description = "A Nix-flake-based Rust development environment";
   nixConfig = {
-    extra-substituters = [ "https://nixcache.vlt81.de" ];
+    extra-substituters = [
-    extra-trusted-public-keys = [ "nixcache.vlt81.de:nw0FfUpePtL6P3IMNT9X6oln0Wg9REZINtkkI9SisqQ=" ];
+      "https://nixcache.vlt81.de"
+      "https://llama-cpp.cachix.org"
+      "https://cuda-maintainers.cachix.org"
+    ];
+    extra-trusted-public-keys = [
+      "nixcache.vlt81.de:nw0FfUpePtL6P3IMNT9X6oln0Wg9REZINtkkI9SisqQ="
+      "llama-cpp.cachix.org-1:H75X+w83wUKTIPSO1KWy9ADUrzThyGs8P5tmAbkWhQc="
+      "cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E="
+    ];
   };
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     rust-overlay.url = "github:oxalica/rust-overlay";
     flake-utils.url = "github:numtide/flake-utils";
+    flake-parts.url = "github:hercules-ci/flake-parts";
     devshell.url = "github:numtide/devshell";
     npmlock2nix = {
       url = "github:nix-community/npmlock2nix";
       flake = false;
     };
+    llama-cpp = {
+      url = "github:ggerganov/llama.cpp/b3707";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.flake-parts.follows = "flake-parts";
+    };
   };
 
   outputs =
@@ -22,12 +36,14 @@
     , flake-utils
     , devshell
     , npmlock2nix
-    ,
+    , llama-cpp
+    , ...
     }:
     flake-utils.lib.eachDefaultSystem
       (system:
       let
         overlays = [
+          llama-cpp.overlays.default
           rust-overlay.overlays.default
           devshell.overlays.default
           (final: prev: {
@@ -39,6 +55,10 @@
         ];
         pkgs = import nixpkgs {
           inherit system overlays;
+          config = {
+            allowUnfree = true;
+            rocmSupport = true;
+          };
         };
         customNodeModules = pkgs.npmlock2nix.v2.node_modules {
           src = ./.;
@@ -68,17 +88,45 @@
           libGL
           wayland
         ];
+        rocmTargets = [
+          "gfx1100"
+          "gfx1102"
+        ];
       in
       {
         apps.devshell = self.outputs.devShells.${system}.default.flakeApp;
         packages = {
           cargo-leptos = pkgs.callPackage ./.nix/cargo-leptos.nix { };
+          # llama-cpp = pkgs.callPackage "${llama-cpp}/.devops/nix/scope.nix" { };
+          myllamacpp = with pkgs;
+            llamaPackages.llama-cpp.overrideDerivation (oldAttrs: {
+              # speeds up builts by only building for a needed rocmTargets...
+              cmakeFlags = [
+                (lib.cmakeBool "LLAMA_BUILD_SERVER" true)
+                (lib.cmakeBool "BUILD_SHARED_LIBS" true)
+                (lib.cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
+                (lib.cmakeBool "LLAMA_CURL" true)
+                (lib.cmakeBool "GGML_NATIVE" true)
+                (lib.cmakeBool "GGML_BLAS" false)
+                (lib.cmakeBool "GGML_CUDA" false)
+                (lib.cmakeBool "GGML_HIPBLAS" true)
+                (lib.cmakeBool "GGML_METAL" false)
+                (lib.cmakeBool "GGML_VULKAN" false)
+                (lib.cmakeBool "GGML_STATIC" false)
+                (lib.cmakeBool "GGML_FMA" true)
+                (lib.cmakeBool "GGML_F16C" true)
+                (lib.cmakeBool "GGML_AVX512" true)
+                (lib.cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
+                (lib.cmakeFeature "CMAKE_HIP_ARCHITECTURES" (builtins.concatStringsSep ";" rocmTargets))
+              ];
+            });
         };
         devShells.default = pkgs.mkShell {
           packages = with pkgs;
             [
               customNodeModules
               customRustToolchain
+              self.packages.${system}.myllamacpp
               binaryen
               cacert
               cargo-bloat

llama_forge_rs/flake.nix

@@ -10,9 +10,6 @@
       url = "github:nix-community/npmlock2nix";
       flake = false;
     };
-    llama-cpp = {
-      url = "git+https://git.vlt81.de/tristan/llama.cpp.git";
-    };
   };
 
   outputs =
@@ -22,7 +19,7 @@
     , rust-overlay
     , flake-utils
     , npmlock2nix
-    , llama-cpp
+    ,
     }:
     flake-utils.lib.eachDefaultSystem
       (system:
@@ -78,11 +75,6 @@
           src = ./.;
           nodejs = pkgs.nodejs_20;
         };
-        myllamacpp = llama-cpp.packages.${system}.rocm.override {
-          rocmTargets = [ "gfx1100" ];
-          llamaNative = true;
-          llamaLTO = false;
-        };
         buildInputs = with pkgs; [
           atk
           cairo
@@ -142,7 +134,6 @@
               sccache
               sqlx-cli
               unzip
-              myllamacpp
             ]
             ++ buildInputs;