From 3c3c232972c7fd72f88d6f5be466485ba778628c Mon Sep 17 00:00:00 2001 From: Tristan Druyen Date: Mon, 9 Sep 2024 14:56:13 +0200 Subject: [PATCH] Add llama-cpp to main flake --- flake.lock | 56 ++++++++++++++++++++++++++++++++++++++++ flake.nix | 54 +++++++++++++++++++++++++++++++++++--- llama_forge_rs/flake.nix | 11 +------- 3 files changed, 108 insertions(+), 13 deletions(-) diff --git a/flake.lock b/flake.lock index d314ca3..c136b12 100644 --- a/flake.lock +++ b/flake.lock @@ -18,6 +18,24 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1725234343, + "narHash": "sha256-+ebgonl3NbiKD2UD0x4BszCZQ6sTfL4xioaM49o5B3Y=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "567b938d64d4b4112ee253b9274472dc3a346eb6", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -36,6 +54,30 @@ "type": "github" } }, + "llama-cpp": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1725824628, + "narHash": "sha256-oiVEb+PMKumREdoV1vEzxfSWFlHNNMgxADgfpFsR8pE=", + "owner": "ggerganov", + "repo": "llama.cpp", + "rev": "daa9623ab051a8162ae750b150b9522571b55f21", + "type": "github" + }, + "original": { + "owner": "ggerganov", + "ref": "b3707", + "repo": "llama.cpp", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1722073938, @@ -52,6 +94,18 @@ "type": "github" } }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1725233747, + "narHash": "sha256-Ss8QWLXdr2JCBPcYChJhz4xJm+h/xjl4G0c0XlP6a74=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/356624c12086a18f2ea2825fed34523d60ccc4e3.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/356624c12086a18f2ea2825fed34523d60ccc4e3.tar.gz" + } + }, "nixpkgs_2": { "locked": { "lastModified": 1724208548, @@ -103,7 +157,9 @@ "root": { "inputs": { "devshell": "devshell", + "flake-parts": "flake-parts", "flake-utils": "flake-utils", + "llama-cpp": "llama-cpp", "nixpkgs": "nixpkgs_2", "npmlock2nix": "npmlock2nix", "rust-overlay": "rust-overlay" diff --git a/flake.nix b/flake.nix index 005f76e..279593d 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,32 @@ { description = "A Nix-flake-based Rust development environment"; nixConfig = { - extra-substituters = [ "https://nixcache.vlt81.de" ]; - extra-trusted-public-keys = [ "nixcache.vlt81.de:nw0FfUpePtL6P3IMNT9X6oln0Wg9REZINtkkI9SisqQ=" ]; + extra-substituters = [ + "https://nixcache.vlt81.de" + "https://llama-cpp.cachix.org" + "https://cuda-maintainers.cachix.org" + ]; + extra-trusted-public-keys = [ + "nixcache.vlt81.de:nw0FfUpePtL6P3IMNT9X6oln0Wg9REZINtkkI9SisqQ=" + "llama-cpp.cachix.org-1:H75X+w83wUKTIPSO1KWy9ADUrzThyGs8P5tmAbkWhQc=" + "cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=" + ]; }; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; rust-overlay.url = "github:oxalica/rust-overlay"; flake-utils.url = "github:numtide/flake-utils"; + flake-parts.url = "github:hercules-ci/flake-parts"; devshell.url = "github:numtide/devshell"; npmlock2nix = { url = "github:nix-community/npmlock2nix"; flake = false; }; + llama-cpp = { + url = "github:ggerganov/llama.cpp/b3707"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-parts.follows = "flake-parts"; + }; }; outputs = @@ -22,12 +36,14 @@ , flake-utils , devshell , npmlock2nix - , + , llama-cpp + , ... }: flake-utils.lib.eachDefaultSystem (system: let overlays = [ + llama-cpp.overlays.default rust-overlay.overlays.default devshell.overlays.default (final: prev: { @@ -39,6 +55,10 @@ ]; pkgs = import nixpkgs { inherit system overlays; + config = { + allowUnfree = true; + rocmSupport = true; + }; }; customNodeModules = pkgs.npmlock2nix.v2.node_modules { src = ./.; @@ -68,17 +88,45 @@ libGL wayland ]; + rocmTargets = [ + "gfx1100" + "gfx1102" + ]; in { apps.devshell = self.outputs.devShells.${system}.default.flakeApp; packages = { cargo-leptos = pkgs.callPackage ./.nix/cargo-leptos.nix { }; + # llama-cpp = pkgs.callPackage "${llama-cpp}/.devops/nix/scope.nix" { }; + myllamacpp = with pkgs; + llamaPackages.llama-cpp.overrideDerivation (oldAttrs: { + # speeds up builts by only building for a needed rocmTargets... + cmakeFlags = [ + (lib.cmakeBool "LLAMA_BUILD_SERVER" true) + (lib.cmakeBool "BUILD_SHARED_LIBS" true) + (lib.cmakeBool "CMAKE_SKIP_BUILD_RPATH" true) + (lib.cmakeBool "LLAMA_CURL" true) + (lib.cmakeBool "GGML_NATIVE" true) + (lib.cmakeBool "GGML_BLAS" false) + (lib.cmakeBool "GGML_CUDA" false) + (lib.cmakeBool "GGML_HIPBLAS" true) + (lib.cmakeBool "GGML_METAL" false) + (lib.cmakeBool "GGML_VULKAN" false) + (lib.cmakeBool "GGML_STATIC" false) + (lib.cmakeBool "GGML_FMA" true) + (lib.cmakeBool "GGML_F16C" true) + (lib.cmakeBool "GGML_AVX512" true) + (lib.cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang") + (lib.cmakeFeature "CMAKE_HIP_ARCHITECTURES" (builtins.concatStringsSep ";" rocmTargets)) + ]; + }); }; devShells.default = pkgs.mkShell { packages = with pkgs; [ customNodeModules customRustToolchain + self.packages.${system}.myllamacpp binaryen cacert cargo-bloat diff --git a/llama_forge_rs/flake.nix b/llama_forge_rs/flake.nix index 7c51e0c..8d0e5e2 100644 --- a/llama_forge_rs/flake.nix +++ b/llama_forge_rs/flake.nix @@ -10,9 +10,6 @@ url = "github:nix-community/npmlock2nix"; flake = false; }; - llama-cpp = { - url = "git+https://git.vlt81.de/tristan/llama.cpp.git"; - }; }; outputs = @@ -22,7 +19,7 @@ , rust-overlay , flake-utils , npmlock2nix - , llama-cpp + , }: flake-utils.lib.eachDefaultSystem (system: @@ -78,11 +75,6 @@ src = ./.; nodejs = pkgs.nodejs_20; }; - myllamacpp = llama-cpp.packages.${system}.rocm.override { - rocmTargets = [ "gfx1100" ]; - llamaNative = true; - llamaLTO = false; - }; buildInputs = with pkgs; [ atk cairo @@ -142,7 +134,6 @@ sccache sqlx-cli unzip - myllamacpp ] ++ buildInputs;