Add llama_proxy_man pkg

Cargo.lock

@@ -170,9 +170,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.87"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8"
+checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
 dependencies = [
  "backtrace",
 ]
@@ -188,9 +188,9 @@ dependencies = [
 
 [[package]]
 name = "arrayref"
-version = "0.3.8"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
 
 [[package]]
 name = "arrayvec"
@@ -309,9 +309,9 @@ dependencies = [
 
 [[package]]
 name = "async-process"
-version = "2.2.4"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8a07789659a4d385b79b18b9127fc27e1a59e1e89117c78c5ea3b806f016374"
+checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb"
 dependencies = [
  "async-channel",
  "async-io",
@@ -324,7 +324,6 @@ dependencies = [
  "futures-lite",
  "rustix",
  "tracing",
- "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -775,9 +774,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.7.1"
+version = "1.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
+checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
 
 [[package]]
 name = "cached"
@@ -896,9 +895,9 @@ checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
 
 [[package]]
 name = "cc"
-version = "1.1.18"
+version = "1.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476"
+checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0"
 dependencies = [
  "jobserver",
  "libc",
@@ -1199,6 +1198,34 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "cookie"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
+dependencies = [
+ "percent-encoding",
+ "time",
+ "version_check",
+]
+
+[[package]]
+name = "cookie_store"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4934e6b7e8419148b6ef56950d277af8561060b56afd59e2aadf98b59fce6baa"
+dependencies = [
+ "cookie",
+ "idna 0.5.0",
+ "log",
+ "publicsuffix",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "time",
+ "url",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.4"
@@ -1572,6 +1599,15 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "deranged"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
+dependencies = [
+ "powerfmt",
+]
+
 [[package]]
 name = "derive-where"
 version = "1.2.7"
@@ -1884,9 +1920,9 @@ dependencies = [
 
 [[package]]
 name = "error-code"
-version = "3.2.0"
+version = "3.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0474425d51df81997e2f90a21591180b38eccf27292d755f3e30750225c175b"
+checksum = "a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f"
 
 [[package]]
 name = "etagere"
@@ -3066,9 +3102,9 @@ dependencies = [
 
 [[package]]
 name = "hyper-util"
-version = "0.1.7"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9"
+checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -3086,9 +3122,9 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.60"
+version = "0.1.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141"
+checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
@@ -3109,8 +3145,8 @@ dependencies = [
 
 [[package]]
 name = "iced"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "iced_core",
  "iced_futures",
@@ -3122,8 +3158,8 @@ dependencies = [
 
 [[package]]
 name = "iced_core"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "bitflags 2.6.0",
  "bytes",
@@ -3141,8 +3177,8 @@ dependencies = [
 
 [[package]]
 name = "iced_futures"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "futures",
  "iced_core",
@@ -3154,8 +3190,8 @@ dependencies = [
 
 [[package]]
 name = "iced_graphics"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "bitflags 2.6.0",
  "bytemuck",
@@ -3173,8 +3209,8 @@ dependencies = [
 
 [[package]]
 name = "iced_renderer"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "iced_graphics",
  "iced_tiny_skia",
@@ -3185,8 +3221,8 @@ dependencies = [
 
 [[package]]
 name = "iced_runtime"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "bytes",
  "iced_core",
@@ -3197,8 +3233,8 @@ dependencies = [
 
 [[package]]
 name = "iced_tiny_skia"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "bytemuck",
  "cosmic-text",
@@ -3212,8 +3248,8 @@ dependencies = [
 
 [[package]]
 name = "iced_wgpu"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "bitflags 2.6.0",
  "bytemuck",
@@ -3231,8 +3267,8 @@ dependencies = [
 
 [[package]]
 name = "iced_widget"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "iced_renderer",
  "iced_runtime",
@@ -3245,8 +3281,8 @@ dependencies = [
 
 [[package]]
 name = "iced_winit"
-version = "0.13.0-dev"
+version = "0.13.1"
-source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e"
+source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534"
 dependencies = [
  "iced_futures",
  "iced_graphics",
@@ -3279,6 +3315,16 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
+[[package]]
+name = "idna"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
 [[package]]
 name = "idna"
 version = "0.5.0"
@@ -3338,6 +3384,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
 dependencies = [
  "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
 ]
 
 [[package]]
@@ -4119,6 +4168,29 @@ dependencies = [
  "x11-dl",
 ]
 
+[[package]]
+name = "llama_proxy_man"
+version = "0.1.1"
+dependencies = [
+ "anyhow",
+ "axum",
+ "futures",
+ "hyper",
+ "pin-project-lite",
+ "reqwest",
+ "reqwest-middleware",
+ "reqwest-retry",
+ "serde",
+ "serde_yaml",
+ "thiserror",
+ "tokio",
+ "tower",
+ "tower-http",
+ "tracing",
+ "tracing-subscriber",
+ "uuid",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.4.12"
@@ -4241,9 +4313,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
 [[package]]
 name = "memmap2"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322"
+checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
 dependencies = [
  "libc",
 ]
@@ -4533,6 +4605,12 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "num-conv"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+
 [[package]]
 name = "num-integer"
 version = "0.1.46"
@@ -5095,7 +5173,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.5.3",
+ "redox_syscall 0.5.4",
  "smallvec",
  "windows-targets 0.52.6",
 ]
@@ -5129,9 +5207,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
 [[package]]
 name = "pest"
-version = "2.7.12"
+version = "2.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c73c26c01b8c87956cea613c907c9d6ecffd8d18a2a5908e5de0adfaa185cea"
+checksum = "fdbef9d1d47087a895abd220ed25eb4ad973a5e26f6a4367b038c25e28dfc2d9"
 dependencies = [
  "memchr",
  "thiserror",
@@ -5140,9 +5218,9 @@ dependencies = [
 
 [[package]]
 name = "pest_derive"
-version = "2.7.12"
+version = "2.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "664d22978e2815783adbdd2c588b455b1bd625299ce36b2a99881ac9627e6d8d"
+checksum = "4d3a6e3394ec80feb3b6393c725571754c6188490265c61aaf260810d6b95aa0"
 dependencies = [
  "pest",
  "pest_generator",
@@ -5150,9 +5228,9 @@ dependencies = [
 
 [[package]]
 name = "pest_generator"
-version = "2.7.12"
+version = "2.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2d5487022d5d33f4c30d91c22afa240ce2a644e87fe08caad974d4eab6badbe"
+checksum = "94429506bde1ca69d1b5601962c73f4172ab4726571a59ea95931218cb0e930e"
 dependencies = [
  "pest",
  "pest_meta",
@@ -5163,9 +5241,9 @@ dependencies = [
 
 [[package]]
 name = "pest_meta"
-version = "2.7.12"
+version = "2.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0091754bbd0ea592c4deb3a122ce8ecbb0753b738aa82bc055fcc2eccc8d8174"
+checksum = "ac8a071862e93690b6e34e9a5fb8e33ff3734473ac0245b27232222c4906a33f"
 dependencies = [
  "once_cell",
  "pest",
@@ -5404,6 +5482,12 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.20"
@@ -5460,7 +5544,7 @@ version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b"
 dependencies = [
- "toml_edit 0.22.20",
+ "toml_edit 0.22.21",
 ]
 
 [[package]]
@@ -5596,6 +5680,12 @@ dependencies = [
  "prost",
 ]
 
+[[package]]
+name = "psl-types"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
+
 [[package]]
 name = "ptr_meta"
 version = "0.1.4"
@@ -5616,6 +5706,16 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "publicsuffix"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457"
+dependencies = [
+ "idna 0.3.0",
+ "psl-types",
+]
+
 [[package]]
 name = "pulldown-cmark"
 version = "0.12.1"
@@ -5924,9 +6024,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
+checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853"
 dependencies = [
  "bitflags 2.6.0",
 ]
@@ -6009,6 +6109,8 @@ checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63"
 dependencies = [
  "base64 0.22.1",
  "bytes",
+ "cookie",
+ "cookie_store",
  "encoding_rs",
  "futures-core",
  "futures-util",
@@ -6024,6 +6126,7 @@ dependencies = [
  "js-sys",
  "log",
  "mime",
+ "mime_guess",
  "native-tls",
  "once_cell",
  "percent-encoding",
@@ -6068,6 +6171,51 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "reqwest-middleware"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "http 1.1.0",
+ "reqwest",
+ "serde",
+ "thiserror",
+ "tower-service",
+]
+
+[[package]]
+name = "reqwest-retry"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a83df1aaec00176d0fabb65dea13f832d2a446ca99107afc17c5d2d4981221d0"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "futures",
+ "getrandom 0.2.15",
+ "http 1.1.0",
+ "hyper",
+ "parking_lot 0.11.2",
+ "reqwest",
+ "reqwest-middleware",
+ "retry-policies",
+ "tokio",
+ "tracing",
+ "wasm-timer",
+]
+
+[[package]]
+name = "retry-policies"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c"
+dependencies = [
+ "rand 0.8.5",
+]
+
 [[package]]
 name = "ring"
 version = "0.17.8"
@@ -6240,9 +6388,9 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.36"
+version = "0.38.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36"
+checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
 dependencies = [
  "bitflags 2.6.0",
  "errno",
@@ -6253,9 +6401,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.12"
+version = "0.23.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044"
+checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8"
 dependencies = [
  "log",
  "once_cell",
@@ -6310,9 +6458,9 @@ checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.102.7"
+version = "0.102.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56"
+checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
 dependencies = [
  "ring",
  "rustls-pki-types",
@@ -6590,6 +6738,19 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.34+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
+dependencies = [
+ "indexmap 2.5.0",
+ "itoa 1.0.11",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "server_fn"
 version = "0.6.15"
@@ -6930,7 +7091,7 @@ dependencies = [
  "objc2-foundation",
  "objc2-quartz-core",
  "raw-window-handle 0.6.2",
- "redox_syscall 0.5.3",
+ "redox_syscall 0.5.4",
  "rustix",
  "tiny-xlib",
  "wasm-bindgen",
@@ -7752,6 +7913,37 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "time"
+version = "0.3.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
+dependencies = [
+ "deranged",
+ "itoa 1.0.11",
+ "num-conv",
+ "powerfmt",
+ "serde",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
+
+[[package]]
+name = "time-macros"
+version = "0.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
+dependencies = [
+ "num-conv",
+ "time-core",
+]
+
 [[package]]
 name = "tiny-skia"
 version = "0.11.4"
@@ -7892,7 +8084,7 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit 0.22.20",
+ "toml_edit 0.22.21",
 ]
 
 [[package]]
@@ -7928,9 +8120,9 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.22.20"
+version = "0.22.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
+checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf"
 dependencies = [
  "indexmap 2.5.0",
  "serde",
@@ -8294,9 +8486,9 @@ checksum = "1df77b101bcc4ea3d78dafc5ad7e4f58ceffe0b2b16bf446aeb50b6cb4157656"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.12"
+version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
 
 [[package]]
 name = "unicode-linebreak"
@@ -8306,9 +8498,9 @@ checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
+checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
 dependencies = [
  "tinyvec",
 ]
@@ -8327,9 +8519,9 @@ checksum = "ad8d71f5726e5f285a935e9fe8edfd53f0491eb6e9a5774097fdabee7cd8c9cd"
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
 
 [[package]]
 name = "unicode-width"
@@ -8349,6 +8541,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"
@@ -8362,7 +8560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
 dependencies = [
  "form_urlencoded",
- "idna",
+ "idna 0.5.0",
  "percent-encoding",
 ]
 
@@ -8761,9 +8959,9 @@ dependencies = [
 
 [[package]]
 name = "webpki-roots"
-version = "0.26.5"
+version = "0.26.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a"
+checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -8917,7 +9115,7 @@ version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d"
 dependencies = [
- "redox_syscall 0.5.3",
+ "redox_syscall 0.5.4",
  "wasite",
 ]
 

Cargo.toml

@@ -27,7 +27,7 @@ lto = "fat"
 panic = "abort"
 
 [workspace]
-members = ["llama_forge_rs", "leptos_stub", "frozen_llama"]
+members = ["llama_forge_rs", "leptos_stub", "frozen_llama", "llama_proxy_man"]
 resolver = "2"
 
 [workspace.package]

llama_proxy_man/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "llama_proxy_man"
+edition = "2021"
+authors.workspace = true
+description.workspace = true
+license.workspace = true
+publish.workspace = true
+readme.workspace = true
+repository.workspace = true
+version.workspace = true
+
+[dependencies]
+tokio = { version = "1", features = ["full", "process"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_yaml = "0.9"
+axum = { version = "0.7", features = ["macros"] }
+hyper = { version = "1.4", features = ["full"] }
+reqwest = { version = "0.12", features = ["cookies", "multipart", "json", "stream", "native-tls"] }
+futures = "0.3.30"
+anyhow = { version = "1.0.89", features = ["backtrace"] }
+thiserror = "1.0.63"
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+tracing = "0.1.40"
+uuid = { version = "1.10.0", features = ["v4", "v7", "serde"] }
+pin-project-lite = "0.2.14"
+tower = { version = "0.4", features = ["tokio", "tracing"] }
+tower-http = { version = "0.5.2", features = ["trace"] }
+reqwest-retry = "0.6.1"
+reqwest-middleware = { version = "0.3.3", features = ["charset", "http2", "json", "multipart", "rustls-tls"] }

llama_proxy_man/README.md

@@ -0,0 +1,5 @@
+# LLama Herder
+
+- manages multiple llama.cpp instances in the background
+- keeps track of used & available video & cpu memory
+- starts/stops llama.cpp instances as needed, to ensure memory limit is never reached

llama_proxy_man/TODO.org

@@ -0,0 +1,34 @@
+#+title: Todo
+
+
+* Name ideas
+- llama herder
+- llama herdsman/women/boy ??
+- llama shepherd ?
+
+* MVP
+- [X] fix stopping (doesn't work correctly at all)
+- seems done
+
+* Future Features
+- [ ] support for model selection by name on a unified port for /api & /completions
+  - [ ] separation of proxy/selection stuff ? config for unmanaged instances for auto model-selection by name
+- [ ] automatic internal port management (search for free ports)
+- [ ] Diagnostic Overview UI/API
+- [ ] Config UI/API ?
+- [ ] better book-keeping abt inflight requests ? (needed ?)
+- [ ] multi node stuff
+  - how exactly ?
+    - clustering ? (one manager per node ?)
+    - ssh support ???
+- [ ] automatic ram usage calc ?
+- [ ] other runners
+  - e.g. docker/ run in path etc
+- [ ] other backends ?
+- [ ] more advanced start/stop behavior
+  - more config ? e.g. pinning/priorities/prefer-to-kill/start-initially
+  - LRU /most used prioritized to keep running
+  - speculative relaunch
+  - scheduling of how to order in-flight requests + restarts to handle them optimally
+- [ ] advanced high-level foo
+  - automatic context-size selection per request/ start with bigger context if current instance has to low context

llama_proxy_man/config.yaml

@@ -0,0 +1,26 @@
+hardware:
+  ram: 64G
+  vram: 8G
+models:
+  - port: 18080
+    internal_port: 28080
+    env:
+      CUDA_VISIBLE_DEVICES: 0
+      HSA_OVERRIDE_GFX_VERSION: '11.0.0'
+    args:
+      model: /home/tristand/Downloads/models/Phi-3.5-mini-instruct-Q6_K_L.gguf
+      gpu-layers: 9999
+      ctx-size: 4096
+    vram_usage: 6G
+    ram_usage: 500M
+  - port: 18081
+    internal_port: 28081
+    env:
+      CUDA_VISIBLE_DEVICES: 0
+      HSA_OVERRIDE_GFX_VERSION: '11.0.0'
+    args:
+      model: /home/tristand/Downloads/models/Phi-3.5-mini-instruct-Q6_K_L.gguf
+      gpu-layers: 9999
+      ctx-size: 4096
+    vram_usage: 6G
+    ram_usage: 500M

llama_proxy_man/src/audit.json

@@ -0,0 +1,16 @@
+{
+    "keepSettings": {
+        "type": 1,
+        "amount": 30
+    },
+    "auditFilename": "audit.json",
+    "hashType": "md5",
+    "extension": ".log",
+    "files": [
+        {
+            "date": 1726742235723,
+            "name": "/home/tristand/.tabby-client/agent/logs/20240919.0.log",
+            "hash": "13392f16f09c9e264d7c9b82880ae40c"
+        }
+    ]
+}

llama_proxy_man/src/logging.rs

@@ -0,0 +1,83 @@
+use std::{
+    future::Future,
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use axum::{body::Body, http::Request};
+use pin_project_lite::pin_project;
+use tower::{Layer, Service};
+use uuid::Uuid; // Make sure to include `uuid` crate in your Cargo.toml
+
+pub struct LoggingLayer;
+
+impl<S> Layer<S> for LoggingLayer {
+    type Service = LoggingService<S>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        LoggingService { inner }
+    }
+}
+
+pub struct LoggingService<T> {
+    inner: T,
+}
+
+impl<T> Service<Request<Body>> for LoggingService<T>
+where
+    T: Service<Request<Body>>,
+{
+    type Error = T::Error;
+    type Future = LoggingServiceFuture<T::Future>;
+    type Response = T::Response;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, req: Request<Body>) -> Self::Future {
+        let request_uuid = Uuid::now_v7(); // Generate UUID v7
+
+        tracing::info!("[{}] {} {}", request_uuid, req.method(), req.uri());
+        for (k, v) in req.headers() {
+            tracing::info!(
+                "[{}] {}: {}",
+                request_uuid,
+                k,
+                v.to_str().unwrap_or("invalid header")
+            );
+        }
+
+        LoggingServiceFuture {
+            inner: self.inner.call(req),
+            uuid: Arc::new(request_uuid), // Store UUID in an Arc for shared ownership
+        }
+    }
+}
+
+pin_project! {
+    pub struct LoggingServiceFuture<T> {
+        #[pin]
+        inner: T,
+        uuid: Arc<Uuid>, // Shared state between LoggingService and LoggingServiceFuture
+    }
+}
+
+impl<T> Future for LoggingServiceFuture<T>
+where
+    T: Future,
+{
+    type Output = T::Output;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let this = self.project();
+        match this.inner.poll(cx) {
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(output) => {
+                tracing::trace!("[{}] request finished", this.uuid);
+                Poll::Ready(output)
+            }
+        }
+    }
+}

llama_proxy_man/src/main.rs

@@ -0,0 +1,363 @@
+mod logging;
+
+use anyhow::anyhow;
+use axum::{
+    body::Body,
+    http::{self, Request, Response},
+    response::IntoResponse,
+    routing::any,
+    Router,
+};
+use futures;
+use reqwest::Client;
+use serde::Deserialize;
+use std::{collections::HashMap, net::SocketAddr, process::Stdio, sync::Arc};
+use tokio::{
+    net::TcpStream,
+    process::{Child, Command},
+    sync::Mutex,
+    time::{sleep, Duration},
+};
+use tower_http::trace::{
+    DefaultMakeSpan, DefaultOnEos, DefaultOnFailure, DefaultOnRequest, DefaultOnResponse,
+    TraceLayer,
+};
+use tracing::Level;
+
+use std::sync::Once;
+
+pub static INIT: Once = Once::new();
+
+pub fn initialize_logger() {
+    INIT.call_once(|| {
+        let env_filter = tracing_subscriber::EnvFilter::builder()
+            .with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
+            .from_env_lossy();
+
+        tracing_subscriber::fmt()
+            .compact()
+            .with_env_filter(env_filter)
+            .init();
+    });
+}
+
+#[derive(Debug, Deserialize)]
+struct Config {
+    hardware: Hardware,
+    models: Vec<ModelConfig>,
+}
+
+#[derive(Debug, Deserialize)]
+struct Hardware {
+    ram: String,
+    vram: String,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+struct ModelConfig {
+    port: u16,
+    internal_port: u16,
+    env: HashMap<String, String>,
+    args: HashMap<String, String>,
+    vram_usage: String,
+    ram_usage: String,
+}
+
+#[derive(Clone)]
+struct LlamaInstance {
+    config: ModelConfig,
+    process: Arc<Mutex<Child>>,
+    // busy: bool,
+}
+
+struct SharedState {
+    total_ram: u64,
+    total_vram: u64,
+    used_ram: u64,
+    used_vram: u64,
+    instances: HashMap<u16, LlamaInstance>,
+}
+
+#[tokio::main]
+async fn main() {
+    initialize_logger();
+    // Read and parse the YAML configuration
+    let config_str = std::fs::read_to_string("config.yaml").expect("Failed to read config.yaml");
+    let config: Config = serde_yaml::from_str(&config_str).expect("Failed to parse config.yaml");
+
+    // Parse hardware resources
+    let total_ram = parse_size(&config.hardware.ram).expect("Invalid RAM size in config");
+    let total_vram = parse_size(&config.hardware.vram).expect("Invalid VRAM size in config");
+
+    // Initialize shared state
+    let shared_state = Arc::new(Mutex::new(SharedState {
+        total_ram,
+        total_vram,
+        used_ram: 0,
+        used_vram: 0,
+        instances: HashMap::new(),
+    }));
+
+    // For each model, set up an axum server listening on the specified port
+    let mut handles = Vec::new();
+
+    for model_config in config.models {
+        let state = shared_state.clone();
+        let model_config = model_config.clone();
+
+        let handle = tokio::spawn(async move {
+            let model_config = model_config.clone();
+
+            let app = Router::new()
+                .route(
+                    "/",
+                    any({
+                        let state = state.clone();
+                        let model_config = model_config.clone();
+                        move |req| handle_request(req, model_config, state)
+                    }),
+                )
+                .route(
+                    "/*path",
+                    any({
+                        let state = state.clone();
+                        let model_config = model_config.clone();
+                        move |req| handle_request(req, model_config, state)
+                    }),
+                )
+                .layer(
+                    TraceLayer::new_for_http()
+                        .make_span_with(DefaultMakeSpan::new().include_headers(true))
+                        .on_request(DefaultOnRequest::new().level(Level::DEBUG))
+                        .on_response(DefaultOnResponse::new().level(Level::TRACE))
+                        .on_eos(DefaultOnEos::new().level(Level::DEBUG))
+                        .on_failure(DefaultOnFailure::new().level(Level::ERROR)),
+                );
+
+            let addr = SocketAddr::from(([0, 0, 0, 0], model_config.port));
+
+            println!("Listening on port {}", model_config.port);
+            let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();
+
+            axum::serve(listener, app.into_make_service())
+                .await
+                .unwrap();
+        });
+
+        handles.push(handle);
+    }
+
+    futures::future::join_all(handles).await;
+}
+
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum AppError {
+    #[error("Axum Error")]
+    AxumError(#[from] axum::Error),
+    #[error("Axum Http Error")]
+    AxumHttpError(#[from] axum::http::Error),
+    #[error("Reqwest Error")]
+    ReqwestError(#[from] reqwest::Error),
+    #[error("Reqwest Middleware Error")]
+    ReqwestMiddlewareError(#[from] reqwest_middleware::Error),
+    #[error("Client Error")]
+    ClientError(#[from] hyper::Error),
+    #[error("Unknown error")]
+    Unknown(#[from] anyhow::Error),
+}
+
+// Tell axum how to convert `AppError` into a response.
+impl IntoResponse for AppError {
+    fn into_response(self) -> axum::response::Response {
+        tracing::error!("::AppError::into_response:: {:?}", self);
+        (
+            http::StatusCode::INTERNAL_SERVER_ERROR,
+            format!("Something went wrong: {:?}", self),
+        )
+            .into_response()
+    }
+}
+
+async fn handle_request(
+    req: Request<Body>,
+    model_config: ModelConfig,
+    state: Arc<Mutex<SharedState>>,
+    // ) -> Result<Response<Body>, anyhow::Error> {
+) -> impl IntoResponse {
+    let model_ram_usage = parse_size(&model_config.ram_usage).expect("Invalid ram_usage");
+    let model_vram_usage = parse_size(&model_config.vram_usage).expect("Invalid vram_usage");
+
+    let instance = {
+        let mut state = state.lock().await;
+
+        // Check if instance is running
+        if let Some(instance) = state.instances.get_mut(&model_config.port) {
+            // instance.busy = true;
+            instance.to_owned()
+        } else {
+            // Check resources
+            if state.used_ram + model_ram_usage > state.total_ram
+                || state.used_vram + model_vram_usage > state.total_vram
+            {
+                // Stop other instances
+                let mut to_remove = Vec::new();
+                for (port, instance) in state.instances.clone() {
+                    // if !instance.busy {
+                    tracing::info!("Stopping instance on port {}", port);
+                    let mut process = instance.process.lock().await;
+                    process.kill().await.ok();
+                    state.used_ram -= parse_size(&instance.config.ram_usage).unwrap_or(0);
+                    state.used_vram -= parse_size(&instance.config.vram_usage).unwrap_or(0);
+                    to_remove.push(port);
+                    if state.used_ram + model_ram_usage <= state.total_ram
+                        && state.used_vram + model_vram_usage <= state.total_vram
+                    {
+                        break;
+                    }
+                    // }
+                }
+                for port in to_remove {
+                    tracing::info!("Removing instance on port {}", port);
+                    state.instances.remove(&port);
+                }
+            }
+
+            // Start new instance
+            let args = model_config
+                .args
+                .iter()
+                .flat_map(|(k, v)| vec![format!("--{}", k), v.clone()])
+                .collect::<Vec<_>>();
+
+            let mut cmd = Command::new("llama-server");
+            cmd.envs(model_config.env.clone());
+            cmd.args(&args);
+            cmd.arg("--port");
+            cmd.arg(format!("{}", model_config.internal_port));
+            cmd.stdout(Stdio::null()).stderr(Stdio::null());
+
+            tracing::info!("Starting llama-server with {:?}", cmd);
+            let process = Arc::new(Mutex::new(
+                cmd.spawn().expect("Failed to start llama-server"),
+            ));
+
+            state.used_ram += model_ram_usage;
+            state.used_vram += model_vram_usage;
+
+            let instance = LlamaInstance {
+                config: model_config.clone(),
+                process,
+                // busy: true,
+            };
+            sleep(Duration::from_millis(500)).await;
+            state.instances.insert(model_config.port, instance.clone());
+
+            instance
+        }
+    };
+
+    // Wait for the instance to be ready
+    is_llama_instance_running(&instance).await?;
+    wait_for_port(model_config.internal_port).await?;
+
+    // Proxy the request
+    let retry_policy = reqwest_retry::policies::ExponentialBackoff::builder()
+        .retry_bounds(Duration::from_secs(1), Duration::from_secs(8))
+        .jitter(reqwest_retry::Jitter::None)
+        .base(2)
+        .build_with_max_retries(8);
+
+    let client = reqwest_middleware::ClientBuilder::new(Client::new())
+        .with(reqwest_retry::RetryTransientMiddleware::new_with_policy(
+            retry_policy,
+        ))
+    .build();
+
+    let uri = format!(
+        "http://127.0.0.1:{}{}",
+        model_config.internal_port,
+        req.uri().path_and_query().map(|x| x.as_str()).unwrap_or("")
+    );
+
+    let mut request_builder = client.request(req.method().clone(), &uri);
+    // let mut request_builder = reqwest::RequestBuilder::from_parts(client, req.method().clone())
+
+    for (name, value) in req.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+
+    let body_bytes = axum::body::to_bytes(
+        req.into_body(),
+        parse_size("1G").unwrap().try_into().unwrap(),
+    )
+    .await?;
+    let request = request_builder.body(body_bytes).build()?;
+    // tracing::info!("Proxying request to {}", uri);
+    let response = client.execute(request).await?;
+
+    // tracing::info!("Received response from {}", uri);
+    let mut builder = Response::builder().status(response.status());
+    for (name, value) in response.headers() {
+        builder = builder.header(name, value);
+    }
+
+    // let bytes = response.bytes().await?;
+
+    let byte_stream = response.bytes_stream();
+
+    let body = axum::body::Body::from_stream(byte_stream);
+
+    tracing::info!("streaming response on port: {}", model_config.port);
+    let response = builder.body(body)?;
+    Ok::<axum::http::Response<Body>, AppError>(response)
+}
+
+async fn wait_for_port(port: u16) -> Result<(), anyhow::Error> {
+    for _ in 0..10 {
+        if TcpStream::connect(("127.0.0.1", port)).await.is_ok() {
+            return Ok(());
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+    Err(anyhow!("Timeout waiting for port"))
+}
+
+// reimplement wait_for_port using a LlamaInstance, also check if the process is still running
+async fn is_llama_instance_running(instance: &LlamaInstance) -> Result<(), anyhow::Error> {
+    match instance.process.to_owned().lock_owned().await.try_wait()? {
+        Some(exit_status) => {
+            let msg = "Llama instance exited";
+            tracing::error!(msg, ?exit_status);
+            Err(anyhow!(msg))
+        }
+
+        None => Ok(()),
+    }
+}
+
+fn parse_size(size_str: &str) -> Option<u64> {
+    let mut num = String::new();
+    let mut unit = String::new();
+
+    for c in size_str.chars() {
+        if c.is_digit(10) {
+            num.push(c);
+        } else {
+            unit.push(c);
+        }
+    }
+
+    let num: f64 = num.parse().ok()?;
+
+    let multiplier = match unit.to_lowercase().as_str() {
+        "g" | "gb" => 1024 * 1024 * 1024,
+        "m" | "mb" => 1024 * 1024,
+        "k" | "kb" => 1024,
+        _ => 1,
+    };
+
+    let res = (num * multiplier as f64) as u64;
+    Some(res)
+}