diff --git a/Cargo.lock b/Cargo.lock index f125565..dda78f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,9 +170,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.87" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" dependencies = [ "backtrace", ] @@ -188,9 +188,9 @@ dependencies = [ [[package]] name = "arrayref" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" @@ -309,9 +309,9 @@ dependencies = [ [[package]] name = "async-process" -version = "2.2.4" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a07789659a4d385b79b18b9127fc27e1a59e1e89117c78c5ea3b806f016374" +checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" dependencies = [ "async-channel", "async-io", @@ -324,7 +324,6 @@ dependencies = [ "futures-lite", "rustix", "tracing", - "windows-sys 0.59.0", ] [[package]] @@ -775,9 +774,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cached" @@ -896,9 +895,9 @@ checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" [[package]] name = "cc" -version = "1.1.18" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "jobserver", "libc", @@ -1199,6 +1198,34 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4934e6b7e8419148b6ef56950d277af8561060b56afd59e2aadf98b59fce6baa" +dependencies = [ + "cookie", + "idna 0.5.0", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1572,6 +1599,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "derive-where" version = "1.2.7" @@ -1884,9 +1920,9 @@ dependencies = [ [[package]] name = "error-code" -version = "3.2.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0474425d51df81997e2f90a21591180b38eccf27292d755f3e30750225c175b" +checksum = "a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f" [[package]] name = "etagere" @@ -3066,9 +3102,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -3086,9 +3122,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3109,8 +3145,8 @@ dependencies = [ [[package]] name = "iced" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "iced_core", "iced_futures", @@ -3122,8 +3158,8 @@ dependencies = [ [[package]] name = "iced_core" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "bitflags 2.6.0", "bytes", @@ -3141,8 +3177,8 @@ dependencies = [ [[package]] name = "iced_futures" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "futures", "iced_core", @@ -3154,8 +3190,8 @@ dependencies = [ [[package]] name = "iced_graphics" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "bitflags 2.6.0", "bytemuck", @@ -3173,8 +3209,8 @@ dependencies = [ [[package]] name = "iced_renderer" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "iced_graphics", "iced_tiny_skia", @@ -3185,8 +3221,8 @@ dependencies = [ [[package]] name = "iced_runtime" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "bytes", "iced_core", @@ -3197,8 +3233,8 @@ dependencies = [ [[package]] name = "iced_tiny_skia" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "bytemuck", "cosmic-text", @@ -3212,8 +3248,8 @@ dependencies = [ [[package]] name = "iced_wgpu" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "bitflags 2.6.0", "bytemuck", @@ -3231,8 +3267,8 @@ dependencies = [ [[package]] name = "iced_widget" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "iced_renderer", "iced_runtime", @@ -3245,8 +3281,8 @@ dependencies = [ [[package]] name = "iced_winit" -version = "0.13.0-dev" -source = "git+https://github.com/iced-rs/iced.git?branch=master#630f3525ddc58f1ee8dfec2f7567f52ea77eaa6e" +version = "0.13.1" +source = "git+https://github.com/iced-rs/iced.git?branch=master#bf3b6f100df7b1585dfac88da432bc29784ed534" dependencies = [ "iced_futures", "iced_graphics", @@ -3279,6 +3315,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.5.0" @@ -3338,6 +3384,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -4119,6 +4168,29 @@ dependencies = [ "x11-dl", ] +[[package]] +name = "llama_proxy_man" +version = "0.1.1" +dependencies = [ + "anyhow", + "axum", + "futures", + "hyper", + "pin-project-lite", + "reqwest", + "reqwest-middleware", + "reqwest-retry", + "serde", + "serde_yaml", + "thiserror", + "tokio", + "tower", + "tower-http", + "tracing", + "tracing-subscriber", + "uuid", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -4241,9 +4313,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" dependencies = [ "libc", ] @@ -4533,6 +4605,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -5095,7 +5173,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.3", + "redox_syscall 0.5.4", "smallvec", "windows-targets 0.52.6", ] @@ -5129,9 +5207,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.12" +version = "2.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c73c26c01b8c87956cea613c907c9d6ecffd8d18a2a5908e5de0adfaa185cea" +checksum = "fdbef9d1d47087a895abd220ed25eb4ad973a5e26f6a4367b038c25e28dfc2d9" dependencies = [ "memchr", "thiserror", @@ -5140,9 +5218,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.12" +version = "2.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "664d22978e2815783adbdd2c588b455b1bd625299ce36b2a99881ac9627e6d8d" +checksum = "4d3a6e3394ec80feb3b6393c725571754c6188490265c61aaf260810d6b95aa0" dependencies = [ "pest", "pest_generator", @@ -5150,9 +5228,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.12" +version = "2.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d5487022d5d33f4c30d91c22afa240ce2a644e87fe08caad974d4eab6badbe" +checksum = "94429506bde1ca69d1b5601962c73f4172ab4726571a59ea95931218cb0e930e" dependencies = [ "pest", "pest_meta", @@ -5163,9 +5241,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.12" +version = "2.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0091754bbd0ea592c4deb3a122ce8ecbb0753b738aa82bc055fcc2eccc8d8174" +checksum = "ac8a071862e93690b6e34e9a5fb8e33ff3734473ac0245b27232222c4906a33f" dependencies = [ "once_cell", "pest", @@ -5404,6 +5482,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -5460,7 +5544,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.22.20", + "toml_edit 0.22.21", ] [[package]] @@ -5596,6 +5680,12 @@ dependencies = [ "prost", ] +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + [[package]] name = "ptr_meta" version = "0.1.4" @@ -5616,6 +5706,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "publicsuffix" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457" +dependencies = [ + "idna 0.3.0", + "psl-types", +] + [[package]] name = "pulldown-cmark" version = "0.12.1" @@ -5924,9 +6024,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags 2.6.0", ] @@ -6009,6 +6109,8 @@ checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "base64 0.22.1", "bytes", + "cookie", + "cookie_store", "encoding_rs", "futures-core", "futures-util", @@ -6024,6 +6126,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "native-tls", "once_cell", "percent-encoding", @@ -6068,6 +6171,51 @@ dependencies = [ "thiserror", ] +[[package]] +name = "reqwest-middleware" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" +dependencies = [ + "anyhow", + "async-trait", + "http 1.1.0", + "reqwest", + "serde", + "thiserror", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a83df1aaec00176d0fabb65dea13f832d2a446ca99107afc17c5d2d4981221d0" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "getrandom 0.2.15", + "http 1.1.0", + "hyper", + "parking_lot 0.11.2", + "reqwest", + "reqwest-middleware", + "retry-policies", + "tokio", + "tracing", + "wasm-timer", +] + +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "ring" version = "0.17.8" @@ -6240,9 +6388,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.36" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -6253,9 +6401,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "log", "once_cell", @@ -6310,9 +6458,9 @@ checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -6590,6 +6738,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.5.0", + "itoa 1.0.11", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "server_fn" version = "0.6.15" @@ -6930,7 +7091,7 @@ dependencies = [ "objc2-foundation", "objc2-quartz-core", "raw-window-handle 0.6.2", - "redox_syscall 0.5.3", + "redox_syscall 0.5.4", "rustix", "tiny-xlib", "wasm-bindgen", @@ -7752,6 +7913,37 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa 1.0.11", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-skia" version = "0.11.4" @@ -7892,7 +8084,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.20", + "toml_edit 0.22.21", ] [[package]] @@ -7928,9 +8120,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" dependencies = [ "indexmap 2.5.0", "serde", @@ -8294,9 +8486,9 @@ checksum = "1df77b101bcc4ea3d78dafc5ad7e4f58ceffe0b2b16bf446aeb50b6cb4157656" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-linebreak" @@ -8306,9 +8498,9 @@ checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -8327,9 +8519,9 @@ checksum = "ad8d71f5726e5f285a935e9fe8edfd53f0491eb6e9a5774097fdabee7cd8c9cd" [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" @@ -8349,6 +8541,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -8362,7 +8560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", - "idna", + "idna 0.5.0", "percent-encoding", ] @@ -8761,9 +8959,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] @@ -8917,7 +9115,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.5.3", + "redox_syscall 0.5.4", "wasite", ] diff --git a/Cargo.toml b/Cargo.toml index 9426cad..fd0ffba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ lto = "fat" panic = "abort" [workspace] -members = ["llama_forge_rs", "leptos_stub", "frozen_llama"] +members = ["llama_forge_rs", "leptos_stub", "frozen_llama", "llama_proxy_man"] resolver = "2" [workspace.package] diff --git a/llama_proxy_man/Cargo.toml b/llama_proxy_man/Cargo.toml new file mode 100644 index 0000000..e2a0ce2 --- /dev/null +++ b/llama_proxy_man/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "llama_proxy_man" +edition = "2021" +authors.workspace = true +description.workspace = true +license.workspace = true +publish.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true + +[dependencies] +tokio = { version = "1", features = ["full", "process"] } +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" +axum = { version = "0.7", features = ["macros"] } +hyper = { version = "1.4", features = ["full"] } +reqwest = { version = "0.12", features = ["cookies", "multipart", "json", "stream", "native-tls"] } +futures = "0.3.30" +anyhow = { version = "1.0.89", features = ["backtrace"] } +thiserror = "1.0.63" +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +tracing = "0.1.40" +uuid = { version = "1.10.0", features = ["v4", "v7", "serde"] } +pin-project-lite = "0.2.14" +tower = { version = "0.4", features = ["tokio", "tracing"] } +tower-http = { version = "0.5.2", features = ["trace"] } +reqwest-retry = "0.6.1" +reqwest-middleware = { version = "0.3.3", features = ["charset", "http2", "json", "multipart", "rustls-tls"] } diff --git a/llama_proxy_man/README.md b/llama_proxy_man/README.md new file mode 100644 index 0000000..a3dd2f2 --- /dev/null +++ b/llama_proxy_man/README.md @@ -0,0 +1,5 @@ +# LLama Herder + +- manages multiple llama.cpp instances in the background +- keeps track of used & available video & cpu memory +- starts/stops llama.cpp instances as needed, to ensure memory limit is never reached diff --git a/llama_proxy_man/TODO.org b/llama_proxy_man/TODO.org new file mode 100644 index 0000000..d44d8d6 --- /dev/null +++ b/llama_proxy_man/TODO.org @@ -0,0 +1,34 @@ +#+title: Todo + + +* Name ideas +- llama herder +- llama herdsman/women/boy ?? +- llama shepherd ? + +* MVP +- [X] fix stopping (doesn't work correctly at all) +- seems done + +* Future Features +- [ ] support for model selection by name on a unified port for /api & /completions + - [ ] separation of proxy/selection stuff ? config for unmanaged instances for auto model-selection by name +- [ ] automatic internal port management (search for free ports) +- [ ] Diagnostic Overview UI/API +- [ ] Config UI/API ? +- [ ] better book-keeping abt inflight requests ? (needed ?) +- [ ] multi node stuff + - how exactly ? + - clustering ? (one manager per node ?) + - ssh support ??? +- [ ] automatic ram usage calc ? +- [ ] other runners + - e.g. docker/ run in path etc +- [ ] other backends ? +- [ ] more advanced start/stop behavior + - more config ? e.g. pinning/priorities/prefer-to-kill/start-initially + - LRU /most used prioritized to keep running + - speculative relaunch + - scheduling of how to order in-flight requests + restarts to handle them optimally +- [ ] advanced high-level foo + - automatic context-size selection per request/ start with bigger context if current instance has to low context diff --git a/llama_proxy_man/config.yaml b/llama_proxy_man/config.yaml new file mode 100644 index 0000000..667ed47 --- /dev/null +++ b/llama_proxy_man/config.yaml @@ -0,0 +1,26 @@ +hardware: + ram: 64G + vram: 8G +models: + - port: 18080 + internal_port: 28080 + env: + CUDA_VISIBLE_DEVICES: 0 + HSA_OVERRIDE_GFX_VERSION: '11.0.0' + args: + model: /home/tristand/Downloads/models/Phi-3.5-mini-instruct-Q6_K_L.gguf + gpu-layers: 9999 + ctx-size: 4096 + vram_usage: 6G + ram_usage: 500M + - port: 18081 + internal_port: 28081 + env: + CUDA_VISIBLE_DEVICES: 0 + HSA_OVERRIDE_GFX_VERSION: '11.0.0' + args: + model: /home/tristand/Downloads/models/Phi-3.5-mini-instruct-Q6_K_L.gguf + gpu-layers: 9999 + ctx-size: 4096 + vram_usage: 6G + ram_usage: 500M diff --git a/llama_proxy_man/src/audit.json b/llama_proxy_man/src/audit.json new file mode 100644 index 0000000..4e3bb61 --- /dev/null +++ b/llama_proxy_man/src/audit.json @@ -0,0 +1,16 @@ +{ + "keepSettings": { + "type": 1, + "amount": 30 + }, + "auditFilename": "audit.json", + "hashType": "md5", + "extension": ".log", + "files": [ + { + "date": 1726742235723, + "name": "/home/tristand/.tabby-client/agent/logs/20240919.0.log", + "hash": "13392f16f09c9e264d7c9b82880ae40c" + } + ] +} \ No newline at end of file diff --git a/llama_proxy_man/src/logging.rs b/llama_proxy_man/src/logging.rs new file mode 100644 index 0000000..33e45fc --- /dev/null +++ b/llama_proxy_man/src/logging.rs @@ -0,0 +1,83 @@ +use std::{ + future::Future, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use axum::{body::Body, http::Request}; +use pin_project_lite::pin_project; +use tower::{Layer, Service}; +use uuid::Uuid; // Make sure to include `uuid` crate in your Cargo.toml + +pub struct LoggingLayer; + +impl Layer for LoggingLayer { + type Service = LoggingService; + + fn layer(&self, inner: S) -> Self::Service { + LoggingService { inner } + } +} + +pub struct LoggingService { + inner: T, +} + +impl Service> for LoggingService +where + T: Service>, +{ + type Error = T::Error; + type Future = LoggingServiceFuture; + type Response = T::Response; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, req: Request) -> Self::Future { + let request_uuid = Uuid::now_v7(); // Generate UUID v7 + + tracing::info!("[{}] {} {}", request_uuid, req.method(), req.uri()); + for (k, v) in req.headers() { + tracing::info!( + "[{}] {}: {}", + request_uuid, + k, + v.to_str().unwrap_or("invalid header") + ); + } + + LoggingServiceFuture { + inner: self.inner.call(req), + uuid: Arc::new(request_uuid), // Store UUID in an Arc for shared ownership + } + } +} + +pin_project! { + pub struct LoggingServiceFuture { + #[pin] + inner: T, + uuid: Arc, // Shared state between LoggingService and LoggingServiceFuture + } +} + +impl Future for LoggingServiceFuture +where + T: Future, +{ + type Output = T::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + match this.inner.poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(output) => { + tracing::trace!("[{}] request finished", this.uuid); + Poll::Ready(output) + } + } + } +} diff --git a/llama_proxy_man/src/main.rs b/llama_proxy_man/src/main.rs new file mode 100644 index 0000000..8d6ce5c --- /dev/null +++ b/llama_proxy_man/src/main.rs @@ -0,0 +1,363 @@ +mod logging; + +use anyhow::anyhow; +use axum::{ + body::Body, + http::{self, Request, Response}, + response::IntoResponse, + routing::any, + Router, +}; +use futures; +use reqwest::Client; +use serde::Deserialize; +use std::{collections::HashMap, net::SocketAddr, process::Stdio, sync::Arc}; +use tokio::{ + net::TcpStream, + process::{Child, Command}, + sync::Mutex, + time::{sleep, Duration}, +}; +use tower_http::trace::{ + DefaultMakeSpan, DefaultOnEos, DefaultOnFailure, DefaultOnRequest, DefaultOnResponse, + TraceLayer, +}; +use tracing::Level; + +use std::sync::Once; + +pub static INIT: Once = Once::new(); + +pub fn initialize_logger() { + INIT.call_once(|| { + let env_filter = tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::INFO.into()) + .from_env_lossy(); + + tracing_subscriber::fmt() + .compact() + .with_env_filter(env_filter) + .init(); + }); +} + +#[derive(Debug, Deserialize)] +struct Config { + hardware: Hardware, + models: Vec, +} + +#[derive(Debug, Deserialize)] +struct Hardware { + ram: String, + vram: String, +} + +#[derive(Debug, Deserialize, Clone)] +struct ModelConfig { + port: u16, + internal_port: u16, + env: HashMap, + args: HashMap, + vram_usage: String, + ram_usage: String, +} + +#[derive(Clone)] +struct LlamaInstance { + config: ModelConfig, + process: Arc>, + // busy: bool, +} + +struct SharedState { + total_ram: u64, + total_vram: u64, + used_ram: u64, + used_vram: u64, + instances: HashMap, +} + +#[tokio::main] +async fn main() { + initialize_logger(); + // Read and parse the YAML configuration + let config_str = std::fs::read_to_string("config.yaml").expect("Failed to read config.yaml"); + let config: Config = serde_yaml::from_str(&config_str).expect("Failed to parse config.yaml"); + + // Parse hardware resources + let total_ram = parse_size(&config.hardware.ram).expect("Invalid RAM size in config"); + let total_vram = parse_size(&config.hardware.vram).expect("Invalid VRAM size in config"); + + // Initialize shared state + let shared_state = Arc::new(Mutex::new(SharedState { + total_ram, + total_vram, + used_ram: 0, + used_vram: 0, + instances: HashMap::new(), + })); + + // For each model, set up an axum server listening on the specified port + let mut handles = Vec::new(); + + for model_config in config.models { + let state = shared_state.clone(); + let model_config = model_config.clone(); + + let handle = tokio::spawn(async move { + let model_config = model_config.clone(); + + let app = Router::new() + .route( + "/", + any({ + let state = state.clone(); + let model_config = model_config.clone(); + move |req| handle_request(req, model_config, state) + }), + ) + .route( + "/*path", + any({ + let state = state.clone(); + let model_config = model_config.clone(); + move |req| handle_request(req, model_config, state) + }), + ) + .layer( + TraceLayer::new_for_http() + .make_span_with(DefaultMakeSpan::new().include_headers(true)) + .on_request(DefaultOnRequest::new().level(Level::DEBUG)) + .on_response(DefaultOnResponse::new().level(Level::TRACE)) + .on_eos(DefaultOnEos::new().level(Level::DEBUG)) + .on_failure(DefaultOnFailure::new().level(Level::ERROR)), + ); + + let addr = SocketAddr::from(([0, 0, 0, 0], model_config.port)); + + println!("Listening on port {}", model_config.port); + let listener = tokio::net::TcpListener::bind(&addr).await.unwrap(); + + axum::serve(listener, app.into_make_service()) + .await + .unwrap(); + }); + + handles.push(handle); + } + + futures::future::join_all(handles).await; +} + +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AppError { + #[error("Axum Error")] + AxumError(#[from] axum::Error), + #[error("Axum Http Error")] + AxumHttpError(#[from] axum::http::Error), + #[error("Reqwest Error")] + ReqwestError(#[from] reqwest::Error), + #[error("Reqwest Middleware Error")] + ReqwestMiddlewareError(#[from] reqwest_middleware::Error), + #[error("Client Error")] + ClientError(#[from] hyper::Error), + #[error("Unknown error")] + Unknown(#[from] anyhow::Error), +} + +// Tell axum how to convert `AppError` into a response. +impl IntoResponse for AppError { + fn into_response(self) -> axum::response::Response { + tracing::error!("::AppError::into_response:: {:?}", self); + ( + http::StatusCode::INTERNAL_SERVER_ERROR, + format!("Something went wrong: {:?}", self), + ) + .into_response() + } +} + +async fn handle_request( + req: Request, + model_config: ModelConfig, + state: Arc>, + // ) -> Result, anyhow::Error> { +) -> impl IntoResponse { + let model_ram_usage = parse_size(&model_config.ram_usage).expect("Invalid ram_usage"); + let model_vram_usage = parse_size(&model_config.vram_usage).expect("Invalid vram_usage"); + + let instance = { + let mut state = state.lock().await; + + // Check if instance is running + if let Some(instance) = state.instances.get_mut(&model_config.port) { + // instance.busy = true; + instance.to_owned() + } else { + // Check resources + if state.used_ram + model_ram_usage > state.total_ram + || state.used_vram + model_vram_usage > state.total_vram + { + // Stop other instances + let mut to_remove = Vec::new(); + for (port, instance) in state.instances.clone() { + // if !instance.busy { + tracing::info!("Stopping instance on port {}", port); + let mut process = instance.process.lock().await; + process.kill().await.ok(); + state.used_ram -= parse_size(&instance.config.ram_usage).unwrap_or(0); + state.used_vram -= parse_size(&instance.config.vram_usage).unwrap_or(0); + to_remove.push(port); + if state.used_ram + model_ram_usage <= state.total_ram + && state.used_vram + model_vram_usage <= state.total_vram + { + break; + } + // } + } + for port in to_remove { + tracing::info!("Removing instance on port {}", port); + state.instances.remove(&port); + } + } + + // Start new instance + let args = model_config + .args + .iter() + .flat_map(|(k, v)| vec![format!("--{}", k), v.clone()]) + .collect::>(); + + let mut cmd = Command::new("llama-server"); + cmd.envs(model_config.env.clone()); + cmd.args(&args); + cmd.arg("--port"); + cmd.arg(format!("{}", model_config.internal_port)); + cmd.stdout(Stdio::null()).stderr(Stdio::null()); + + tracing::info!("Starting llama-server with {:?}", cmd); + let process = Arc::new(Mutex::new( + cmd.spawn().expect("Failed to start llama-server"), + )); + + state.used_ram += model_ram_usage; + state.used_vram += model_vram_usage; + + let instance = LlamaInstance { + config: model_config.clone(), + process, + // busy: true, + }; + sleep(Duration::from_millis(500)).await; + state.instances.insert(model_config.port, instance.clone()); + + instance + } + }; + + // Wait for the instance to be ready + is_llama_instance_running(&instance).await?; + wait_for_port(model_config.internal_port).await?; + + // Proxy the request + let retry_policy = reqwest_retry::policies::ExponentialBackoff::builder() + .retry_bounds(Duration::from_secs(1), Duration::from_secs(8)) + .jitter(reqwest_retry::Jitter::None) + .base(2) + .build_with_max_retries(8); + + let client = reqwest_middleware::ClientBuilder::new(Client::new()) + .with(reqwest_retry::RetryTransientMiddleware::new_with_policy( + retry_policy, + )) + .build(); + + let uri = format!( + "http://127.0.0.1:{}{}", + model_config.internal_port, + req.uri().path_and_query().map(|x| x.as_str()).unwrap_or("") + ); + + let mut request_builder = client.request(req.method().clone(), &uri); + // let mut request_builder = reqwest::RequestBuilder::from_parts(client, req.method().clone()) + + for (name, value) in req.headers() { + request_builder = request_builder.header(name, value); + } + + let body_bytes = axum::body::to_bytes( + req.into_body(), + parse_size("1G").unwrap().try_into().unwrap(), + ) + .await?; + let request = request_builder.body(body_bytes).build()?; + // tracing::info!("Proxying request to {}", uri); + let response = client.execute(request).await?; + + // tracing::info!("Received response from {}", uri); + let mut builder = Response::builder().status(response.status()); + for (name, value) in response.headers() { + builder = builder.header(name, value); + } + + // let bytes = response.bytes().await?; + + let byte_stream = response.bytes_stream(); + + let body = axum::body::Body::from_stream(byte_stream); + + tracing::info!("streaming response on port: {}", model_config.port); + let response = builder.body(body)?; + Ok::, AppError>(response) +} + +async fn wait_for_port(port: u16) -> Result<(), anyhow::Error> { + for _ in 0..10 { + if TcpStream::connect(("127.0.0.1", port)).await.is_ok() { + return Ok(()); + } + sleep(Duration::from_millis(500)).await; + } + Err(anyhow!("Timeout waiting for port")) +} + +// reimplement wait_for_port using a LlamaInstance, also check if the process is still running +async fn is_llama_instance_running(instance: &LlamaInstance) -> Result<(), anyhow::Error> { + match instance.process.to_owned().lock_owned().await.try_wait()? { + Some(exit_status) => { + let msg = "Llama instance exited"; + tracing::error!(msg, ?exit_status); + Err(anyhow!(msg)) + } + + None => Ok(()), + } +} + +fn parse_size(size_str: &str) -> Option { + let mut num = String::new(); + let mut unit = String::new(); + + for c in size_str.chars() { + if c.is_digit(10) { + num.push(c); + } else { + unit.push(c); + } + } + + let num: f64 = num.parse().ok()?; + + let multiplier = match unit.to_lowercase().as_str() { + "g" | "gb" => 1024 * 1024 * 1024, + "m" | "mb" => 1024 * 1024, + "k" | "kb" => 1024, + _ => 1, + }; + + let res = (num * multiplier as f64) as u64; + Some(res) +}