vault81/redvault-ai
4
0
Fork 0
Code Issues 1 Pull requests 1 Projects 1 Releases Packages Wiki Activity

Fmt

Browse source
This commit is contained in:
Tristan D. 2025-01-31 18:45:19 +01:00
parent 628e5e63fb
commit 98b4f45706
Signed by: tristan
SSH key fingerprint: SHA256:9oFM1J63hYWJjCnLG6C0fxBS15rwNcWwdQNMOHYKJ/4
1 changed files with 18 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
llama_proxy_man/src/main.rs
View file

@ -107,7 +107,9 @@ async fn main() {
initialize_logger();
// Read and parse the YAML configuration
let config_str = std::fs::read_to_string("config.yaml").expect("Failed to read config.yaml");
let config: Config = serde_yaml::from_str::<Config>(&config_str).expect("Failed to parse config.yaml").pick_open_ports();
let config: Config = serde_yaml::from_str::<Config>(&config_str)
.expect("Failed to parse config.yaml")
.pick_open_ports();
// Parse hardware resources
let total_ram = parse_size(&config.hardware.ram).expect("Invalid RAM size in config");
@ -286,7 +288,12 @@ async fn handle_request(
cmd.args(&args);
// TODO use openport crate via pick_random_unused_port for determining these
cmd.arg("--port");
cmd.arg(format!("{}", model_config.internal_port.expect("Unexpected empty port, should've been picked")));
cmd.arg(format!(
"{}",
model_config
.internal_port
.expect("Unexpected empty port, should've been picked")
));
cmd.stdout(Stdio::null()).stderr(Stdio::null()); // TODO save output and allow retrieval via api
tracing::info!("Starting llama-server with {:?}", cmd);
@ -311,7 +318,12 @@ async fn handle_request(
// Wait for the instance to be ready
is_llama_instance_running(&instance).await?;
wait_for_port(model_config.internal_port.expect("Unexpected empty port, should've been picked")).await?;
wait_for_port(
model_config
.internal_port
.expect("Unexpected empty port, should've been picked"),
)
.await?;
// Proxy the request
let retry_policy = reqwest_retry::policies::ExponentialBackoff::builder()
@ -328,7 +340,9 @@ async fn handle_request(
let uri = format!(
"http://127.0.0.1:{}{}",
model_config.internal_port.expect("Unexpected empty port, should've been picked"),
model_config
.internal_port
.expect("Unexpected empty port, should've been picked"),
req.uri().path_and_query().map(|x| x.as_str()).unwrap_or("")
);