diff --git a/llama_proxy_man/src/main.rs b/llama_proxy_man/src/main.rs index 353502b..53d0d74 100644 --- a/llama_proxy_man/src/main.rs +++ b/llama_proxy_man/src/main.rs @@ -84,6 +84,10 @@ struct SharedState { #[tokio::main] async fn main() { + // TODO add autostart of models based on config + // abstract starting logic out of handler for this to allow seperate calls to start + // maybe add to SharedState & LLamaInstance ? + initialize_logger(); // Read and parse the YAML configuration let config_str = std::fs::read_to_string("config.yaml").expect("Failed to read config.yaml"); @@ -264,9 +268,10 @@ async fn handle_request( cmd.kill_on_drop(true); cmd.envs(model_config.env.clone()); cmd.args(&args); + // TODO use openport crate via pick_random_unused_port for determining these cmd.arg("--port"); cmd.arg(format!("{}", model_config.internal_port)); - cmd.stdout(Stdio::null()).stderr(Stdio::null()); + cmd.stdout(Stdio::null()).stderr(Stdio::null()); // TODO save output and allow retrieval via api tracing::info!("Starting llama-server with {:?}", cmd); let process = Arc::new(Mutex::new(