vault81/redvault-ai
4
0
Fork 0
Code Issues 1 Pull requests 1 Projects 1 Releases Packages Wiki Activity

Add TODOs

Browse source
This commit is contained in:
Tristan D. 2024-11-26 18:28:49 +01:00
parent 89dec115b7
commit 3b4655728d
Signed by: tristan
SSH key fingerprint: SHA256:3RU4RLOoM8oAjFU19f1W6t8uouZbA7GWkaSW6rjp1k8
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
llama_proxy_man/src/main.rs
View file

@ -84,6 +84,10 @@ struct SharedState {
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
// TODO add autostart of models based on config
// abstract starting logic out of handler for this to allow seperate calls to start
// maybe add to SharedState & LLamaInstance ?
initialize_logger(); initialize_logger();
// Read and parse the YAML configuration // Read and parse the YAML configuration
let config_str = std::fs::read_to_string("config.yaml").expect("Failed to read config.yaml"); let config_str = std::fs::read_to_string("config.yaml").expect("Failed to read config.yaml");
@ -264,9 +268,10 @@ async fn handle_request(
cmd.kill_on_drop(true); cmd.kill_on_drop(true);
cmd.envs(model_config.env.clone()); cmd.envs(model_config.env.clone());
cmd.args(&args); cmd.args(&args);
// TODO use openport crate via pick_random_unused_port for determining these
cmd.arg("--port"); cmd.arg("--port");
cmd.arg(format!("{}", model_config.internal_port)); cmd.arg(format!("{}", model_config.internal_port));
cmd.stdout(Stdio::null()).stderr(Stdio::null()); cmd.stdout(Stdio::null()).stderr(Stdio::null()); // TODO save output and allow retrieval via api
tracing::info!("Starting llama-server with {:?}", cmd); tracing::info!("Starting llama-server with {:?}", cmd);
let process = Arc::new(Mutex::new( let process = Arc::new(Mutex::new(