hardware:
  ram: 64G
  vram: 8G
models:
  - port: 18080
    internal_port: 28080
    env:
      CUDA_VISIBLE_DEVICES: 0
      HSA_OVERRIDE_GFX_VERSION: '11.0.0'
    args:
      model: /home/tristand/Downloads/models/Phi-3.5-mini-instruct-Q6_K_L.gguf
      gpu-layers: 9999
      ctx-size: 4096
    vram_usage: 6G
    ram_usage: 500M
  - port: 18081
    internal_port: 28081
    env:
      CUDA_VISIBLE_DEVICES: 0
      HSA_OVERRIDE_GFX_VERSION: '11.0.0'
    args:
      model: /home/tristand/Downloads/models/Phi-3.5-mini-instruct-Q6_K_L.gguf
      gpu-layers: 9999
      ctx-size: 4096
    vram_usage: 6G
    ram_usage: 500M