26 lines
626 B
YAML
26 lines
626 B
YAML
hardware:
|
|
ram: 64G
|
|
vram: 8G
|
|
models:
|
|
- port: 18080
|
|
internal_port: 28080
|
|
env:
|
|
CUDA_VISIBLE_DEVICES: 0
|
|
HSA_OVERRIDE_GFX_VERSION: '11.0.0'
|
|
args:
|
|
model: /home/tristand/Downloads/models/Phi-3.5-mini-instruct-Q6_K_L.gguf
|
|
gpu-layers: 9999
|
|
ctx-size: 4096
|
|
vram_usage: 6G
|
|
ram_usage: 500M
|
|
- port: 18081
|
|
internal_port: 28081
|
|
env:
|
|
CUDA_VISIBLE_DEVICES: 0
|
|
HSA_OVERRIDE_GFX_VERSION: '11.0.0'
|
|
args:
|
|
model: /home/tristand/Downloads/models/Phi-3.5-mini-instruct-Q6_K_L.gguf
|
|
gpu-layers: 9999
|
|
ctx-size: 4096
|
|
vram_usage: 6G
|
|
ram_usage: 500M
|