LM/llama-swap

Config

apiKeys:
  - "sk-invalid-api-key"

# Optional: Global timeout settings
healthCheckTimeout: 120  # Max seconds to wait for model server to boot up
logLevel: info

# Allow both models to run simultaneously without auto-unloading each other
groups:
  parallel:
    swap: false
    members:
      - gemma4-e4b
      - gemma4-12b

# Run warmup requests automatically on llama-swap startup
hooks:
  on_startup:
    preload:
      - gemma4-e4b
      - gemma4-12b

# Map model names to their host execution commands
models:
  gemma4-e4b:
    cmd: >
      taskset -c 0,2,4 ./llama.cpp/sycl/bin/llama-server
      -hf google/gemma-4-E4B-it-qat-q4_0-gguf
      -t 3 -tb 3 --mlock -c 64000 -ngl 99
      --flash-attn on --cache-type-k q4_0 --cache-type-v q4_0
      --temp 0.3
      --port ${PORT}
    ttl: 0

  gemma4-12b:
    # Use multi-line strings for readable shell commands
    cmd: >
      taskset -c 6,8,10 ./llama.cpp/sycl/bin/llama-server
      -hf google/gemma-4-12B-it-qat-q4_0-gguf
      -t 3 -tb 3 --mlock -c 64000 -ngl 99
      --flash-attn on --cache-type-k q4_0 --cache-type-v q4_0
      --temp 0.3
      --port ${PORT}
    ttl: 0

Run

llama-swap --config llama-swap.yaml --listen 0.0.0.0:9000

Install

wget https://github.com/mostlygeek/llama-swap/releases/download/v240/llama-swap_240_linux_amd64.tar.gz
tar -zxv -C ~/.local/bin -f llama-swap_240_linux_amd64.tar.gz
nano config.yaml