LM/llama-swap

Config

https://github.com/mostlygeek/llama-swap/blob/main/docs/config.example.yaml

apiKeys:
  - "sk-invalid-api-key"

# Optional: Global timeout settings
healthCheckTimeout: 120  # Max seconds to wait for model server to boot up
logLevel: info

# Allow both models to run simultaneously without auto-unloading each other
groups:
  parallel:
    swap: false
    members:
      - autocomplete
      - develop

# Run warmup requests automatically on llama-swap startup
hooks:
  on_startup:
    preload:
      - autocomplete
      - develop

# Map model names to their host execution commands
models:
  autocomplete:
    cmd: >
      ./llama.cpp/sycl/bin/llama-server
        -hf Qwen/Qwen2.5-Coder-3B-Instruct-GGUF:q5_k_m
        -c 4000 -ctk q8_0 -ctv q8_0
        -fa on -ngl 99 -t 4
        --port ${PORT}
    ttl: 0

  # 3.4G VRAM / starcode2 / Q8 C2048
  #autocomplete:
  #  cmd: >
  #    ./llama.cpp/sycl/bin/llama-server
  #      -hf DevQuasar-9/bigcode.starcoder2-3b-GGUF:Q8_0
  #      -c 2048 -fa on -ngl 99 --load-mode mlock -t 4
  #      --port ${PORT}

  # Qwen 3.8 27B
  develop:
   cmd: >
     ./llama.cpp/sycl/bin/llama-server
       -hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M
       --spec-type draft-mtp --spec-draft-n-max 3 -ngld 99
       -c 64000 -ctk q8_0 -ctv q8_0
       -fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 6
       --port ${PORT}
   ttl: 0

  # Devstral Small 24B
  # develop:
  #   cmd: >
  #     ./llama.cpp/sycl/bin/llama-server
  #       -hf unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF:UD-Q4_K_XL
  #       -c 64000 --cache-type-k q8_0 --cache-type-v q8_0
  #       --jinja --temp 0.15 --min-p 0.01
  #       -fa on -ngl 99 --mlock -t 6
  #       --port ${PORT}
  #   ttl: 0

  # Muse Glimmer 30B
  #develop:
  #  cmd: >
  #    ./llama.cpp/sycl/bin/llama-server
  #      -hf unsloth/Muse-Glimmer-30B-GGUF:UD-Q5_K_XL
  #      --spec-type draft-dflash --spec-draft-n-max 4 -ngld 99
  #      -c 64000 --cache-type-k q8_0 --cache-type-v q8_0
  #      --jinja -fa on -ngl 99 --load-mode mlock --reasoning-preserve -t 6
  #      --port ${PORT}
  #  ttl: 0

systemd

sudo nano /etc/systemd/system/llama-swap.service

[Unit]
Description=llama-swap Service
After=network.target

[Service]
Type=simple
User=your_username
WorkingDirectory=/path/to/llama-swap
ExecStart=/path/to/llama-swap --config /path/to/config.yaml
Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target

# 重新載入 systemd 設定
sudo systemctl daemon-reload

# 設定開機自動啟動並立即執行
sudo systemctl enable --now llama-swap

# 檢查執行狀態
sudo systemctl status llama-swap

Run

llama-swap --config lmsw.yaml --listen 0.0.0.0:9000

Install

wget https://github.com/mostlygeek/llama-swap/releases/download/v240/llama-swap_240_linux_amd64.tar.gz
tar -zxv -C ~/.local/bin -f llama-swap_240_linux_amd64.tar.gz
nano lmsw.yaml