LM/llama-swap - Revision history

Tacoball: /* Config */

2026-09-09T02:37:19Z

Config

← Older revision		Revision as of 02:37, 9 September 2026
Line 1:		Line 1:
	== Config ==		== Config ==

			https://github.com/mostlygeek/llama-swap/blob/main/docs/config.example.yaml

	<syntaxhighlight lang="yaml">		<syntaxhighlight lang="yaml">

Tacoball: /* systemd */

2026-08-31T02:16:08Z

systemd

@@ Line 98: / Line 98: @@
 [Install]
 WantedBy=multi-user.target
 </syntaxhighlight>

Tacoball at 02:10, 31 August 2026

2026-08-31T02:10:39Z

@@ Line 75: / Line 75: @@
    #      --port ${PORT}
    #  ttl: 0
 </syntaxhighlight>

Tacoball: /* Config */

2026-08-21T10:34:22Z

Config

← Older revision		Revision as of 10:34, 21 August 2026
Line 48:		Line 48:
	./llama.cpp/sycl/bin/llama-server		./llama.cpp/sycl/bin/llama-server
	-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M		-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M
	--spec-type draft-mtp --spec-draft-n-max 4 -ngld 99		--spec-type draft-mtp --spec-draft-n-max 3 -ngld 99
	-c 64000 -ctk q8_0 -ctv q8_0		-c 64000 -ctk q8_0 -ctv q8_0
	-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 6		-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 6

Tacoball: /* Install */

2026-08-21T10:18:40Z

Install

← Older revision		Revision as of 10:18, 21 August 2026
Line 88:		Line 88:
	wget https://github.com/mostlygeek/llama-swap/releases/download/v240/llama-swap_240_linux_amd64.tar.gz		wget https://github.com/mostlygeek/llama-swap/releases/download/v240/llama-swap_240_linux_amd64.tar.gz
	tar -zxv -C ~/.local/bin -f llama-swap_240_linux_amd64.tar.gz		tar -zxv -C ~/.local/bin -f llama-swap_240_linux_amd64.tar.gz
	nano ~~llama-swap~~.yaml		nano lmsw.yaml
	</syntaxhighlight>		</syntaxhighlight>

Tacoball: /* Run */

2026-08-21T10:18:20Z

Run

← Older revision		Revision as of 10:18, 21 August 2026
Line 80:		Line 80:

	<syntaxhighlight lang="bash">		<syntaxhighlight lang="bash">
	llama-swap --config ~~llama-swap~~.yaml --listen 0.0.0.0:9000		llama-swap --config lmsw.yaml --listen 0.0.0.0:9000
	</syntaxhighlight>		</syntaxhighlight>

Tacoball: /* Config */

2026-08-21T10:17:48Z

Config

← Older revision		Revision as of 10:17, 21 August 2026
Line 14:		Line 14:
	swap: false		swap: false
	members:		members:
	- ~~gemma4-e4b~~		- autocomplete
	- ~~gemma4-12b~~		- develop

	# Run warmup requests automatically on llama-swap startup		# Run warmup requests automatically on llama-swap startup
Line 21:		Line 21:
	on_startup:		on_startup:
	preload:		preload:
	- ~~gemma4-e4b~~		- autocomplete
	- ~~gemma4-12b~~		- develop

	# Map model names to their host execution commands		# Map model names to their host execution commands
	models:		models:
	~~gemma4-e4b~~:		autocomplete:
	cmd: >		cmd: >
	~~taskset -c 0,2,4~~ ./llama.cpp/sycl/bin/llama-server		./llama.cpp/sycl/bin/llama-server
	-hf ~~google~~/~~gemma-4-E4B~~-it-~~qat~~-~~q4_0~~-~~gguf~~		-hf Qwen/Qwen2.5-Coder-3B-Instruct-GGUF:q5_k_m
	-~~t 3~~ -~~tb 3~~ --~~mlock -c 64000~~ -ngl 99		-c 4000 -ctk q8_0 -ctv q8_0
	-~~-flash-attn on --cache-type-k q4_0 --cache-type-v q4_0~~		-fa on -ngl 99 -t 4
	~~--temp 0.3~~		--port ${PORT}
	--port ${PORT}
	ttl: 0		ttl: 0

	~~gemma4~~-~~12b~~:		# 3.4G VRAM / starcode2 / Q8 C2048
	# ~~Use multi~~-~~line strings for readable shell commands~~		#autocomplete:
	cmd: >		# cmd: >
	~~taskset -c 6,8,10~~ ./llama.cpp/sycl/bin/llama-server		# ./llama.cpp/sycl/bin/llama-server
	-hf ~~google~~/~~gemma~~-4-~~12B~~-it-~~qat~~-~~q4_0~~-~~gguf~~		# -hf DevQuasar-9/bigcode.starcoder2-3b-GGUF:Q8_0
	-~~t 3~~ -~~tb 3~~ --~~mlock~~ -c 64000 -ngl 99		# -c 2048 -fa on -ngl 99 --load-mode mlock -t 4
	--~~flash~~-~~attn on~~ --cache-type-k ~~q4_0~~ --cache-type-v ~~q4_0~~		# --port ${PORT}
	--temp 0.3
	--port ${PORT}		# Qwen 3.8 27B
	ttl: 0		develop:
			cmd: >
			./llama.cpp/sycl/bin/llama-server
			-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M
			--spec-type draft-mtp --spec-draft-n-max 4 -ngld 99
			-c 64000 -ctk q8_0 -ctv q8_0
			-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 6
			--port ${PORT}
			ttl: 0

			# Devstral Small 24B
			# develop:
			# cmd: >
			# ./llama.cpp/sycl/bin/llama-server
			# -hf unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF:UD-Q4_K_XL
			# -c 64000 --cache-type-k q8_0 --cache-type-v q8_0
			# --jinja --temp 0.15 --min-p 0.01
			# -fa on -ngl 99 --mlock -t 6
			# --port ${PORT}
			# ttl: 0

			# Muse Glimmer 30B
			#develop:
			# cmd: >
			# ./llama.cpp/sycl/bin/llama-server
			# -hf unsloth/Muse-Glimmer-30B-GGUF:UD-Q5_K_XL
			# --spec-type draft-dflash --spec-draft-n-max 4 -ngld 99
			# -c 64000 --cache-type-k q8_0 --cache-type-v q8_0
			# --jinja -fa on -ngl 99 --load-mode mlock --reasoning-preserve -t 6
			# --port ${PORT}
			# ttl: 0
	</syntaxhighlight>		</syntaxhighlight>

Tacoball: /* Install */

2026-08-13T17:44:40Z

Install

← Older revision		Revision as of 17:44, 13 August 2026
Line 59:		Line 59:
	wget https://github.com/mostlygeek/llama-swap/releases/download/v240/llama-swap_240_linux_amd64.tar.gz		wget https://github.com/mostlygeek/llama-swap/releases/download/v240/llama-swap_240_linux_amd64.tar.gz
	tar -zxv -C ~/.local/bin -f llama-swap_240_linux_amd64.tar.gz		tar -zxv -C ~/.local/bin -f llama-swap_240_linux_amd64.tar.gz
	nano ~~config~~.yaml		nano llama-swap.yaml
	</syntaxhighlight>		</syntaxhighlight>

Tacoball: /* Run */

2026-08-13T17:44:10Z

Run

← Older revision		Revision as of 17:44, 13 August 2026
Line 51:		Line 51:

	<syntaxhighlight lang="bash">		<syntaxhighlight lang="bash">
	llama-swap --config ~~config~~.yaml --listen 0.0.0.0:9000		llama-swap --config llama-swap.yaml --listen 0.0.0.0:9000
	</syntaxhighlight>		</syntaxhighlight>

Tacoball: /* Config */

2026-08-13T15:11:16Z

Config

← Older revision		Revision as of 15:11, 13 August 2026
Line 3:		Line 3:
	<syntaxhighlight lang="yaml">		<syntaxhighlight lang="yaml">
	apiKeys:		apiKeys:
	- "sk-no-~~such~~-key"		- "sk-invalid-api-key"

	# Optional: Global timeout settings		# Optional: Global timeout settings