LM/Qwen 3.8 27B

llama.cpp

• https://unsloth.ai/docs/basics/dynamic-3.0-ggufs

Running

# works
./llama.cpp/sycl/bin/llama-server \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M \
  --spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
  -c 64000 -ctk q8_0 -ctv q8_0 \
  -fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8

# with turboquant
./llama.cpp/vulkan/bin/llama-server \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M \
  --spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
  -c 64000 -ctk q8_0 -ctv turbo4 \
  -fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8

# assign draft model (cannot run)
./llama.cpp/sycl/bin/llama-server \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M \
  -hfd unsloth/Qwen3.8-27B-GGUF:MTP/mtp-Qwen3.8-27B-Q4_0.gguf \
  --spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
  -c 64000 -ctk q8_0 -ctv q8_0 \
  -fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8 -lv 5

Benchmarking

# control group
./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
  -fa on -ctk q8_0 -ctv q8_0 -ngl 99 -ub 2048

# Test Quantization
./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:Q4_0 \
  -fa on -ctk q8_0 -ctv q8_0 -ngl 99 -ub 2048

./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:Q8_0 \
  -fa on -ctk q8_0 -ctv q8_0 -ngl 99 -ub 2048

./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-IQ4_XS \
  -fa on -ctk q8_0 -ctv q8_0 -ngl 99 -ub 2048

./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL \
  -fa on -ctk q8_0 -ctv q8_0 -ngl 99 -ub 2048

./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_S \
  -fa on -ctk q8_0 -ctv q8_0 -ngl 99 -ub 2048

./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M \
  -fa on -ctk q8_0 -ctv q8_0 -ngl 99 -ub 2048

# Test -ctv
./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
  -fa on -ctk q8_0 -ctv q4_0 -ngl 99 -ub 2048

# Test -ub
./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
  -fa on -ctk q8_0 -ctv q8_0 -ngl 99 -ub 1024

# Test -fa off
./llama.cpp/sycl/bin/llama-bench \
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
  -fa off -ngl 99 -ub 2048

| model                          |       size |     params | backend    | ngl | type_k | type_v |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q4_K - Medium       |  16.34 GiB |    27.32 B | SYCL       |  99 |   q8_0 |   q8_0 |   1 |           pp512 |        788.86 ± 2.85 |
| qwen35 27B Q4_K - Medium       |  16.34 GiB |    27.32 B | SYCL       |  99 |   q8_0 |   q8_0 |   1 |          pp1024 |        805.80 ± 1.28 |
| qwen35 27B Q4_K - Medium       |  16.34 GiB |    27.32 B | SYCL       |  99 |   q8_0 |   q8_0 |   1 |           tg512 |         20.53 ± 0.07 |

build: cd26896c1 (10553)

| model                          |       size |     params | backend    | ngl | type_k | type_v |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Medium       |  19.43 GiB |    27.32 B | SYCL       |  99 |   q8_0 |   q8_0 |   1 |           pp512 |        949.64 ± 4.58 |
| qwen35 27B Q5_K - Medium       |  19.43 GiB |    27.32 B | SYCL       |  99 |   q8_0 |   q8_0 |   1 |          pp1024 |        970.65 ± 3.16 |
| qwen35 27B Q5_K - Medium       |  19.43 GiB |    27.32 B | SYCL       |  99 |   q8_0 |   q8_0 |   1 |           tg512 |         21.41 ± 0.06 |

build: cd26896c1 (10553)

| model                          |       size |     params | backend    | ngl | type_k | type_v |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q6_K                |  21.49 GiB |    27.32 B | SYCL       |  99 |   q8_0 |   q8_0 |   1 |           pp512 |        950.58 ± 3.76 |
| qwen35 27B Q6_K                |  21.49 GiB |    27.32 B | SYCL       |  99 |   q8_0 |   q8_0 |   1 |          pp1024 |        965.97 ± 0.65 |
| qwen35 27B Q6_K                |  21.49 GiB |    27.32 B | SYCL       |  99 |   q8_0 |   q8_0 |   1 |           tg512 |         19.75 ± 0.03 |

build: cd26896c1 (10553)

Experiment

hf models ls --search "Qwen3.8-27B" --apps llama.cpp --expand "downloads,likes,lastModified" --sort downloads --no-truncate --limit 10

hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF

hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-IQ4_XS*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K.gguf"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_M*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q4_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q8_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf"

tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF

Accuracy

https://unsloth.ai/docs/~gitbook/image?url=https%3A%2F%2F3215535692-files.gitbook.io%2F%7E%2Ffiles%2Fv0%2Fb%2Fgitbook-x-prod.appspot.com%2Fo%2Fspaces%252FxhOjnexMCB3dmuQFQ2Zq%252Fuploads%252Fl9piThTmmUsePst5w3F2%252Fimage.png%3Falt%3Dmedia%26token%3D821e88e2-ad4e-40bf-adf2-e13837228e82&width=768&dpr=3&quality=100&sign=dd5f7cb5&sv=2