LM/Qwen 3.8 27B: Difference between revisions
< LM
Jump to navigation
Jump to search
| Line 67: | Line 67: | ||
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF | hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF | ||
./llama download -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL | ./llama download -mtp --mmproj-auto -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL | ||
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*" | hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*" | ||
Revision as of 03:07, 20 August 2026
llama.cpp
Running
./llama.cpp/sycl/bin/llama-server \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL \
--spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
-c 64000 --ctk q8_0 --ctv q8_0 \
-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8
Benchmarking
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL \
-ctk q8_0 -ctv q8_0 -fa on -ngl 99 \
-p 512,1024 -n 512
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
-ctk q8_0 -ctv q8_0 -fa on -ngl 99 \
-p 512,1024 -n 512
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_XL \
-ctk q8_0 -ctv q8_0 -fa on -ngl 99 \
-p 512,1024 -n 512
$ ./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -p 512,1024 -n 512 | model | size | params | backend | ngl | type_k | type_v | fa | test | t/s | | ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: | | qwen35 27B Q4_K - Small | 16.68 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | pp512 | 769.54 ± 4.22 | | qwen35 27B Q4_K - Small | 16.68 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | pp1024 | 777.55 ± 1.27 | | qwen35 27B Q4_K - Small | 16.68 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | tg512 | 21.06 ± 0.00 | build: 8e7f22b67 (10398) $ ./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -p 512,1024 -n 512 | model | size | params | backend | ngl | type_k | type_v | fa | test | t/s | | ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: | | qwen35 27B Q4_K - Small | 18.82 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | pp512 | 891.35 ± 1.42 | | qwen35 27B Q4_K - Small | 18.82 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | pp1024 | 885.81 ± 0.79 | | qwen35 27B Q4_K - Small | 18.82 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | tg512 | 21.98 ± 0.03 | build: 8e7f22b67 (10398) $ ./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_XL -ctk q8_0 -ctv q8_0 -fa on -ngl 99 -p 512,1024 -n 512 | model | size | params | backend | ngl | type_k | type_v | fa | test | t/s | | ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: | | qwen35 27B Q4_K - Small | 24.13 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | pp512 | 806.99 ± 6.07 | | qwen35 27B Q4_K - Small | 24.13 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | pp1024 | 841.24 ± 3.10 | | qwen35 27B Q4_K - Small | 24.13 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | tg512 | 17.24 ± 0.02 | build: 8e7f22b67 (10398)
Experiment
hf models ls --search "Qwen3.8-27B" --apps llama.cpp --expand "downloads,likes,lastModified" --sort downloads --no-truncate --limit 10
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF
./llama download -mtp --mmproj-auto -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_M*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mtp-Qwen3.8-27B-Q4_0.gguf"
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF
