LM/Qwen 3.8 27B: Difference between revisions
< LM
Jump to navigation
Jump to search
| Line 190: | Line 190: | ||
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*" | hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*" | ||
hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf" | hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf" | ||
hf models ls --tree -R -h bartowski/Qwen3.8-27B-GGUF | |||
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q4_0*" | |||
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q4_1*" | |||
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q4_K_L*" | |||
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q5_K_L*" | |||
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q6_K_L*" | |||
hf download bartowski/Qwen3.8-27B-GGUF --include "*mmproj-Qwen3.8-27B-bf16.gguf*" | |||
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf" | find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf" | ||
Revision as of 03:58, 27 August 2026
llama.cpp
Running
# works
./llama.cpp/sycl/bin/llama-server \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M \
--spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
-c 64000 -ctk q8_0 -ctv q8_0 \
-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8
# with turboquant
./llama.cpp/vulkan/bin/llama-server \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M \
--spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
-c 64000 -ctk q8_0 -ctv turbo4 \
-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8
# assign draft model (cannot run)
./llama.cpp/sycl/bin/llama-server \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M \
-hfd unsloth/Qwen3.8-27B-GGUF:MTP/mtp-Qwen3.8-27B-Q4_0.gguf \
--spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
-c 64000 -ctk q8_0 -ctv q8_0 \
-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8 -lv 5
Benchmarking
# control group
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Medium | 19.43 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | pp512 | 951.05 ± 7.72 |
| qwen35 27B Q5_K - Medium | 19.43 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | tg128 | 22.17 ± 0.06 |
# Test Quantization
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:Q4_0 \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q4_0 | 14.94 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | pp512 | 798.17 ± 1.28 |
| qwen35 27B Q4_0 | 14.94 GiB | 27.32 B | SYCL | 99 | q8_0 | q8_0 | 1 | tg128 | 27.30 ± 0.05 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:Q8_0 \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q8_0 | 27.04 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 817.25 ± 15.79 |
| qwen35 27B Q8_0 | 27.04 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 15.48 ± 0.01 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-IQ4_XS \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B IQ4_XS - 4.25 bpw | 13.26 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 573.51 ± 0.58 |
| qwen35 27B IQ4_XS - 4.25 bpw | 13.26 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 16.37 ± 0.05 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_S \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q4_K - Small | 14.29 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 644.94 ± 1.00 |
| qwen35 27B Q4_K - Small | 14.29 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 19.09 ± 0.08 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q4_K - Medium | 16.34 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 793.27 ± 0.74 |
| qwen35 27B Q4_K - Medium | 16.34 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 21.37 ± 0.03 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_S \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Small | 17.37 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 880.03 ± 1.55 |
| qwen35 27B Q5_K - Small | 17.37 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 22.35 ± 0.05 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_M \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Medium | 18.40 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 927.22 ± 0.90 |
| qwen35 27B Q5_K - Medium | 18.40 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 22.43 ± 0.05 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q6_K | 20.46 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 945.67 ± 4.57 |
| qwen35 27B Q6_K | 20.46 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 21.06 ± 0.05 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_M \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q6_K | 21.49 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 953.90 ± 5.70 |
| qwen35 27B Q6_K | 21.49 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 20.14 ± 0.04 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_L \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q6_K | 22.52 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 950.43 ± 5.17 |
| qwen35 27B Q6_K | 22.52 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 19.30 ± 0.03 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_XL \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q6_K | 23.55 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 923.94 ± 1.96 |
| qwen35 27B Q6_K | 23.55 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 18.27 ± 0.02 |
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_L \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99
| model | size | params | backend | ngl | n_ubatch | type_k | type_v | fa | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q8_0 | 26.11 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | pp512 | 836.06 ± 2.05 |
| qwen35 27B Q8_0 | 26.11 GiB | 27.32 B | SYCL | 99 | 2048 | q8_0 | q8_0 | 1 | tg128 | 16.28 ± 0.01 |
# Test -ctv
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
-fa on -ctk q8_0 -ctv q4_0 -ngl 99
# Test -ub
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
-fa on -ctk q8_0 -ctv q8_0 -ngl 99 -ub 2048
# Test -fa off
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
-fa off -ngl 99
# Test OpenVINO
GGML_OPENVINO_DEVICE=GPU GGML_OPENVINO_STATEFUL_EXECUTION=1 ./llama.cpp/openvino/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:Q4_0 \
-fa on -ngl 99
Experiment
hf models ls --search "Qwen3.8-27B" --apps llama.cpp --expand "downloads,likes,lastModified" --sort downloads --no-truncate --limit 10
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-IQ4_XS*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K.gguf"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_M*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_L*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q8_K_L*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q4_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q8_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf"
hf models ls --tree -R -h bartowski/Qwen3.8-27B-GGUF
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q4_0*"
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q4_1*"
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q4_K_L*"
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q5_K_L*"
hf download bartowski/Qwen3.8-27B-GGUF --include "*Q6_K_L*"
hf download bartowski/Qwen3.8-27B-GGUF --include "*mmproj-Qwen3.8-27B-bf16.gguf*"
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf"
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf" -exec sh -c 'rm -f "$1" "$(readlink -f "$1")"' _ {} \;
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF
