LM/Qwen 3.8 27B: Difference between revisions
< LM
Jump to navigation
Jump to search
No edit summary |
No edit summary |
||
| Line 1: | Line 1: | ||
== | == llama.cpp == | ||
=== Running === | |||
<syntaxhighlight lang="bash"> | <syntaxhighlight lang="bash"> | ||
| Line 7: | Line 9: | ||
-c 64000 --cache-type-k q8_0 --cache-type-v q8_0 \ | -c 64000 --cache-type-k q8_0 --cache-type-v q8_0 \ | ||
-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8 | -fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8 | ||
</syntaxhighlight> | |||
=== Benchmarking === | |||
<syntaxhighlight lang="bash"> | |||
./llama.cpp/sycl/bin/llama-bench \ | |||
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL \ | |||
--spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \ | |||
-c 64000 --cache-type-k q8_0 --cache-type-v q8_0 \ | |||
-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8 \ | |||
-p 512,1024 -n 512 | |||
</syntaxhighlight> | </syntaxhighlight> | ||
Revision as of 02:20, 17 August 2026
llama.cpp
Running
./llama.cpp/sycl/bin/llama-server \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL \
--spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
-c 64000 --cache-type-k q8_0 --cache-type-v q8_0 \
-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8
Benchmarking
./llama.cpp/sycl/bin/llama-bench \
-hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL \
--spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
-c 64000 --cache-type-k q8_0 --cache-type-v q8_0 \
-fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8 \
-p 512,1024 -n 512
Experiment
hf models ls --search "Qwen3.8-27B" --apps llama.cpp --expand "downloads,likes,lastModified" --sort downloads --no-truncate --limit 10
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF