LM/Qwen 3.8 27B: Difference between revisions

From Fundamental Ramen
< LM
Jump to navigation Jump to search
← Older edit
VisualWikitext
 
(252 intermediate revisions by the same user not shown)
Line 1: Line 1:
== llama.cpp ==
== vLLM ==
 
=== Best-Fit Models for B70 ===
 
The B70 delivers the best performance with W4A8 quantized models.
 
The underlying rationale can be referenced from TechPowerUp’s spec data, and the listed models show solid performance.
 
{| class="wikitable" style="text-align:center;"
! rowspan="2" | Model
! rowspan="2" | Last-Modified
! rowspan="2" | W
! rowspan="2" | A
! colspan="2" | TP2 / C1
! colspan="2" | TP2 / C2
! colspan="2" | TP2 / C4
! colspan="2" | TP2 / C8
! colspan="2" | Single
|-
! PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128
|-
| style="font-size:10pt; text-align:left;" | cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-BF16-INT4
| 2026-10-19 || 4 || 16
| 1737 || 65 || 2087 || 97 || 2121 || 139 || 2664 || 240
| ||
|-
| style="font-size:10pt; text-align:left;" | cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-INT4
| 2026-10-19 || 4 || 16
| 1908 || 79 || 2062 || 115 || 2196 || 158 || 2632 || 269
| ||
|-
| style="font-size:10pt; text-align:left;" | Launch80/Swift-1.5-Qwen3.8-27B-Uncensored-Int4-baked
| 2026-10-13 || 4 || 16
| 1899 || 102 || 2084 || 165 || 2595 || 241 || 2758 || 307
| ||
|-
| style="font-size:10pt; text-align:left;" | davetha/Huihui-Qwen3.8-27B-abliterated-AutoRound-Int4-baked-embed-int8
| 2026-10-09 || 4 || 16
| 2026 || 118 || 2411 || 164 || 2801 || 266 || 2785 || 319
| ||
|-
| style="font-size:10pt; text-align:left;" | 🔥 Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 🔥
| 2026-10-01 || 4 || 16
| 2335 || 110 || 2429 || 190 || 2822 || 286 || 2928 || 506
| ||
|}
 
=== Available Requests ===


* https://unsloth.ai/docs/basics/dynamic-3.0-ggufs
<pre>
/openapi.json, Methods: GET, HEAD
/docs, Methods: GET, HEAD
/docs/oauth2-redirect, Methods: GET, HEAD
/redoc, Methods: GET, HEAD
/load, Methods: GET
/version, Methods: GET
/health, Methods: GET
/metrics, Methods: GET
/tokenize, Methods: POST
/detokenize, Methods: POST
/v1/models, Methods: GET
/ping, Methods: GET
/ping, Methods: POST
/invocations, Methods: POST
/v1/chat/completions, Methods: POST
/v1/chat/completions/batch, Methods: POST
/v1/responses, Methods: POST
/v1/responses/{response_id}, Methods: GET
/v1/responses/{response_id}/cancel, Methods: POST
/v1/completions, Methods: POST
/v1/messages, Methods: POST
/v1/messages/count_tokens, Methods: POST
/generative_scoring, Methods: POST
/scale_elastic_ep, Methods: POST
/is_scaling_elastic_ep, Methods: POST
</pre>


=== Running ===
=== hf operations ===


<syntaxhighlight lang="bash">
<syntaxhighlight lang="bash">
./llama.cpp/sycl/bin/llama-server \
# search latest vllm models
   -hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL \
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
   --spec-type draft-mtp --spec-draft-n-max 2 -ngld 99 \
  --expand "downloads,likes,lastModified" \
   -c 64000 --ctk q8_0 --ctv q8_0 \
   --limit 25 --search "Qwen3.8 27B int4"
   -fa on -ngl 99 --load-mode mlock -np 1 --jinja --reasoning-preserve -t 8
 
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
   --limit 25 --search "Qwen3.8 27B int8 int4"
 
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
   --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B autoround"
 
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
   --limit 25 --search "Qwen3.8 27B w4a8"
 
# list / download / tree
hf models ls --tree -R -h Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
hf download Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
tree -h ~/.cache/huggingface/hub/models--Wrapzii--Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
</syntaxhighlight>
</syntaxhighlight>


=== Benchmarking ===
== llama.cpp ==


<syntaxhighlight lang="bash">
* https://unsloth.ai/docs/models/qwen3.8
./llama.cpp/sycl/bin/llama-bench \
* https://unsloth.ai/docs/basics/dynamic-3.0-ggufs
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL \
  -ctk q8_0 -ctv q8_0 -fa on -ngl 99 \
  -p 512,1024 -n 512


./llama.cpp/sycl/bin/llama-bench \
=== Running ===
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL \
  -ctk q8_0 -ctv q8_0 -fa on -ngl 99 \
  -p 512,1024 -n 512


./llama.cpp/sycl/bin/llama-bench \
<syntaxhighlight lang="bash">
  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_XL \
# works
   -ctk q8_0 -ctv q8_0 -fa on -ngl 99 \
llama-server -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL
   -p 512,1024 -n 512
   --spec-type draft-mtp --spec-draft-n-max 2 --spec-draft-p-min 0.7 -ngld 99
  -fa 1 -c 128000 -ctk f16 -ctv f16
  --temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.0
  --chat-template-kwargs '{"reasoning_effort":"low"}'
  -b 2048 -ub 1024
  -ngl 99 -np 1 -t 6
   --load-mode mlock --jinja --reasoning-preserve
</syntaxhighlight>
</syntaxhighlight>


<pre>
=== Benchmark for me ===
$ ./llama.cpp/sycl/bin/llama-bench  -hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL  -ctk q8_0 -ctv q8_0 -fa on -ngl 99  -p 512,1024 -n 512
| model                          |      size |    params | backend    | ngl | type_k | type_v |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q4_K - Small        |  16.68 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          pp512 |        769.54 ± 4.22 |
| qwen35 27B Q4_K - Small        |  16.68 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          pp1024 |        777.55 ± 1.27 |
| qwen35 27B Q4_K - Small        |  16.68 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          tg512 |        21.06 ± 0.00 |


build: 8e7f22b67 (10398)
{| class="wikitable"
! model || size || params || backend || ngl || type_k || type_v || fa || test || t/s
|-
! qwen35 27B Q5_K - Medium
| 19.43 GiB || | 27.32 B || | SYCL || 99 || q8_0 || q8_0 || 1 || pp512 || 951.05 ± 7.72
|-
! qwen35 27B Q5_K - Medium
| 19.43 GiB || 27.32 B || SYCL || 99 || q8_0 || q8_0 || 1 || tg128 || 22.17 ± 0.06
|}


$ ./llama.cpp/sycl/bin/llama-bench   -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL   -ctk q8_0 -ctv q8_0 -fa on -ngl 99  -p 512,1024 -n 512
<syntaxhighlight lang="bash">
| model                          |      size |    params | backend    | ngl | type_k | type_v |  fa |            test |                  t/s |
# control group
| ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: |
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q8_0
| qwen35 27B Q4_K - Small        |  18.82 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          pp512 |        891.35 ± 1.42 |
| qwen35 27B Q4_K - Small        |  18.82 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          pp1024 |        885.81 ± 0.79 |
| qwen35 27B Q4_K - Small        |  18.82 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          tg512 |        21.98 ± 0.03 |


build: 8e7f22b67 (10398)
# Test -ctv q4_0
$ ./llama.cpp/sycl/bin/llama-bench   -hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K_XL  -ctk q8_0 -ctv q8_0 -fa on -ngl 99  -p 512,1024 -n 512
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q4_0
| model                          |      size |    params | backend    | ngl | type_k | type_v |  fa |            test |                  t/s |
</syntaxhighlight>
| ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q4_K - Small        |  24.13 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          pp512 |        806.99 ± 6.07 |
| qwen35 27B Q4_K - Small        |  24.13 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          pp1024 |        841.24 ± 3.10 |
| qwen35 27B Q4_K - Small        |  24.13 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          tg512 |        17.24 ± 0.02 |
 
build: 8e7f22b67 (10398)
</pre>


== Experiment ==
=== hf operations ===


<syntaxhighlight lang="bash">
<syntaxhighlight lang="bash">
hf models ls --search "Qwen3.8-27B" --apps llama.cpp --expand "downloads,likes,lastModified" --sort downloads --no-truncate --limit 10
# search
hf models ls --apps llama.cpp --sort downloads --no-truncate \
  --expand "downloads,likes,lastModified" \
  --limit 10 --search "Qwen3.8-27B"


# list remote tree
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF


# download quantization
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_M*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q4_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mtp-Qwen3.8-27B-Q4_0.gguf"
hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf"


# list local tree
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf"
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf" -exec sh -c 'rm -f "$1" "$(readlink -f "$1")"' _ {} \;
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF
</syntaxhighlight>
</syntaxhighlight>
== Accuracy ==
https://unsloth.ai/docs/~gitbook/image?url=https%3A%2F%2F3215535692-files.gitbook.io%2F%7E%2Ffiles%2Fv0%2Fb%2Fgitbook-x-prod.appspot.com%2Fo%2Fspaces%252FxhOjnexMCB3dmuQFQ2Zq%252Fuploads%252Fl9piThTmmUsePst5w3F2%252Fimage.png%3Falt%3Dmedia%26token%3D821e88e2-ad4e-40bf-adf2-e13837228e82&width=768&dpr=3&quality=100&sign=dd5f7cb5&sv=2

Latest revision as of 08:30, 27 October 2026

vLLM

Best-Fit Models for B70

The B70 delivers the best performance with W4A8 quantized models.

The underlying rationale can be referenced from TechPowerUp’s spec data, and the listed models show solid performance.

Model Last-Modified W A TP2 / C1 TP2 / C2 TP2 / C4 TP2 / C8 Single
PP 512 TG 128 PP 512 TG 128 PP 512 TG 128 PP 512 TG 128 PP 512 TG 128
cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-BF16-INT4 2026-10-19 4 16 1737 65 2087 97 2121 139 2664 240
cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-INT4 2026-10-19 4 16 1908 79 2062 115 2196 158 2632 269
Launch80/Swift-1.5-Qwen3.8-27B-Uncensored-Int4-baked 2026-10-13 4 16 1899 102 2084 165 2595 241 2758 307
davetha/Huihui-Qwen3.8-27B-abliterated-AutoRound-Int4-baked-embed-int8 2026-10-09 4 16 2026 118 2411 164 2801 266 2785 319
🔥 Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 🔥 2026-10-01 4 16 2335 110 2429 190 2822 286 2928 506

Available Requests

/openapi.json, Methods: GET, HEAD
/docs, Methods: GET, HEAD
/docs/oauth2-redirect, Methods: GET, HEAD
/redoc, Methods: GET, HEAD
/load, Methods: GET
/version, Methods: GET
/health, Methods: GET
/metrics, Methods: GET
/tokenize, Methods: POST
/detokenize, Methods: POST
/v1/models, Methods: GET
/ping, Methods: GET
/ping, Methods: POST
/invocations, Methods: POST
/v1/chat/completions, Methods: POST
/v1/chat/completions/batch, Methods: POST
/v1/responses, Methods: POST
/v1/responses/{response_id}, Methods: GET
/v1/responses/{response_id}/cancel, Methods: POST
/v1/completions, Methods: POST
/v1/messages, Methods: POST
/v1/messages/count_tokens, Methods: POST
/generative_scoring, Methods: POST
/scale_elastic_ep, Methods: POST
/is_scaling_elastic_ep, Methods: POST

hf operations

# search latest vllm models
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B int4"

hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B int8 int4"

hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B autoround"

hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B w4a8" 

# list / download / tree
hf models ls --tree -R -h Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
hf download Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
tree -h ~/.cache/huggingface/hub/models--Wrapzii--Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8

llama.cpp

Running

# works
llama-server -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL
  --spec-type draft-mtp --spec-draft-n-max 2 --spec-draft-p-min 0.7 -ngld 99
  -fa 1 -c 128000 -ctk f16 -ctv f16
  --temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.0
  --chat-template-kwargs '{"reasoning_effort":"low"}'
  -b 2048 -ub 1024
  -ngl 99 -np 1 -t 6
  --load-mode mlock --jinja --reasoning-preserve

Benchmark for me

model size params backend ngl type_k type_v fa test t/s
qwen35 27B Q5_K - Medium 19.43 GiB 27.32 B SYCL 99 q8_0 q8_0 1 pp512 951.05 ± 7.72
qwen35 27B Q5_K - Medium 19.43 GiB 27.32 B SYCL 99 q8_0 q8_0 1 tg128 22.17 ± 0.06 
# control group
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q8_0

# Test -ctv q4_0
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q4_0

hf operations

# search
hf models ls --apps llama.cpp --sort downloads --no-truncate \
  --expand "downloads,likes,lastModified" \
  --limit 10 --search "Qwen3.8-27B"

# list remote tree
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF

# download quantization
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q4_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf"

# list local tree
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf"
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf" -exec sh -c 'rm -f "$1" "$(readlink -f "$1")"' _ {} \;
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF
Retrieved from "https://wiki.fundamental-ramen.com/index.php?title=LM/Qwen_3.8_27B&oldid=4055"