LM/Qwen 3.8 27B: Difference between revisions

From Fundamental Ramen
< LM
Jump to navigation Jump to search
← Older edit
VisualWikitext
 
(27 intermediate revisions by the same user not shown)
Line 12: Line 12:
! rowspan="2" | W
! rowspan="2" | W
! rowspan="2" | A
! rowspan="2" | A
! colspan="2" | Single
! colspan="2" | TP2 / C1
! colspan="2" | TP2 / C1
! colspan="2" | TP2 / C2
! colspan="2" | TP2 / C2
! colspan="2" | TP2 / C4
! colspan="2" | TP2 / C4
! colspan="2" | TP2 / C8
! colspan="2" | TP2 / C8
! colspan="2" | Single
|-
|-
! PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128
! PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128
|-
|-
| style="font-size:10pt; text-align:left;" | cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-BF16-INT4 || 2026-10-19 || 4 || ?
| style="font-size:10pt; text-align:left;" | cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-BF16-INT4
| || || || || || || || || ||
| 2026-10-19 || 4 || 16
| 1737 || 65 || 2087 || 97 || 2121 || 139 || 2664 || 240
| ||
|-
|-
| style="font-size:10pt; text-align:left;" | cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-INT4 || 2026-10-19 || 4 || ?
| style="font-size:10pt; text-align:left;" | cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-INT4
| || || 1908 || 79 || 2062 || 115 || 2196 || 158 || 2632 || 269  
| 2026-10-19 || 4 || 16
| 1908 || 79 || 2062 || 115 || 2196 || 158 || 2632 || 269  
| ||
|-
|-
| style="font-size:10pt; text-align:left;" | Launch80/Swift-1.5-Qwen3.8-27B-Uncensored-Int4-baked || 2026-10-13 || 4 || ?
| style="font-size:10pt; text-align:left;" | Launch80/Swift-1.5-Qwen3.8-27B-Uncensored-Int4-baked
| || || 1899 || 102 || 2084 || 165 || 2595 || 241 || 2758 || 307
| 2026-10-13 || 4 || 16
| 1899 || 102 || 2084 || 165 || 2595 || 241 || 2758 || 307
| ||
|-
|-
| style="font-size:10pt; text-align:left;" | davetha/Huihui-Qwen3.8-27B-abliterated-AutoRound-Int4-baked-embed-int8 || 2026-10-09 || 4 || 8
| style="font-size:10pt; text-align:left;" | davetha/Huihui-Qwen3.8-27B-abliterated-AutoRound-Int4-baked-embed-int8
| || || 2026 || 118 || 2411 || 164 || 2801 || 266 || 2785 || 319
| 2026-10-09 || 4 || 16
| 2026 || 118 || 2411 || 164 || 2801 || 266 || 2785 || 319
| ||
|-
|-
| style="font-size:10pt; text-align:left;" | Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 || 2026-10-01 || 4 || 8
| style="font-size:10pt; text-align:left;" | 🔥 Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 🔥
| || || 2335 || 110 || 2429 || 190 || 2822 || 286 || 2928 || 506
| 2026-10-01 || 4 || 16
| 2335 || 110 || 2429 || 190 || 2822 || 286 || 2928 || 506
| ||
|}
|}
=== Available Requests ===
<pre>
/openapi.json, Methods: GET, HEAD
/docs, Methods: GET, HEAD
/docs/oauth2-redirect, Methods: GET, HEAD
/redoc, Methods: GET, HEAD
/load, Methods: GET
/version, Methods: GET
/health, Methods: GET
/metrics, Methods: GET
/tokenize, Methods: POST
/detokenize, Methods: POST
/v1/models, Methods: GET
/ping, Methods: GET
/ping, Methods: POST
/invocations, Methods: POST
/v1/chat/completions, Methods: POST
/v1/chat/completions/batch, Methods: POST
/v1/responses, Methods: POST
/v1/responses/{response_id}, Methods: GET
/v1/responses/{response_id}/cancel, Methods: POST
/v1/completions, Methods: POST
/v1/messages, Methods: POST
/v1/messages/count_tokens, Methods: POST
/generative_scoring, Methods: POST
/scale_elastic_ep, Methods: POST
/is_scaling_elastic_ep, Methods: POST
</pre>


=== hf operations ===
=== hf operations ===
Line 40: Line 80:
<syntaxhighlight lang="bash">
<syntaxhighlight lang="bash">
# search latest vllm models
# search latest vllm models
hf models ls --search "Qwen3.8 27B w4a8" --apps vllm --expand "downloads,likes,lastModified" --sort last_modified --no-truncate --no-gated --limit 25
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
hf models ls --search "Qwen3.8 27B int8 int4" --apps vllm --expand "downloads,likes,lastModified" --sort last_modified --no-truncate --no-gated --limit 25
  --expand "downloads,likes,lastModified" \
hf models ls --search "Qwen3.8 27B int4" --apps vllm --expand "downloads,likes,lastModified" --sort last_modified --no-truncate --no-gated --limit 25
  --limit 25 --search "Qwen3.8 27B int4"
hf models ls --search "Qwen3.8 27B autoround" --apps vllm --expand "downloads,likes,lastModified" --sort last_modified --no-truncate --no-gated --limit 25
 
hf models ls --search "Bonsai 2" --apps vllm --expand "downloads,likes,lastModified" --sort last_modified --no-truncate --no-gated --limit 25
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B int8 int4"
 
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B autoround"
 
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B w4a8"


# SergiioB
# list / download / tree
hf models ls --tree -R -h SergiioB/Qwen3.8-27B-GPTQ-Int4-sym-G128-MTP-BF16
hf models ls --tree -R -h Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
hf download SergiioB/Qwen3.8-27B-GPTQ-Int4-sym-G128-MTP-BF16
hf download Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
tree -h ~/.cache/huggingface/hub/models--SergiioB--Qwen3.8-27B-GPTQ-Int4-sym-G128-MTP-BF16
tree -h ~/.cache/huggingface/hub/models--Wrapzii--Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
</syntaxhighlight>
</syntaxhighlight>


Line 58: Line 108:


=== Running ===
=== Running ===
The five options are important, especially <code>--chat-template-kwargs '{"reasoning_effort":"..."}'</code>.
The option <code>reasoning_effort</code> can be set to <code>xhigh</code>, <code>medium</code>, and <code>low</code>, but <code>xhigh</code> causes significant performance degradation.
* <code>--temp 1.0</code>
* <code>--top-p 0.95</code>
* <code>--top-k 20</code>
* <code>--min-p 0.0</code>
* <code>--chat-template-kwargs '{"reasoning_effort":"low"}'</code>


<syntaxhighlight lang="bash">
<syntaxhighlight lang="bash">
Line 82: Line 122:


=== Benchmark for me ===
=== Benchmark for me ===
{| class="wikitable"
! model || size || params || backend || ngl || type_k || type_v || fa || test || t/s
|-
! qwen35 27B Q5_K - Medium
| 19.43 GiB || | 27.32 B || | SYCL || 99 || q8_0 || q8_0 || 1 || pp512 || 951.05 ± 7.72
|-
! qwen35 27B Q5_K - Medium
| 19.43 GiB || 27.32 B || SYCL || 99 || q8_0 || q8_0 || 1 || tg128 || 22.17 ± 0.06
|}


<syntaxhighlight lang="bash">
<syntaxhighlight lang="bash">
# control group
# control group
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q8_0
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q8_0
| model                          |      size |    params | backend    | ngl | type_k | type_v |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -----: | -----: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          pp512 |        951.05 ± 7.72 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  99 |  q8_0 |  q8_0 |  1 |          tg128 |        22.17 ± 0.06 |


# Test -ctv q4_0
# Test -ctv q4_0
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q4_0
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q4_0
# Test -ub 2048
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q8_0 -ub 2048
# Test OpenVINO
GGML_OPENVINO_DEVICE=GPU GGML_OPENVINO_STATEFUL_EXECUTION=1 ./llama.cpp/openvino/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 0
</syntaxhighlight>
</syntaxhighlight>


=== Benchmark for community ===
=== hf operations ===
 
<syntaxhighlight lang="markdown">
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q6_K -fa 0,1
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 0,1
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_M -fa 0,1
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_S -fa 0,1
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL -fa 0,1
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL -fa 1 -ctk q8_0 -ctv q8_0
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:Q4_0 -fa 0,1
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:Q4_0 -fa 1 -ctk q8_0 -ctv q8_0
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-IQ4_XS -fa 0,1
 
./llama.cpp/sycl-nodnn/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 0,1
./llama.cpp/sycl-nodnn/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:Q4_0 -fa 0,1
./llama.cpp/sycl-nodnn/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-IQ4_XS -fa 0,1
 
./llama.cpp/sycl-f32/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 0,1
./llama.cpp/sycl-f32/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:Q4_0 -fa 0,1
./llama.cpp/sycl-f32/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-IQ4_XS -fa 0,1
</syntaxhighlight>
 
<pre>
## unsloth/Qwen3.8-27B-GGUF:UD-Q6_K
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q6_K                |  20.46 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        979.33 ± 4.64 |
| qwen35 27B Q6_K                |  20.46 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        21.25 ± 0.05 |
| qwen35 27B Q6_K                |  20.46 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        793.28 ± 1.67 |
| qwen35 27B Q6_K                |  20.46 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        21.15 ± 0.02 |
 
build: 9723942ad (10711)
 
## F16 unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        970.49 ± 3.90 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        22.30 ± 0.01 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        840.86 ± 4.11 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        22.22 ± 0.01 |
 
build: 9723942ad (10711)
 
## F16 unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_M
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Medium      |  18.40 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |      858.88 ± 22.25 |
| qwen35 27B Q5_K - Medium      |  18.40 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        22.22 ± 0.03 |
| qwen35 27B Q5_K - Medium      |  18.40 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        815.38 ± 6.21 |
| qwen35 27B Q5_K - Medium      |  18.40 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        22.25 ± 0.05 |
 
build: 9723942ad (10711)
 
## F16 unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_S
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Small        |  17.37 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        914.77 ± 2.72 |
| qwen35 27B Q5_K - Small        |  17.37 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        22.40 ± 0.02 |
| qwen35 27B Q5_K - Small        |  17.37 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        845.78 ± 1.73 |
| qwen35 27B Q5_K - Small        |  17.37 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        22.33 ± 0.02 |
 
build: 9723942ad (10711)
 
## F16 unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q4_K - Medium      |  16.34 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        811.78 ± 1.40 |
| qwen35 27B Q4_K - Medium      |  16.34 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        21.39 ± 0.02 |
| qwen35 27B Q4_K - Medium      |  16.34 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        784.04 ± 1.72 |
| qwen35 27B Q4_K - Medium      |  16.34 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        21.33 ± 0.02 |
 
build: 9723942ad (10711)
 
## F16 unsloth/Qwen3.8-27B-GGUF:Q4_0
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        813.56 ± 1.92 |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        27.48 ± 0.02 |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        355.72 ± 0.43 |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        27.37 ± 0.03 |
 
build: 9723942ad (10711)
 
## F16 unsloth/Qwen3.8-27B-GGUF:UD-IQ4_XS
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        583.02 ± 0.61 |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        16.40 ± 0.03 |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        586.82 ± 0.83 |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        16.34 ± 0.03 |
 
build: 9723942ad (10711)
 
## F16 without DNN unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        948.86 ± 4.32 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        22.31 ± 0.03 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        791.62 ± 1.74 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        22.17 ± 0.02 |
 
build: 9723942ad (10711)
 
## F16 without DNN unsloth/Qwen3.8-27B-GGUF:Q4_0
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        792.89 ± 2.42 |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        27.53 ± 0.02 |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        353.01 ± 1.91 |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        27.36 ± 0.01 |
 
build: 9723942ad (10711)
 
## F16 without DNN unsloth/Qwen3.8-27B-GGUF:UD-IQ4_XS
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        571.95 ± 0.63 |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        16.43 ± 0.04 |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        560.78 ± 0.38 |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        16.40 ± 0.01 |
 
build: 9723942ad (10711)
 
## F32 unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        311.00 ± 0.43 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        22.26 ± 0.03 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        304.89 ± 0.13 |
| qwen35 27B Q5_K - Medium      |  19.43 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        22.17 ± 0.02 |
 
build: 9723942ad (10711)
 
## F32 unsloth/Qwen3.8-27B-GGUF:Q4_0
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        239.50 ± 2.02 |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        27.07 ± 0.04 |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        270.79 ± 0.47 |
| qwen35 27B Q4_0                |  14.94 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        27.05 ± 0.05 |
 
build: 9723942ad (10711)
 
## F32 unsloth/Qwen3.8-27B-GGUF:UD-IQ4_XS
 
| model                          |      size |    params | backend    | ngl |  fa |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | --: | --------------: | -------------------: |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  0 |          pp512 |        274.41 ± 0.40 |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  0 |          tg128 |        16.51 ± 0.06 |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  1 |          pp512 |        273.20 ± 0.13 |
| qwen35 27B IQ4_XS - 4.25 bpw  |  13.26 GiB |    27.32 B | SYCL      |  -1 |  1 |          tg128 |        16.42 ± 0.01 |
 
build: 9723942ad (10711)
</pre>
 
== Experiment ==


<syntaxhighlight lang="bash">
<syntaxhighlight lang="bash">
# search
# search
hf models ls --search "Qwen3.8-27B" --apps llama.cpp --expand "downloads,likes,lastModified" --sort downloads --no-truncate --limit 10
hf models ls --apps llama.cpp --sort downloads --no-truncate \
  --expand "downloads,likes,lastModified" \
  --limit 10 --search "Qwen3.8-27B"


# unsloth
# list remote tree
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF


hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-IQ4_XS*"
# download quantization
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K.gguf"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_M*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_L*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q8_K_L*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q4_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q4_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q8_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf"
hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf"


# list local tree
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf"
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf"
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf" -exec sh -c 'rm -f "$1" "$(readlink -f "$1")"' _ {} \;
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf" -exec sh -c 'rm -f "$1" "$(readlink -f "$1")"' _ {} \;
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF
</syntaxhighlight>
</syntaxhighlight>
== Accuracy ==
https://unsloth.ai/docs/~gitbook/image?url=https%3A%2F%2F3215535692-files.gitbook.io%2F%7E%2Ffiles%2Fv0%2Fb%2Fgitbook-x-prod.appspot.com%2Fo%2Fspaces%252FxhOjnexMCB3dmuQFQ2Zq%252Fuploads%252Fl9piThTmmUsePst5w3F2%252Fimage.png%3Falt%3Dmedia%26token%3D821e88e2-ad4e-40bf-adf2-e13837228e82&width=768&dpr=3&quality=100&sign=dd5f7cb5&sv=2

Latest revision as of 08:30, 27 October 2026

vLLM

Best-Fit Models for B70

The B70 delivers the best performance with W4A8 quantized models.

The underlying rationale can be referenced from TechPowerUp’s spec data, and the listed models show solid performance.

Model Last-Modified W A TP2 / C1 TP2 / C2 TP2 / C4 TP2 / C8 Single
PP 512 TG 128 PP 512 TG 128 PP 512 TG 128 PP 512 TG 128 PP 512 TG 128
cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-BF16-INT4 2026-10-19 4 16 1737 65 2087 97 2121 139 2664 240
cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-INT4 2026-10-19 4 16 1908 79 2062 115 2196 158 2632 269
Launch80/Swift-1.5-Qwen3.8-27B-Uncensored-Int4-baked 2026-10-13 4 16 1899 102 2084 165 2595 241 2758 307
davetha/Huihui-Qwen3.8-27B-abliterated-AutoRound-Int4-baked-embed-int8 2026-10-09 4 16 2026 118 2411 164 2801 266 2785 319
🔥 Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 🔥 2026-10-01 4 16 2335 110 2429 190 2822 286 2928 506

Available Requests

/openapi.json, Methods: GET, HEAD
/docs, Methods: GET, HEAD
/docs/oauth2-redirect, Methods: GET, HEAD
/redoc, Methods: GET, HEAD
/load, Methods: GET
/version, Methods: GET
/health, Methods: GET
/metrics, Methods: GET
/tokenize, Methods: POST
/detokenize, Methods: POST
/v1/models, Methods: GET
/ping, Methods: GET
/ping, Methods: POST
/invocations, Methods: POST
/v1/chat/completions, Methods: POST
/v1/chat/completions/batch, Methods: POST
/v1/responses, Methods: POST
/v1/responses/{response_id}, Methods: GET
/v1/responses/{response_id}/cancel, Methods: POST
/v1/completions, Methods: POST
/v1/messages, Methods: POST
/v1/messages/count_tokens, Methods: POST
/generative_scoring, Methods: POST
/scale_elastic_ep, Methods: POST
/is_scaling_elastic_ep, Methods: POST

hf operations

# search latest vllm models
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B int4"

hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B int8 int4"

hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B autoround"

hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \
  --expand "downloads,likes,lastModified" \
  --limit 25 --search "Qwen3.8 27B w4a8" 

# list / download / tree
hf models ls --tree -R -h Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
hf download Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8
tree -h ~/.cache/huggingface/hub/models--Wrapzii--Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8

llama.cpp

Running

# works
llama-server -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL
  --spec-type draft-mtp --spec-draft-n-max 2 --spec-draft-p-min 0.7 -ngld 99
  -fa 1 -c 128000 -ctk f16 -ctv f16
  --temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.0
  --chat-template-kwargs '{"reasoning_effort":"low"}'
  -b 2048 -ub 1024
  -ngl 99 -np 1 -t 6
  --load-mode mlock --jinja --reasoning-preserve

Benchmark for me

model size params backend ngl type_k type_v fa test t/s
qwen35 27B Q5_K - Medium 19.43 GiB 27.32 B SYCL 99 q8_0 q8_0 1 pp512 951.05 ± 7.72
qwen35 27B Q5_K - Medium 19.43 GiB 27.32 B SYCL 99 q8_0 q8_0 1 tg128 22.17 ± 0.06 
# control group
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q8_0

# Test -ctv q4_0
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q4_0

hf operations

# search
hf models ls --apps llama.cpp --sort downloads --no-truncate \
  --expand "downloads,likes,lastModified" \
  --limit 10 --search "Qwen3.8-27B"

# list remote tree
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF

# download quantization
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_XL*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q4_0*"
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"
hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf"

# list local tree
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf"
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf" -exec sh -c 'rm -f "$1" "$(readlink -f "$1")"' _ {} \;
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF
Retrieved from "https://wiki.fundamental-ramen.com/index.php?title=LM/Qwen_3.8_27B&oldid=4055"