https://wiki.fundamental-ramen.com/api.php?action=feedcontributions&feedformat=atom&user=Tacoball
Fundamental Ramen - User contributions [en]
2026-11-01T12:44:03Z
User contributions
MediaWiki 1.43.11
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4104
LM/vLLM
2026-10-29T10:42:17Z
<p>Tacoball: /* vLLM and oneccl mapping */</p>
<hr />
<div>== docker compose for B70 (TP=2) ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables<br />
|title=HF_* Environments<br />
|site=huggingface.co<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html<br />
|title=CCL_* Environments<br />
|site=www.intel.com<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
|<br />
<div style="overflow:scroll; max-height:250px;"><br />
<pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
</div><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.33.0 || || || ||<br />
|-<br />
| 0.32.0 || 2026-10-18 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4103
LM/vLLM
2026-10-29T08:41:40Z
<p>Tacoball: /* vLLM and oneccl mapping */</p>
<hr />
<div>== docker compose for B70 (TP=2) ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables<br />
|title=HF_* Environments<br />
|site=huggingface.co<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html<br />
|title=CCL_* Environments<br />
|site=www.intel.com<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
|<br />
<div style="overflow:scroll; max-height:250px;"><br />
<pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
</div><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.33.0 || || || ||<br />
|-<br />
| 0.32.0 || 2026-10-18 || || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4102
LM/vLLM
2026-10-29T07:13:49Z
<p>Tacoball: /* vLLM and oneccl mapping */</p>
<hr />
<div>== docker compose for B70 (TP=2) ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables<br />
|title=HF_* Environments<br />
|site=huggingface.co<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html<br />
|title=CCL_* Environments<br />
|site=www.intel.com<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
|<br />
<div style="overflow:scroll; max-height:250px;"><br />
<pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
</div><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.33.0 || || || ||<br />
|-<br />
| 0.32.0 || 2026-10-18 || || ||<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM&diff=4101
LM
2026-10-29T06:33:33Z
<p>Tacoball: /* Topics */</p>
<hr />
<div>= Topics =<br />
<br />
{| class="wikitable"<br />
! 🤖 Agent / Framework || 🧠 Model || 🚀 Engine || ⚙️ GPU & Driver || 📊 Evaluation<br />
|-<br />
| style="width:200px;vertical-align:top" |<br />
: [[LM/Codex|Codex]]<br />
: [[LM/AnythingLLM|AnythingLLM]]<br />
: [[LM/jrswab_axe|axe]]<br />
: [[LM/MCP|MCP]]<br />
: [[LM/Software Development|Software Development]]<br />
| style="width:200px;vertical-align:top" |<br />
: [[LM/hf|hf (Model Management)]]<br />
: [[LM/Qwen 3.8 27B|Qwen 3.8 27B]] ⭐<br />
: [[LM/Flint|Flint]] 🐣<br />
: [[LM/Gemma 4|Gemma 4]]<br />
: [[LM/Bonsai 2 27B|Bonsai 2 27B]]<br />
: [[LM/Ornith 1.5 35B A3B|Ornith 1.5 35B A3B]]<br />
: [[LM/Experience|Experience]]<br />
| style="width:200px;vertical-align:top" |<br />
: [[LM/vLLM|vLLM]] ⭐<br />
: [[LM/SGLang|SGLang]] 🐣<br />
: [[LM/llama-swap|llama-swap]]<br />
: [[LM/ComfyUI|ComfyUI]]<br />
: [[LM/llama.cpp for SYCL|llama.cpp for SYCL]]<br />
: [[LM/llama.cpp for CUDA|llama.cpp for CUDA]]<br />
: [[LM/Inference Engine Comparison|Comparison]]<br />
| style="width:200px;vertical-align:top" |<br />
: Intel Arc Pro B70<br />
:: [[LM/Install driver|Install driver]]<br />
:: [[LM/Install oneAPI|Install oneAPI (SYCL)]]<br />
:: [[LM/Install OpenVINO|Install OpenVINO]]<br />
:: [[LM/.bashrc for B70|.bashrc for B70]]<br />
:: [[LM/Intel Pro Arc B70|Experiment]]<br />
: NVIDIA RTX 3060<br />
:: [[LM/Install NVIDIA driver|Install driver]]<br />
: AMD Ryzen AI 7 350<br />
:: [[LM/Ryzen AI 7 350|Experiment]]<br />
: Tools<br />
:: [[LM/Install nvtop|Install nvtop]]<br />
:: [[LM/PCIe Trouble Shooting|PCIe Trouble Shooting]]<br />
: Comparison<br />
:: [[LM/GPU Comparison]]<br />
| style="width:200px;vertical-align:top" |<br />
: [[LM/Performance|Performance]]<br />
|}<br />
<br />
= News =<br />
<br />
* [https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro?leaderboard_max_params=32B SWE Pro Benchmark for models < 32B]<br />
* [https://storage.openvinotoolkit.org/repositories/openvino/packages/2026.3/linux/ OpenVINO 2026.3 supports Ubuntu 26] - 2026-08-04<br />
<br />
= Models management =<br />
<br />
<quickmmd name="models-handling-infra"><br />
flowchart TD<br />
A(LiteLLM)<br />
B(vLLM<br>Qwen 3.8 27B)<br />
C1(Intel Arc Pro B70)<br />
C2(Intel Arc Pro B70)<br />
<br />
A --> |Forward| B<br />
B --> |Tensor Parallelism| C1 & C2<br />
%% C1 --- |XCCL| C2<br />
</quickmmd><br />
<br />
* llama-swap can launch model, cannot manage users.<br />
* LiteLLM cannot launch model, can manage users.<br />
<br />
= Investigating vLLM on Intel Arc Pro B70 =<br />
<br />
<quickmmd name="tinkering-b70-vllm"><br />
flowchart TD<br />
subgraph Hardware<br />
A1(Intel Arc Pro B70):::node-okay<br />
A2(Asus ProArt B850 WiFi NEO):::node-okay<br />
A3(AMD Ryzen 5 9600X):::node-okay<br />
end<br />
<br />
subgraph OS<br />
B1(Install<br>Ubuntu 26.04):::node-okay<br />
A1 & A2 & A3 --> B1<br />
end<br />
<br />
subgraph Driver<br />
C1(Install PPA driver<br>26.27.39122.14):::node-okay<br />
B1 --> C1<br />
end<br />
<br />
subgraph Tools<br />
T1(Install<br>pipx)<br />
T2(Install<br>hf)<br />
T3(Install<br>nvtop)<br />
T4(Install<br>tmux)<br />
T5(Install<br>docker)<br />
T1 --> T2<br />
B1 --> T1<br />
B1 --> T3<br />
B1 --> T4<br />
B1 --> T5<br />
end<br />
<br />
subgraph Models<br />
M1(Qwen3.8-27B)<br />
T2 --> M1<br />
end<br />
<br />
subgraph Config<br />
V1(Optimize<br>.bashrc)<br />
V2(docker-compose.yaml)<br />
T2 --> V1<br />
T4 ----> V1<br />
T5 --> V2<br />
end<br />
<br />
subgraph Engine<br />
E1(Docker Image<br>vllm-openai-xpu<br>0.30.0+)<br />
C1 --> E1<br />
T5 --> E1<br />
M1 --> E1<br />
V2 --> E1<br />
end<br />
<br />
subgraph Service<br />
end<br />
<br />
classDef node-okay fill:#efe,stroke:#393<br />
classDef node-warn fill:#fff0e0,stroke:#d63<br />
classDef node-fail fill:#fee,stroke:#d33<br />
classDef node-todo fill:#eee,stroke:#777<br />
</quickmmd><br />
<br />
= Investigating llama.cpp on Intel Arc Pro B70 =<br />
<br />
<quickmmd name="tinkering-b70"><br />
flowchart TD<br />
subgraph Hardware<br />
A1(Intel Arc Pro B70):::node-okay<br />
A2(Asus ProArt B850 WiFi NEO):::node-okay<br />
A3(AMD Ryzen 5 9600X):::node-okay<br />
end<br />
<br />
subgraph OS<br />
B1(Install<br>Ubuntu 26.04):::node-okay<br />
A1 & A2 & A3 --> B1<br />
end<br />
<br />
subgraph Driver<br />
C1(Install PPA driver<br>26.27.39122.14):::node-okay<br />
C2(Install oneAPI<br>2026.1.1.20260724):::node-okay<br />
C1 --> C2<br />
B1 --> C1<br />
end<br />
<br />
subgraph Runtime<br />
D1[Build llama.cpp for<br>SYCL]:::node-okay<br />
C2 --> D1<br />
end<br />
<br />
subgraph Tools<br />
T1(Install<br>pipx)<br />
T2(Install<br>hf)<br />
T3(Install<br>nvtop)<br />
T4(Install<br>tmux)<br />
T5(Install<br>lact)<br />
T6(Install<br>llama-swap)<br />
T1 --> T2<br />
B1 --> T1<br />
B1 --> T3<br />
B1 --> T4<br />
B1 --> T5<br />
T5 --> T6<br />
end<br />
<br />
subgraph Models<br />
M1(Qwen3.8-27B)<br />
T2 --> M1<br />
end<br />
<br />
subgraph config<br />
V1(Optimize<br>.bashrc)<br />
V2(Optimize<br>lmsw.yaml)<br />
C2 --> V1<br />
T2 --> V1<br />
T4 ----> V1<br />
T6 ---> V2<br />
end<br />
<br />
subgraph Service<br />
Z(service<br>llama-swap \<br> -config lmsw.yaml \<br> -listen 0.0.0.0:9876)<br />
M1 --> Z<br />
D1 --> Z<br />
V2 --> Z<br />
end<br />
<br />
classDef node-okay fill:#efe,stroke:#393<br />
classDef node-warn fill:#fff0e0,stroke:#d63<br />
classDef node-fail fill:#fee,stroke:#d33<br />
classDef node-todo fill:#eee,stroke:#777<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/Codex&diff=4100
LM/Codex
2026-10-29T06:32:44Z
<p>Tacoball: /* Install plugin Elephant Goldfish Model */</p>
<hr />
<div>== Configure custom provider ==<br />
<br />
TODO<br />
<br />
== Install plugin Elephant Goldfish Model ==<br />
<br />
TODO</div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/Codex&diff=4099
LM/Codex
2026-10-29T06:32:30Z
<p>Tacoball: </p>
<hr />
<div>== Configure custom provider ==<br />
<br />
TODO<br />
<br />
== Install plugin '''Elephant Goldfish Model''' ==<br />
<br />
TODO</div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/Codex&diff=4098
LM/Codex
2026-10-29T06:31:28Z
<p>Tacoball: Created page with "== Configure custom provider =="</p>
<hr />
<div>== Configure custom provider ==</div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM&diff=4097
LM
2026-10-29T06:30:50Z
<p>Tacoball: /* Topics */</p>
<hr />
<div>= Topics =<br />
<br />
{| class="wikitable"<br />
! 🤖 Agent / Framework || 🧠 Model || 🚀 Engine || ⚙️ GPU & Driver || 📊 Evaluation<br />
|-<br />
| style="width:200px;vertical-align:top" |<br />
: [[LM/Codex|Codex]]<br />
: [[LM/jrswab_axe|axe]]<br />
: [[LM/MCP|MCP]]<br />
: [[LM/Software Development|Software Development]]<br />
| style="width:200px;vertical-align:top" |<br />
: [[LM/hf|hf (Model Management)]]<br />
: [[LM/Qwen 3.8 27B|Qwen 3.8 27B]] ⭐<br />
: [[LM/Flint|Flint]] 🐣<br />
: [[LM/Gemma 4|Gemma 4]]<br />
: [[LM/Bonsai 2 27B|Bonsai 2 27B]]<br />
: [[LM/Ornith 1.5 35B A3B|Ornith 1.5 35B A3B]]<br />
: [[LM/Experience|Experience]]<br />
| style="width:200px;vertical-align:top" |<br />
: [[LM/vLLM|vLLM]] ⭐<br />
: [[LM/SGLang|SGLang]] 🐣<br />
: [[LM/llama-swap|llama-swap]]<br />
: [[LM/ComfyUI|ComfyUI]]<br />
: [[LM/llama.cpp for SYCL|llama.cpp for SYCL]]<br />
: [[LM/llama.cpp for CUDA|llama.cpp for CUDA]]<br />
: [[LM/Inference Engine Comparison|Comparison]]<br />
| style="width:200px;vertical-align:top" |<br />
: Intel Arc Pro B70<br />
:: [[LM/Install driver|Install driver]]<br />
:: [[LM/Install oneAPI|Install oneAPI (SYCL)]]<br />
:: [[LM/Install OpenVINO|Install OpenVINO]]<br />
:: [[LM/.bashrc for B70|.bashrc for B70]]<br />
:: [[LM/Intel Pro Arc B70|Experiment]]<br />
: NVIDIA RTX 3060<br />
:: [[LM/Install NVIDIA driver|Install driver]]<br />
: AMD Ryzen AI 7 350<br />
:: [[LM/Ryzen AI 7 350|Experiment]]<br />
: Tools<br />
:: [[LM/Install nvtop|Install nvtop]]<br />
:: [[LM/PCIe Trouble Shooting|PCIe Trouble Shooting]]<br />
: Comparison<br />
:: [[LM/GPU Comparison]]<br />
| style="width:200px;vertical-align:top" |<br />
: [[LM/Performance|Performance]]<br />
|}<br />
<br />
= News =<br />
<br />
* [https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro?leaderboard_max_params=32B SWE Pro Benchmark for models < 32B]<br />
* [https://storage.openvinotoolkit.org/repositories/openvino/packages/2026.3/linux/ OpenVINO 2026.3 supports Ubuntu 26] - 2026-08-04<br />
<br />
= Models management =<br />
<br />
<quickmmd name="models-handling-infra"><br />
flowchart TD<br />
A(LiteLLM)<br />
B(vLLM<br>Qwen 3.8 27B)<br />
C1(Intel Arc Pro B70)<br />
C2(Intel Arc Pro B70)<br />
<br />
A --> |Forward| B<br />
B --> |Tensor Parallelism| C1 & C2<br />
%% C1 --- |XCCL| C2<br />
</quickmmd><br />
<br />
* llama-swap can launch model, cannot manage users.<br />
* LiteLLM cannot launch model, can manage users.<br />
<br />
= Investigating vLLM on Intel Arc Pro B70 =<br />
<br />
<quickmmd name="tinkering-b70-vllm"><br />
flowchart TD<br />
subgraph Hardware<br />
A1(Intel Arc Pro B70):::node-okay<br />
A2(Asus ProArt B850 WiFi NEO):::node-okay<br />
A3(AMD Ryzen 5 9600X):::node-okay<br />
end<br />
<br />
subgraph OS<br />
B1(Install<br>Ubuntu 26.04):::node-okay<br />
A1 & A2 & A3 --> B1<br />
end<br />
<br />
subgraph Driver<br />
C1(Install PPA driver<br>26.27.39122.14):::node-okay<br />
B1 --> C1<br />
end<br />
<br />
subgraph Tools<br />
T1(Install<br>pipx)<br />
T2(Install<br>hf)<br />
T3(Install<br>nvtop)<br />
T4(Install<br>tmux)<br />
T5(Install<br>docker)<br />
T1 --> T2<br />
B1 --> T1<br />
B1 --> T3<br />
B1 --> T4<br />
B1 --> T5<br />
end<br />
<br />
subgraph Models<br />
M1(Qwen3.8-27B)<br />
T2 --> M1<br />
end<br />
<br />
subgraph Config<br />
V1(Optimize<br>.bashrc)<br />
V2(docker-compose.yaml)<br />
T2 --> V1<br />
T4 ----> V1<br />
T5 --> V2<br />
end<br />
<br />
subgraph Engine<br />
E1(Docker Image<br>vllm-openai-xpu<br>0.30.0+)<br />
C1 --> E1<br />
T5 --> E1<br />
M1 --> E1<br />
V2 --> E1<br />
end<br />
<br />
subgraph Service<br />
end<br />
<br />
classDef node-okay fill:#efe,stroke:#393<br />
classDef node-warn fill:#fff0e0,stroke:#d63<br />
classDef node-fail fill:#fee,stroke:#d33<br />
classDef node-todo fill:#eee,stroke:#777<br />
</quickmmd><br />
<br />
= Investigating llama.cpp on Intel Arc Pro B70 =<br />
<br />
<quickmmd name="tinkering-b70"><br />
flowchart TD<br />
subgraph Hardware<br />
A1(Intel Arc Pro B70):::node-okay<br />
A2(Asus ProArt B850 WiFi NEO):::node-okay<br />
A3(AMD Ryzen 5 9600X):::node-okay<br />
end<br />
<br />
subgraph OS<br />
B1(Install<br>Ubuntu 26.04):::node-okay<br />
A1 & A2 & A3 --> B1<br />
end<br />
<br />
subgraph Driver<br />
C1(Install PPA driver<br>26.27.39122.14):::node-okay<br />
C2(Install oneAPI<br>2026.1.1.20260724):::node-okay<br />
C1 --> C2<br />
B1 --> C1<br />
end<br />
<br />
subgraph Runtime<br />
D1[Build llama.cpp for<br>SYCL]:::node-okay<br />
C2 --> D1<br />
end<br />
<br />
subgraph Tools<br />
T1(Install<br>pipx)<br />
T2(Install<br>hf)<br />
T3(Install<br>nvtop)<br />
T4(Install<br>tmux)<br />
T5(Install<br>lact)<br />
T6(Install<br>llama-swap)<br />
T1 --> T2<br />
B1 --> T1<br />
B1 --> T3<br />
B1 --> T4<br />
B1 --> T5<br />
T5 --> T6<br />
end<br />
<br />
subgraph Models<br />
M1(Qwen3.8-27B)<br />
T2 --> M1<br />
end<br />
<br />
subgraph config<br />
V1(Optimize<br>.bashrc)<br />
V2(Optimize<br>lmsw.yaml)<br />
C2 --> V1<br />
T2 --> V1<br />
T4 ----> V1<br />
T6 ---> V2<br />
end<br />
<br />
subgraph Service<br />
Z(service<br>llama-swap \<br> -config lmsw.yaml \<br> -listen 0.0.0.0:9876)<br />
M1 --> Z<br />
D1 --> Z<br />
V2 --> Z<br />
end<br />
<br />
classDef node-okay fill:#efe,stroke:#393<br />
classDef node-warn fill:#fff0e0,stroke:#d63<br />
classDef node-fail fill:#fee,stroke:#d33<br />
classDef node-todo fill:#eee,stroke:#777<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4096
LM/vLLM
2026-10-29T06:13:57Z
<p>Tacoball: /* compose */</p>
<hr />
<div>== docker compose for B70 (TP=2) ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables<br />
|title=HF_* Environments<br />
|site=huggingface.co<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html<br />
|title=CCL_* Environments<br />
|site=www.intel.com<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
|<br />
<div style="overflow:scroll; max-height:250px;"><br />
<pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
</div><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.33.0 || || || ||<br />
|-<br />
| 0.32.0 || || || ||<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4095
LM/vLLM
2026-10-29T06:13:16Z
<p>Tacoball: /* vLLM and oneccl mapping */</p>
<hr />
<div>== compose ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables<br />
|title=HF_* Environments<br />
|site=huggingface.co<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html<br />
|title=CCL_* Environments<br />
|site=www.intel.com<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
|<br />
<div style="overflow:scroll; max-height:250px;"><br />
<pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
</div><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.33.0 || || || ||<br />
|-<br />
| 0.32.0 || || || ||<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_model&diff=4094
Template:Ref model
2026-10-28T07:14:21Z
<p>Tacoball: </p>
<hr />
<div>[https://huggingface.co/{{{name}}} {{{name}}}] <span style="display:inline-table;border-radius:10px;overflow:hidden;margin-left:4px;vertical-align:1px;font-size:9pt;box-shadow:1px 2px 4px rgba(0,0,0,0.4);"><span style="display:table-cell;width:50%;background:linear-gradient(135deg,#FF5090,#A00040);color:#fff;padding:1px 6px;text-align:center;">{{{last_modified}}}</span><span style="display:table-cell;width:50%;background:linear-gradient(135deg,#999,#111);color:#fff;padding:1px 6px;text-align:center;">W{{{w}}}A{{{a}}}</span></span></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_model&diff=4093
Template:Ref model
2026-10-28T07:13:30Z
<p>Tacoball: </p>
<hr />
<div>[https://huggingface.co/{{{name}}} {{{name}}}] <span style="display:inline-table;border-radius:10px;overflow:hidden;margin-left:4px;vertical-align:0px;font-size:9pt;box-shadow:1px 2px 4px rgba(0,0,0,0.4);"><span style="display:table-cell;width:50%;background:linear-gradient(135deg,#FF5090,#A00040);color:#fff;padding:1px 6px;text-align:center;">{{{last_modified}}}</span><span style="display:table-cell;width:50%;background:linear-gradient(135deg,#999,#111);color:#fff;padding:1px 6px;text-align:center;">W{{{w}}}A{{{a}}}</span></span></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_model&diff=4092
Template:Ref model
2026-10-28T07:10:01Z
<p>Tacoball: </p>
<hr />
<div>[https://huggingface.co/{{{name}}} {{{name}}}] <span style="display:inline-table;border-radius:10px;overflow:hidden;margin-left:4px;vertical-align:-2px;font-size:9pt;"><span style="display:table-cell;width:50%;background:linear-gradient(135deg,#FFD5DE,#E0708A);padding:1px 6px;text-align:center;">{{{last_modified}}}</span><span style="display:table-cell;width:50%;background:linear-gradient(135deg,#888,#1a1a1a);color:#fff;padding:1px 6px;text-align:center;">W{{{w}}}A{{{a}}}</span></span></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_model&diff=4091
Template:Ref model
2026-10-28T07:09:01Z
<p>Tacoball: </p>
<hr />
<div>[https://huggingface.co/{{{name}}} {{{name}}}] <span style="display:inline-flex;border-radius:10px;overflow:hidden;margin-left:4px;vertical-align:0px;font-size:9pt;"><span style="flex:1;background:linear-gradient(135deg,#FFD5DE,#E0708A);padding:1px 6px;text-align:center;">{{{last_modified}}}</span><span style="flex:1;background:linear-gradient(135deg,#888,#1a1a1a);color:#fff;padding:1px 6px;text-align:center;">W{{{w}}}A{{{a}}}</span></span></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_model&diff=4090
Template:Ref model
2026-10-28T07:01:08Z
<p>Tacoball: </p>
<hr />
<div>[https://huggingface.co/{{{name}}} {{{name}}}] <span style="display:inline-block;border-radius:10px;overflow:hidden;margin-left:4px;vertical-align:-7px;"><span style="display:inline-block;background-color:#cef;padding:1px 6px;"><small>W{{{w}}}A{{{a}}}</small></span><span style="display:inline-block;background-color:#333;color:#fff;padding:1px 6px;"><small>{{{last_modified}}}</small></span></span></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_model&diff=4089
Template:Ref model
2026-10-28T06:55:30Z
<p>Tacoball: </p>
<hr />
<div>[https://huggingface.co/{{{name}}} {{{name}}}] - <div style="display:inline-block;column-count:2;column-gap:0;text-align:center;"><br />
<div style="color:#000;background-color:#cef;padding:1px 2px;"><small>W{{{w}}}A{{{a}}}</small></div><br />
<div style="color:#fff;background-color:#333;padding:1px 2px;"><small>{{{last_modified}}}</small></div><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4088
Test Template
2026-10-28T06:45:29Z
<p>Tacoball: /* Result */</p>
<hr />
<div>== Template: ref web ==<br />
<br />
=== Edit ===<br />
<br />
[[Template:ref web]]<br />
<br />
=== Result === <br />
<br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
<br />
=== Syntax ===<br />
<br />
<syntaxhighlight lang="mediawiki"><br />
<ref><br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
</ref><br />
</syntaxhighlight><br />
<br />
== Template: ref model ==<br />
<br />
=== Edit ===<br />
<br />
[[Template:ref model]]<br />
<br />
=== Result === <br />
<br />
{{ref model<br />
|name=wiesion/Ornith-1.5-35B-A3B-int4-AutoRound-GPTQ-XPU<br />
|w=4<br />
|a=16<br />
|last_modified=2026-10-01<br />
}}<br />
<br />
=== Syntax ===<br />
<br />
<syntaxhighlight lang="mediawiki"><br />
<ref><br />
{{ref model<br />
|name=wiesion/Ornith-1.5-35B-A3B-int4-AutoRound-GPTQ-XPU<br />
|w=4<br />
|a=16<br />
}}<br />
</ref><br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_model&diff=4087
Template:Ref model
2026-10-28T06:44:16Z
<p>Tacoball: Created page with "[https://huggingface.co/{{{name}}} {{{name}}}] W{{{w}}}A{{{a}}}"</p>
<hr />
<div>[https://huggingface.co/{{{name}}} {{{name}}}] W{{{w}}}A{{{a}}}</div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4086
Test Template
2026-10-28T06:41:24Z
<p>Tacoball: </p>
<hr />
<div>== Template: ref web ==<br />
<br />
=== Edit ===<br />
<br />
[[Template:ref web]]<br />
<br />
=== Result === <br />
<br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
<br />
=== Syntax ===<br />
<br />
<syntaxhighlight lang="mediawiki"><br />
<ref><br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
</ref><br />
</syntaxhighlight><br />
<br />
== Template: ref model ==<br />
<br />
=== Edit ===<br />
<br />
[[Template:ref model]]<br />
<br />
=== Result === <br />
<br />
{{ref model<br />
|name=wiesion/Ornith-1.5-35B-A3B-int4-AutoRound-GPTQ-XPU<br />
|w=4<br />
|a=16<br />
}}<br />
<br />
=== Syntax ===<br />
<br />
<syntaxhighlight lang="mediawiki"><br />
<ref><br />
{{ref model<br />
|name=wiesion/Ornith-1.5-35B-A3B-int4-AutoRound-GPTQ-XPU<br />
|w=4<br />
|a=16<br />
}}<br />
</ref><br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/Ornith_1.5_35B_A3B&diff=4085
LM/Ornith 1.5 35B A3B
2026-10-28T06:38:18Z
<p>Tacoball: /* vLLM */</p>
<hr />
<div>== vLLM ==<br />
<br />
=== Best-Fit Models for B70 ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! rowspan="2" | Model<br />
! rowspan="2" | Last-Modified<br />
! rowspan="2" | W<br />
! rowspan="2" | A<br />
! colspan="2" | TP2 / C1<br />
! colspan="2" | TP2 / C2<br />
! colspan="2" | TP2 / C4<br />
! colspan="2" | TP2 / C8<br />
! colspan="2" | Single<br />
|-<br />
! PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128<br />
|-<br />
| style="font-size:10pt; text-align:left;" | TODO<br />
| 2026-10-19 || 4 || 16<br />
| 1737 || 65 || 2087 || 97 || 2121 || 139 || 2664 || 240<br />
| ||<br />
|}<br />
<br />
=== TODO ===<br />
<br />
* cyankiwi/Ornith-1.5-35B-A3B-AWQ-INT4<br />
* wiesion/Ornith-1.5-35B-A3B-int4-AutoRound-GPTQ-XPU<br />
<br />
=== hf operations ===<br />
<br />
<syntaxhighlight lang="bash"><br />
# search latest vllm models<br />
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \<br />
--expand "downloads,likes,lastModified" \<br />
--limit 25 --search "Ornith 1.5 35B int4"<br />
<br />
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \<br />
--expand "downloads,likes,lastModified" \<br />
--limit 25 --search "Ornith 1.5 35B autoround"<br />
<br />
# list / download / tree<br />
hf models ls --tree -R -h Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8<br />
hf download Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8<br />
tree -h ~/.cache/huggingface/hub/models--Wrapzii--Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 <br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4084
Test Template
2026-10-28T06:10:45Z
<p>Tacoball: /* Template: ref web */</p>
<hr />
<div>== Template: ref web ==<br />
<br />
=== Edit ===<br />
<br />
[[Template:ref web]]<br />
<br />
=== Result === <br />
<br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
<br />
=== Syntax ===<br />
<br />
<syntaxhighlight lang="mediawiki"><br />
<ref><br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
</ref><br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4083
Test Template
2026-10-28T06:10:21Z
<p>Tacoball: </p>
<hr />
<div>== Template: ref web ==<br />
<br />
[[Template:ref web]]<br />
<br />
=== Result === <br />
<br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
<br />
=== Syntax ===<br />
<br />
<syntaxhighlight lang="mediawiki"><br />
<ref><br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
</ref><br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4082
Test Template
2026-10-28T06:09:16Z
<p>Tacoball: /* Template: ref web */</p>
<hr />
<div>== Template: ref web ==<br />
<br />
=== Result === <br />
<br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
<br />
=== Syntax ===<br />
<br />
<syntaxhighlight lang="mediawiki"><br />
<ref><br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
</ref><br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4081
Test Template
2026-10-28T04:13:14Z
<p>Tacoball: /* Template: ref web */</p>
<hr />
<div>== Template: ref web ==<br />
<br />
=== Result === <br />
<br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
<br />
=== Syntax ===<br />
<br />
<syntaxhighlight lang="mediawiki"><br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4080
Test Template
2026-10-28T04:12:22Z
<p>Tacoball: /* Template: ref web */</p>
<hr />
<div>== Template: ref web ==<br />
<br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
<br />
<syntaxhighlight lang="mediawiki"><br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4079
Test Template
2026-10-28T04:11:59Z
<p>Tacoball: </p>
<hr />
<div>== Template: ref web ==<br />
<br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
<br />
<syntaxhighlight lang="wiki"><br />
{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}<br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Extension%5CTest_QuickMMD&diff=4078
MediaWiki Extension\Test QuickMMD
2026-10-28T02:54:49Z
<p>Tacoball: /* Quick Search Tutor */</p>
<hr />
<div>== Quick Search Tutor ==<br />
<br />
<quickmmd name="qst"><br />
flowchart TD<br />
A(輸入)<br />
B1(輸入疑問句)<br />
B2(其他語氣)<br />
B3(輸入命令句)<br />
C1(疑問句是否詢問詞彙差異)<br />
C3(命令句是否請求製作Anki生字卡)<br />
D1(比較詞彙差異)<br />
D2(解說)<br />
D3(製作Anki生詞卡)<br />
E(輸入內容只有英文單字或片語嗎 - 經常失靈)<br />
F1(中文解說單詞片語)<br />
F2(翻譯)<br />
G(含有中文嗎?)<br />
H1(翻譯英文)<br />
H2(翻譯中文)<br />
<br />
A --> B1 & B2 & B3<br />
B1 --> C1<br />
C1 --> |Y| D1<br />
C1 --> |N| D2<br />
B2 --> D2<br />
B3 --> C3<br />
C3 --> |N| D2<br />
C3 --> |Y| D3<br />
D2 --> E<br />
E --> |Y| F1<br />
E --> |N| F2<br />
F2 --> G<br />
G --> |Y| H1<br />
G --> |N| H2<br />
</quickmmd><br />
<br />
== Backup Flow ==<br />
<br />
<quickmmd name="backup-1"><br />
flowchart TD<br />
A(Start)<br />
B1(Check remote connection)<br />
B2(Check remote space)<br />
B3(Check local space)<br />
B4(Check DB connection)<br />
C(Decide whether to backup or cancel)<br />
D(Backup)<br />
E1(Export SQL<br>data only)<br />
E2(Copy files)<br />
F[Archive]<br />
G[Transfer to destinations]<br />
Z1(Done)<br />
Z2(Cancel)<br />
Z3(Notification)<br />
A --> B1 & B3 & B4<br />
B1 --> B2<br />
B2 --> C<br />
B3 --> C<br />
B4 --> C<br />
C --> D<br />
D --> E1 & E2<br />
E1 & E2 --> F<br />
F --> G<br />
G --> Z1<br />
C --> Z2<br />
Z1 & Z2 --> Z3<br />
</quickmmd><br />
<br />
== Q1 ==<br />
<br />
<quickmmd name="Q1" dump-env="true" dump-mmd="true"><br />
flowchart TD<br />
A --> B<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Extension%5CTest_QuickMMD&diff=4077
MediaWiki Extension\Test QuickMMD
2026-10-28T02:46:18Z
<p>Tacoball: /* Quick Search Tutor */</p>
<hr />
<div>== Quick Search Tutor ==<br />
<br />
<quickmmd name="qst"><br />
flowchart TD<br />
A(輸入)<br />
B1(輸入疑問句)<br />
B2(其他語氣)<br />
B3(輸入命令句)<br />
C1(疑問句是否詢問詞彙差異)<br />
C3(命令句是否請求製作Anki生字卡)<br />
D1(比較詞彙差異)<br />
D2(解說)<br />
D3(製作Anki生詞卡)<br />
E(輸入內容只有英文單字或片語嗎)<br />
F1(中文解說單詞片語)<br />
F2(翻譯)<br />
G(含有中文嗎?)<br />
H1(翻譯英文)<br />
H2(翻譯中文)<br />
<br />
A --> B1 & B2 & B3<br />
B1 --> C1<br />
C1 --> |Y| D1<br />
C1 --> |N| D2<br />
B2 --> D2<br />
B3 --> C3<br />
C3 --> |N| D2<br />
C3 --> |Y| D3<br />
D2 --> E<br />
E --> |Y| F1<br />
E --> |N| F2<br />
F2 --> G<br />
G --> |Y| H1<br />
G --> |N| H2<br />
</quickmmd><br />
<br />
== Backup Flow ==<br />
<br />
<quickmmd name="backup-1"><br />
flowchart TD<br />
A(Start)<br />
B1(Check remote connection)<br />
B2(Check remote space)<br />
B3(Check local space)<br />
B4(Check DB connection)<br />
C(Decide whether to backup or cancel)<br />
D(Backup)<br />
E1(Export SQL<br>data only)<br />
E2(Copy files)<br />
F[Archive]<br />
G[Transfer to destinations]<br />
Z1(Done)<br />
Z2(Cancel)<br />
Z3(Notification)<br />
A --> B1 & B3 & B4<br />
B1 --> B2<br />
B2 --> C<br />
B3 --> C<br />
B4 --> C<br />
C --> D<br />
D --> E1 & E2<br />
E1 & E2 --> F<br />
F --> G<br />
G --> Z1<br />
C --> Z2<br />
Z1 & Z2 --> Z3<br />
</quickmmd><br />
<br />
== Q1 ==<br />
<br />
<quickmmd name="Q1" dump-env="true" dump-mmd="true"><br />
flowchart TD<br />
A --> B<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Extension%5CTest_QuickMMD&diff=4076
MediaWiki Extension\Test QuickMMD
2026-10-28T02:34:22Z
<p>Tacoball: /* Quick Search Tutor */</p>
<hr />
<div>== Quick Search Tutor ==<br />
<br />
<quickmmd name="qst"><br />
flowchart TD<br />
A(輸入)<br />
B1(輸入疑問句)<br />
B2(其他語氣)<br />
B3(輸入命令句)<br />
C1(疑問句是否詢問詞彙差異)<br />
C3(命令句是否請求製作Anki生字卡)<br />
D1(比較詞彙差異)<br />
D2(解說)<br />
D3(製作Anki生詞卡)<br />
E(是不是中文單詞、英文單詞、或是英文片語)<br />
F1(中文解說單詞片語)<br />
F2(翻譯)<br />
G(含有中文嗎?)<br />
H1(翻譯英文)<br />
H2(翻譯中文)<br />
<br />
A --> B1 & B2 & B3<br />
B1 --> C1<br />
C1 --> |Y| D1<br />
C1 --> |N| D2<br />
B2 --> D2<br />
B3 --> C3<br />
C3 --> |N| D2<br />
C3 --> |Y| D3<br />
D2 --> E<br />
E --> |Y| F1<br />
E --> |N| F2<br />
F2 --> G<br />
G --> |Y| H1<br />
G --> |N| H2<br />
</quickmmd><br />
<br />
== Backup Flow ==<br />
<br />
<quickmmd name="backup-1"><br />
flowchart TD<br />
A(Start)<br />
B1(Check remote connection)<br />
B2(Check remote space)<br />
B3(Check local space)<br />
B4(Check DB connection)<br />
C(Decide whether to backup or cancel)<br />
D(Backup)<br />
E1(Export SQL<br>data only)<br />
E2(Copy files)<br />
F[Archive]<br />
G[Transfer to destinations]<br />
Z1(Done)<br />
Z2(Cancel)<br />
Z3(Notification)<br />
A --> B1 & B3 & B4<br />
B1 --> B2<br />
B2 --> C<br />
B3 --> C<br />
B4 --> C<br />
C --> D<br />
D --> E1 & E2<br />
E1 & E2 --> F<br />
F --> G<br />
G --> Z1<br />
C --> Z2<br />
Z1 & Z2 --> Z3<br />
</quickmmd><br />
<br />
== Q1 ==<br />
<br />
<quickmmd name="Q1" dump-env="true" dump-mmd="true"><br />
flowchart TD<br />
A --> B<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Extension%5CTest_QuickMMD&diff=4075
MediaWiki Extension\Test QuickMMD
2026-10-28T01:59:03Z
<p>Tacoball: /* Quick Search Tutor */</p>
<hr />
<div>== Quick Search Tutor ==<br />
<br />
<quickmmd name="qst"><br />
flowchart TD<br />
A(輸入)<br />
B1(輸入疑問句)<br />
B2(其他句型、單詞)<br />
B3(輸入命令句)<br />
C1(疑問句是否詢問詞彙差異)<br />
C3(命令句是否請求製作Anki生字卡)<br />
Z1(比較詞彙差異)<br />
Z2(翻譯)<br />
Z3(製作Anki生詞卡)<br />
<br />
A --> B1 & B2 & B3<br />
B1 --> C1<br />
C1 --> |Y| Z1<br />
C1 --> |N| Z2<br />
B2 --> Z2<br />
B3 --> C3<br />
C3 --> |N| Z2<br />
C3 --> |Y| Z3<br />
</quickmmd><br />
<br />
== Backup Flow ==<br />
<br />
<quickmmd name="backup-1"><br />
flowchart TD<br />
A(Start)<br />
B1(Check remote connection)<br />
B2(Check remote space)<br />
B3(Check local space)<br />
B4(Check DB connection)<br />
C(Decide whether to backup or cancel)<br />
D(Backup)<br />
E1(Export SQL<br>data only)<br />
E2(Copy files)<br />
F[Archive]<br />
G[Transfer to destinations]<br />
Z1(Done)<br />
Z2(Cancel)<br />
Z3(Notification)<br />
A --> B1 & B3 & B4<br />
B1 --> B2<br />
B2 --> C<br />
B3 --> C<br />
B4 --> C<br />
C --> D<br />
D --> E1 & E2<br />
E1 & E2 --> F<br />
F --> G<br />
G --> Z1<br />
C --> Z2<br />
Z1 & Z2 --> Z3<br />
</quickmmd><br />
<br />
== Q1 ==<br />
<br />
<quickmmd name="Q1" dump-env="true" dump-mmd="true"><br />
flowchart TD<br />
A --> B<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Extension%5CTest_QuickMMD&diff=4074
MediaWiki Extension\Test QuickMMD
2026-10-28T01:49:31Z
<p>Tacoball: /* Quick Search Tutor */</p>
<hr />
<div>== Quick Search Tutor ==<br />
<br />
<quickmmd name="qst"><br />
flowchart TD<br />
A(輸入)<br />
B1(輸入命令句)<br />
B2(輸入疑問句)<br />
B3(其他句型)<br />
Z1(翻譯)<br />
Z2(製作 Anki 生詞卡)<br />
A --> B1 & B2 & B3<br />
</quickmmd><br />
<br />
== Backup Flow ==<br />
<br />
<quickmmd name="backup-1"><br />
flowchart TD<br />
A(Start)<br />
B1(Check remote connection)<br />
B2(Check remote space)<br />
B3(Check local space)<br />
B4(Check DB connection)<br />
C(Decide whether to backup or cancel)<br />
D(Backup)<br />
E1(Export SQL<br>data only)<br />
E2(Copy files)<br />
F[Archive]<br />
G[Transfer to destinations]<br />
Z1(Done)<br />
Z2(Cancel)<br />
Z3(Notification)<br />
A --> B1 & B3 & B4<br />
B1 --> B2<br />
B2 --> C<br />
B3 --> C<br />
B4 --> C<br />
C --> D<br />
D --> E1 & E2<br />
E1 & E2 --> F<br />
F --> G<br />
G --> Z1<br />
C --> Z2<br />
Z1 & Z2 --> Z3<br />
</quickmmd><br />
<br />
== Q1 ==<br />
<br />
<quickmmd name="Q1" dump-env="true" dump-mmd="true"><br />
flowchart TD<br />
A --> B<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Extension%5CTest_QuickMMD&diff=4073
MediaWiki Extension\Test QuickMMD
2026-10-28T01:48:45Z
<p>Tacoball: /* Quick Search Tutor */</p>
<hr />
<div>== Quick Search Tutor ==<br />
<br />
<quickmmd name="qst"><br />
flowchart TD<br />
A(輸入)<br />
B1(輸入命令句)<br />
B2(輸入疑問句)<br />
B3(其他句型)<br />
A --> B1 & B2 & B3<br />
</quickmmd><br />
<br />
== Backup Flow ==<br />
<br />
<quickmmd name="backup-1"><br />
flowchart TD<br />
A(Start)<br />
B1(Check remote connection)<br />
B2(Check remote space)<br />
B3(Check local space)<br />
B4(Check DB connection)<br />
C(Decide whether to backup or cancel)<br />
D(Backup)<br />
E1(Export SQL<br>data only)<br />
E2(Copy files)<br />
F[Archive]<br />
G[Transfer to destinations]<br />
Z1(Done)<br />
Z2(Cancel)<br />
Z3(Notification)<br />
A --> B1 & B3 & B4<br />
B1 --> B2<br />
B2 --> C<br />
B3 --> C<br />
B4 --> C<br />
C --> D<br />
D --> E1 & E2<br />
E1 & E2 --> F<br />
F --> G<br />
G --> Z1<br />
C --> Z2<br />
Z1 & Z2 --> Z3<br />
</quickmmd><br />
<br />
== Q1 ==<br />
<br />
<quickmmd name="Q1" dump-env="true" dump-mmd="true"><br />
flowchart TD<br />
A --> B<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Extension%5CTest_QuickMMD&diff=4072
MediaWiki Extension\Test QuickMMD
2026-10-28T01:48:25Z
<p>Tacoball: /* Quick Search Tutor */</p>
<hr />
<div>== Quick Search Tutor ==<br />
<br />
<quickmmd name="qst"><br />
flowchart TD<br />
A(輸入)<br />
B1(輸入命令句)<br />
B2(輸入疑問句)<br />
B3(其他句型)<br />
A --> B1 & B2 && B3<br />
</quickmmd><br />
<br />
== Backup Flow ==<br />
<br />
<quickmmd name="backup-1"><br />
flowchart TD<br />
A(Start)<br />
B1(Check remote connection)<br />
B2(Check remote space)<br />
B3(Check local space)<br />
B4(Check DB connection)<br />
C(Decide whether to backup or cancel)<br />
D(Backup)<br />
E1(Export SQL<br>data only)<br />
E2(Copy files)<br />
F[Archive]<br />
G[Transfer to destinations]<br />
Z1(Done)<br />
Z2(Cancel)<br />
Z3(Notification)<br />
A --> B1 & B3 & B4<br />
B1 --> B2<br />
B2 --> C<br />
B3 --> C<br />
B4 --> C<br />
C --> D<br />
D --> E1 & E2<br />
E1 & E2 --> F<br />
F --> G<br />
G --> Z1<br />
C --> Z2<br />
Z1 & Z2 --> Z3<br />
</quickmmd><br />
<br />
== Q1 ==<br />
<br />
<quickmmd name="Q1" dump-env="true" dump-mmd="true"><br />
flowchart TD<br />
A --> B<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Extension%5CTest_QuickMMD&diff=4071
MediaWiki Extension\Test QuickMMD
2026-10-28T01:47:10Z
<p>Tacoball: </p>
<hr />
<div>== Quick Search Tutor ==<br />
<br />
<quickmmd name="qst"><br />
flowchart TD<br />
A(Input)<br />
</quickmmd><br />
<br />
== Backup Flow ==<br />
<br />
<quickmmd name="backup-1"><br />
flowchart TD<br />
A(Start)<br />
B1(Check remote connection)<br />
B2(Check remote space)<br />
B3(Check local space)<br />
B4(Check DB connection)<br />
C(Decide whether to backup or cancel)<br />
D(Backup)<br />
E1(Export SQL<br>data only)<br />
E2(Copy files)<br />
F[Archive]<br />
G[Transfer to destinations]<br />
Z1(Done)<br />
Z2(Cancel)<br />
Z3(Notification)<br />
A --> B1 & B3 & B4<br />
B1 --> B2<br />
B2 --> C<br />
B3 --> C<br />
B4 --> C<br />
C --> D<br />
D --> E1 & E2<br />
E1 & E2 --> F<br />
F --> G<br />
G --> Z1<br />
C --> Z2<br />
Z1 & Z2 --> Z3<br />
</quickmmd><br />
<br />
== Q1 ==<br />
<br />
<quickmmd name="Q1" dump-env="true" dump-mmd="true"><br />
flowchart TD<br />
A --> B<br />
</quickmmd></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4070
LM/vLLM
2026-10-27T17:33:42Z
<p>Tacoball: /* Verify environments in container */</p>
<hr />
<div>== compose ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables<br />
|title=HF_* Environments<br />
|site=huggingface.co<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html<br />
|title=CCL_* Environments<br />
|site=www.intel.com<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
|<br />
<div style="overflow:scroll; max-height:250px;"><br />
<pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
</div><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.32.0 || || || ||<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4069
LM/vLLM
2026-10-27T17:28:10Z
<p>Tacoball: /* TODO */</p>
<hr />
<div>== compose ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables<br />
|title=HF_* Environments<br />
|site=huggingface.co<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html<br />
|title=CCL_* Environments<br />
|site=www.intel.com<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
| <pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.32.0 || || || ||<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4068
LM/vLLM
2026-10-27T17:27:33Z
<p>Tacoball: /* compose */</p>
<hr />
<div>== compose ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables<br />
|title=HF_* Environments<br />
|site=huggingface.co<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html<br />
|title=CCL_* Environments<br />
|site=www.intel.com<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
| <pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.32.0 || || || ||<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables HF_* Environments]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
* [https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html CCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4067
LM/vLLM
2026-10-27T17:25:34Z
<p>Tacoball: /* TODO */</p>
<hr />
<div>== compose ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
| <pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.32.0 || || || ||<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables HF_* Environments]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
* [https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html CCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4066
LM/vLLM
2026-10-27T17:24:28Z
<p>Tacoball: /* compose */</p>
<hr />
<div>== compose ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
<ref><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
</ref><br />
<ref><br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
</ref><br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
| <pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.32.0 || || || ||<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://docs.vllm.ai/en/stable/cli/serve/ vllm serve arguments]<br />
* [https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables HF_* Environments]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
* [https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html CCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_web&diff=4065
Template:Ref web
2026-10-27T17:23:45Z
<p>Tacoball: </p>
<hr />
<div>[{{{url}}} {{{title}}}]<small> - {{{site}}}</small></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_web&diff=4064
Template:Ref web
2026-10-27T17:22:33Z
<p>Tacoball: </p>
<hr />
<div><ref name="{{{name}}}">[{{{url}}} {{{title}}}]<small> - {{{site}}}</small></ref></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/vLLM&diff=4063
LM/vLLM
2026-10-27T17:20:51Z
<p>Tacoball: /* compose */</p>
<hr />
<div>== compose ==<br />
<br />
<syntaxhighlight lang="yaml"><br />
services:<br />
# Test vLLM 0.30.0<br />
vllm-xpu:<br />
image: vllm/vllm-openai-xpu:0.30.0<br />
container_name: vllm-xpu-test<br />
ports:<br />
- "127.0.0.1:56000:8000"<br />
# For oneCCL use<br />
ipc: host<br />
devices:<br />
# Share host GPU<br />
- /dev/dri:/dev/dri<br />
volumes:<br />
# Share host GPU<br />
- /dev/dri/by-path:/dev/dri/by-path:ro<br />
# Share model cache of Hugging Face <br />
- ${HOME_DIR}/.cache/huggingface:/root/.cache/huggingface<br />
# Compiled cache<br />
- ${HOME_DIR}/.cache/vllm:/root/.cache/vllm<br />
environment:<br />
# TP=2 need them<br />
CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD: "4294967296"<br />
CCL_LOG_LEVEL: "info"<br />
PYTORCH_ALLOC_CONF: "expandable_segments:True"<br />
# Optimize decoding performance for 0.30.0<br />
VLLM_XPU_ENABLE_XPU_GRAPH: "1"<br />
# Block internet access<br />
HF_HUB_OFFLINE: "1"<br />
command:<br />
- "Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8"<br />
- "--host=0.0.0.0"<br />
- "--served-model-name=test"<br />
- "--gpu-memory-utilization=${VRAM_RESERVATION}"<br />
- "--dtype=bfloat16"<br />
- "--max-model-len=${CONTEXT_LENGTH}"<br />
- "--kv-cache-dtype=fp8"<br />
- "--tensor-parallel-size=${TP}"<br />
- "--max-num-seqs=8"<br />
- "--max-num-batched-tokens=8192"<br />
- "--enable-auto-tool-choice"<br />
- "--enable-prefix-caching"<br />
- "--reasoning-parser=qwen3"<br />
- "--tool-call-parser=qwen3_xml"<br />
restart: "no"<br />
</syntaxhighlight><br />
{{ref web<br />
|url=https://hub.docker.com/r/vllm/vllm-openai-xpu/tags<br />
|title=Official vLLM image vllm-openai-xpu<br />
|site=hub.docker.com<br />
}}<br />
{{ref web<br />
|url=https://docs.vllm.ai/en/stable/cli/serve/<br />
|title=vllm serve arguments<br />
|site=docs.vllm.ai<br />
}}<br />
<br />
== Verify environments in container ==<br />
<br />
{| class="wikitable"<br />
! Command || Results in vllm v0.30.0<br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c \<br />
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"<br />
</syntaxhighlight><br />
| <pre><br />
Package Version Editable project location<br />
---------------------------------------- --------------- -------------------------------------<br />
dpcpp-cpp-rt 2026.0.0<br />
intel-cmplr-lib-rt 2026.0.0<br />
intel-cmplr-lib-ur 2026.0.0<br />
intel-cmplr-lic-rt 2026.0.0<br />
intel-opencl-rt 2026.0.0<br />
intel-openmp 2026.0.0<br />
intel-pti 0.17.0<br />
intel-sycl-rt 2026.0.0<br />
mkl 2026.0.0<br />
oneccl 2022.0.0<br />
oneccl-devel 2022.0.0<br />
onemkl-license 2026.0.0<br />
onemkl-sycl-blas 2026.0.0<br />
onemkl-sycl-dft 2026.0.0<br />
onemkl-sycl-lapack 2026.0.0<br />
onemkl-sycl-rng 2026.0.0<br />
onemkl-sycl-sparse 2026.0.0<br />
open_clip_torch 2.32.0<br />
torch 2.13.0+xpu<br />
torchaudio 2.11.0+xpu<br />
torchcodec 0.16.0+cpu<br />
torchvision 0.28.0+xpu<br />
triton 3.7.2+xpu<br />
triton-xpu 3.7.2<br />
vllm 0.30.0+xpu<br />
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils<br />
vllm-xpu-kernels 0.1.14.1<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'<br />
</syntaxhighlight><br />
| <pre><br />
Name: vllm-xpu-kernels<br />
Version: 0.1.14.1<br />
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs<br />
Home-page: https://github.com/vllm-project/vllm-xpu-kernels<br />
Author: vLLM Team<br />
Author-email:<br />
License-Expression: Apache-2.0<br />
Location: /opt/venv/lib/python3.12/site-packages<br />
Requires:<br />
Required-by: vllm<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'<br />
</syntaxhighlight><br />
| <pre><br />
Distributor ID: Ubuntu<br />
Description: Ubuntu 24.04.5 LTS<br />
Release: 24.04<br />
Codename: noble<br />
</pre><br />
|-<br />
| valign="top" | <syntaxhighlight lang="bash"><br />
docker exec -it vllm-xpu-qwen38-prod bash<br />
</syntaxhighlight><br />
|<br />
|}<br />
<br />
== About CCL_SYCL_* variables ==<br />
<br />
<pre><br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0)<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa<br />
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode<br />
</pre><br />
<br />
== k8v4 ==<br />
<br />
<syntaxhighlight lang="bash"><br />
# 1. 建立並激活 Python 虛擬環境<br />
python3 -m venv k8v4-env<br />
source k8v4-env/bin/activate<br />
<br />
# 2. 安裝 Intel oneAPI（若尚未安裝）<br />
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html<br />
source /opt/intel/oneapi/setvars.sh<br />
<br />
# 疑似 AI 誤導<br />
# 3. 安裝 vLLM 0.30（XPU 版本）<br />
pip install vllm==0.30.0<br />
<br />
# 4. 克隆 k8v4-xpu 專案<br />
git clone https://github.com/Wrapzii/k8v4-xpu.git<br />
cd k8v4-xpu<br />
<br />
# 疑似 AI 誤導<br />
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）<br />
# 專案包含 C/C++ 核心，需本機編譯<br />
pip install -e .<br />
# 或手動編譯:<br />
mkdir -p build && cd build<br />
cmake .. -DCMAKE_BUILD_TYPE=Release<br />
make -j$(nproc)<br />
sudo make install<br />
<br />
# 6. 設定環境變數<br />
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH<br />
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮<br />
<br />
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）<br />
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \<br />
--tensor-parallel-size 1 \<br />
--gpu-memory-utilization 0.90 \<br />
--max-model-len 32768<br />
</syntaxhighlight><br />
<br />
== CCL Optimization ==<br />
<br />
=== vLLM and oneccl mapping ===<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! vLLM image || released || TP for B70 || pip oneccl || native oneCCL<br />
|-<br />
| 0.32.0 || || || ||<br />
|-<br />
| 0.31.0 || 2026-10-04 || ❌ || 2022.1.2 || 2021.18.0<br />
|-<br />
| 0.30.0 || 2026-09-22 || ✅ || 2022.0.0 || 2021.18.0<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# pip<br />
sudo docker exec vllm-xpu-test pip show oneccl<br />
<br />
# native<br />
sudo docker exec vllm-xpu-test bash -lc '<br />
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \<br />
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50<br />
'<br />
</syntaxhighlight><br />
<br />
== TODO ==<br />
<br />
* [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python]<br />
* [https://docs.vllm.ai/en/stable/cli/serve/ vllm serve arguments]<br />
* [https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables HF_* Environments]<br />
* [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments]<br />
* [https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html CCL_* Environments]<br />
<br />
== References ==<br />
<br />
<div style="column-count: 2; column-gap: 2em;"><br />
<references/><br />
</div></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_web&diff=4062
Template:Ref web
2026-10-27T17:16:52Z
<p>Tacoball: </p>
<hr />
<div>[{{{url}}} {{{title}}}] <small>- {{{site}}}</small><br />
<noinclude><br />
<templatedata><br />
{<br />
"params": {<br />
"url": {},<br />
"title": {},<br />
"site": {}<br />
},<br />
"description": "reference URL"<br />
}<br />
</templatedata><br />
</noinclude></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4061
Test Template
2026-10-27T17:15:20Z
<p>Tacoball: </p>
<hr />
<div>{{ref web<br />
|url=https://www.google.com<br />
|title=Google<br />
|site=google.com<br />
}}</div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Template:Ref_web&diff=4060
Template:Ref web
2026-10-27T17:14:10Z
<p>Tacoball: Created page with "[{{{url}}} {{{title}}}] - {{{site}}} <noinclude> <templatedata> { "params": { "url": {}, "title": {}, "site": {} }, "description": "reference URL" } </templatedata> </noinclude>"</p>
<hr />
<div>[{{{url}}} {{{title}}}] - {{{site}}}<br />
<noinclude><br />
<templatedata><br />
{<br />
"params": {<br />
"url": {},<br />
"title": {},<br />
"site": {}<br />
},<br />
"description": "reference URL"<br />
}<br />
</templatedata><br />
</noinclude></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Test_Template&diff=4059
Test Template
2026-10-27T17:10:16Z
<p>Tacoball: Created page with "{{ref web}}"</p>
<hr />
<div>{{ref web}}</div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=Useful_Extensions&diff=4058
Useful Extensions
2026-10-27T17:08:07Z
<p>Tacoball: /* Extension List */</p>
<hr />
<div>== Extension List ==<br />
<br />
{| class="wikitable"<br />
! Extension ||<br />
|-<br />
| TODO || ...<br />
|}</div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Migration&diff=4057
MediaWiki Migration
2026-10-27T13:34:00Z
<p>Tacoball: /* Steps */</p>
<hr />
<div>== Steps ==<br />
<br />
* Choose new image<br />
* Update docker-compose.yaml<br />
* Check docker-compose.yaml by docker compose config<br />
* docker compose up<br />
* List package versions<br />
* Test extensions<br />
* Check file system permissions<br />
* Check cite this page to make sure no internal URL leaks.<br />
<br />
== Choose new image ==<br />
<br />
=== Comparison ===<br />
<br />
{| class="wikitable"<br />
! Image<br />
! Included Web Server<br />
! PHP Execution Method<br />
! Base OS<br />
! Recommended Use Case<br />
|-<br />
| <code>mediawiki:1.43</code><br />
| Apache<br />
| mod_php<br />
| Debian<br />
| Simple deployment with a single container<br />
|-<br />
| <code>mediawiki:1.43-fpm</code><br />
| None<br />
| PHP-FPM<br />
| Debian<br />
| Use with Nginx or an existing Apache server<br />
|-<br />
| <code>mediawiki:1.43-fpm-alpine</code><br />
| None<br />
| PHP-FPM<br />
| Alpine Linux<br />
| Smaller image size for users familiar with Alpine<br />
|}<br />
<br />
=== package versions ===<br />
<br />
<syntaxhighlight lang="bash"><br />
docker pull mediawiki:1.43.9<br />
docker run -it mediawiki:1.43.9 bash<br />
<br />
cat /etc/os-release<br />
php -v<br />
php -m<br />
curl -V<br />
</syntaxhighlight><br />
<br />
=== debug ===<br />
<br />
Permission problem usually arize when upgrade new version.<br />
At that time, bash into the container is necessary for debugging.<br />
<br />
<syntaxhighlight lang="bash"><br />
sudo docker exec -it mw-main bash<br />
sudo docker exec -it mw-main mmdc -V<br />
sudo docker exec -it mw-main bash -c 'echo $PATH | tr ":" "\n"'<br />
</syntaxhighlight><br />
<br />
== from 1.39 to 1.43 ==<br />
<br />
* Backup<br />
** LocalSettings.php<br />
** images<br />
** database<br />
<br />
<syntaxhighlight lang="bash"><br />
docker compose exec mediawiki php maintenance/run.php update<br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=MediaWiki_Migration&diff=4056
MediaWiki Migration
2026-10-27T13:27:01Z
<p>Tacoball: /* from 1.39 to 1.47 */</p>
<hr />
<div>== Steps ==<br />
<br />
* Choose new image<br />
* Update docker-compose.yaml<br />
* Check docker-compose.yaml by docker compose config<br />
* docker compose up<br />
* List package versions<br />
* Test extensions<br />
* Check file system permissions<br />
<br />
== Choose new image ==<br />
<br />
=== Comparison ===<br />
<br />
{| class="wikitable"<br />
! Image<br />
! Included Web Server<br />
! PHP Execution Method<br />
! Base OS<br />
! Recommended Use Case<br />
|-<br />
| <code>mediawiki:1.43</code><br />
| Apache<br />
| mod_php<br />
| Debian<br />
| Simple deployment with a single container<br />
|-<br />
| <code>mediawiki:1.43-fpm</code><br />
| None<br />
| PHP-FPM<br />
| Debian<br />
| Use with Nginx or an existing Apache server<br />
|-<br />
| <code>mediawiki:1.43-fpm-alpine</code><br />
| None<br />
| PHP-FPM<br />
| Alpine Linux<br />
| Smaller image size for users familiar with Alpine<br />
|}<br />
<br />
=== package versions ===<br />
<br />
<syntaxhighlight lang="bash"><br />
docker pull mediawiki:1.43.9<br />
docker run -it mediawiki:1.43.9 bash<br />
<br />
cat /etc/os-release<br />
php -v<br />
php -m<br />
curl -V<br />
</syntaxhighlight><br />
<br />
=== debug ===<br />
<br />
Permission problem usually arize when upgrade new version.<br />
At that time, bash into the container is necessary for debugging.<br />
<br />
<syntaxhighlight lang="bash"><br />
sudo docker exec -it mw-main bash<br />
sudo docker exec -it mw-main mmdc -V<br />
sudo docker exec -it mw-main bash -c 'echo $PATH | tr ":" "\n"'<br />
</syntaxhighlight><br />
<br />
== from 1.39 to 1.43 ==<br />
<br />
* Backup<br />
** LocalSettings.php<br />
** images<br />
** database<br />
<br />
<syntaxhighlight lang="bash"><br />
docker compose exec mediawiki php maintenance/run.php update<br />
</syntaxhighlight></div>
Tacoball
https://wiki.fundamental-ramen.com/index.php?title=LM/Qwen_3.8_27B&diff=4055
LM/Qwen 3.8 27B
2026-10-27T08:30:43Z
<p>Tacoball: /* Best-Fit Models for B70 */</p>
<hr />
<div>== vLLM ==<br />
<br />
=== Best-Fit Models for B70 ===<br />
<br />
The B70 delivers the best performance with W4A8 quantized models.<br />
<br />
The underlying rationale can be referenced from TechPowerUp’s spec data, and the listed models show solid performance.<br />
<br />
{| class="wikitable" style="text-align:center;"<br />
! rowspan="2" | Model<br />
! rowspan="2" | Last-Modified<br />
! rowspan="2" | W<br />
! rowspan="2" | A<br />
! colspan="2" | TP2 / C1<br />
! colspan="2" | TP2 / C2<br />
! colspan="2" | TP2 / C4<br />
! colspan="2" | TP2 / C8<br />
! colspan="2" | Single<br />
|-<br />
! PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128 || PP 512 || TG 128<br />
|-<br />
| style="font-size:10pt; text-align:left;" | cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-BF16-INT4<br />
| 2026-10-19 || 4 || 16<br />
| 1737 || 65 || 2087 || 97 || 2121 || 139 || 2664 || 240<br />
| ||<br />
|-<br />
| style="font-size:10pt; text-align:left;" | cyankiwi/Swift-1.5-Qwen3.8-27B-AWQ-INT4<br />
| 2026-10-19 || 4 || 16<br />
| 1908 || 79 || 2062 || 115 || 2196 || 158 || 2632 || 269 <br />
| ||<br />
|-<br />
| style="font-size:10pt; text-align:left;" | Launch80/Swift-1.5-Qwen3.8-27B-Uncensored-Int4-baked<br />
| 2026-10-13 || 4 || 16<br />
| 1899 || 102 || 2084 || 165 || 2595 || 241 || 2758 || 307<br />
| ||<br />
|-<br />
| style="font-size:10pt; text-align:left;" | davetha/Huihui-Qwen3.8-27B-abliterated-AutoRound-Int4-baked-embed-int8<br />
| 2026-10-09 || 4 || 16<br />
| 2026 || 118 || 2411 || 164 || 2801 || 266 || 2785 || 319<br />
| ||<br />
|-<br />
| style="font-size:10pt; text-align:left;" | 🔥 Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 🔥<br />
| 2026-10-01 || 4 || 16<br />
| 2335 || 110 || 2429 || 190 || 2822 || 286 || 2928 || 506<br />
| ||<br />
|}<br />
<br />
=== Available Requests ===<br />
<br />
<pre><br />
/openapi.json, Methods: GET, HEAD<br />
/docs, Methods: GET, HEAD<br />
/docs/oauth2-redirect, Methods: GET, HEAD<br />
/redoc, Methods: GET, HEAD<br />
/load, Methods: GET<br />
/version, Methods: GET<br />
/health, Methods: GET<br />
/metrics, Methods: GET<br />
/tokenize, Methods: POST<br />
/detokenize, Methods: POST<br />
/v1/models, Methods: GET<br />
/ping, Methods: GET<br />
/ping, Methods: POST<br />
/invocations, Methods: POST<br />
/v1/chat/completions, Methods: POST<br />
/v1/chat/completions/batch, Methods: POST<br />
/v1/responses, Methods: POST<br />
/v1/responses/{response_id}, Methods: GET<br />
/v1/responses/{response_id}/cancel, Methods: POST<br />
/v1/completions, Methods: POST<br />
/v1/messages, Methods: POST<br />
/v1/messages/count_tokens, Methods: POST<br />
/generative_scoring, Methods: POST<br />
/scale_elastic_ep, Methods: POST<br />
/is_scaling_elastic_ep, Methods: POST<br />
</pre><br />
<br />
=== hf operations ===<br />
<br />
<syntaxhighlight lang="bash"><br />
# search latest vllm models<br />
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \<br />
--expand "downloads,likes,lastModified" \<br />
--limit 25 --search "Qwen3.8 27B int4"<br />
<br />
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \<br />
--expand "downloads,likes,lastModified" \<br />
--limit 25 --search "Qwen3.8 27B int8 int4"<br />
<br />
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \<br />
--expand "downloads,likes,lastModified" \<br />
--limit 25 --search "Qwen3.8 27B autoround"<br />
<br />
hf models ls --apps vllm --sort last_modified --no-truncate --no-gated \<br />
--expand "downloads,likes,lastModified" \<br />
--limit 25 --search "Qwen3.8 27B w4a8" <br />
<br />
# list / download / tree<br />
hf models ls --tree -R -h Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8<br />
hf download Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8<br />
tree -h ~/.cache/huggingface/hub/models--Wrapzii--Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 <br />
</syntaxhighlight><br />
<br />
== llama.cpp ==<br />
<br />
* https://unsloth.ai/docs/models/qwen3.8<br />
* https://unsloth.ai/docs/basics/dynamic-3.0-ggufs<br />
<br />
=== Running ===<br />
<br />
<syntaxhighlight lang="bash"><br />
# works<br />
llama-server -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL<br />
--spec-type draft-mtp --spec-draft-n-max 2 --spec-draft-p-min 0.7 -ngld 99<br />
-fa 1 -c 128000 -ctk f16 -ctv f16<br />
--temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.0<br />
--chat-template-kwargs '{"reasoning_effort":"low"}'<br />
-b 2048 -ub 1024<br />
-ngl 99 -np 1 -t 6<br />
--load-mode mlock --jinja --reasoning-preserve<br />
</syntaxhighlight><br />
<br />
=== Benchmark for me ===<br />
<br />
{| class="wikitable"<br />
! model || size || params || backend || ngl || type_k || type_v || fa || test || t/s<br />
|-<br />
! qwen35 27B Q5_K - Medium<br />
| 19.43 GiB || | 27.32 B || | SYCL || 99 || q8_0 || q8_0 || 1 || pp512 || 951.05 ± 7.72<br />
|-<br />
! qwen35 27B Q5_K - Medium<br />
| 19.43 GiB || 27.32 B || SYCL || 99 || q8_0 || q8_0 || 1 || tg128 || 22.17 ± 0.06<br />
|}<br />
<br />
<syntaxhighlight lang="bash"><br />
# control group<br />
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q8_0<br />
<br />
# Test -ctv q4_0<br />
./llama.cpp/sycl/bin/llama-bench -hf unsloth/Qwen3.8-27B-GGUF:UD-Q5_K_XL -fa 1 -ctk q8_0 -ctv q4_0<br />
</syntaxhighlight><br />
<br />
=== hf operations ===<br />
<br />
<syntaxhighlight lang="bash"><br />
# search<br />
hf models ls --apps llama.cpp --sort downloads --no-truncate \<br />
--expand "downloads,likes,lastModified" \<br />
--limit 10 --search "Qwen3.8-27B"<br />
<br />
# list remote tree<br />
hf models ls --tree -R -h unsloth/Qwen3.8-27B-GGUF<br />
<br />
# download quantization<br />
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q4_K_XL*"<br />
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q5_K_XL*"<br />
hf download unsloth/Qwen3.8-27B-GGUF --include "*UD-Q6_K_XL*"<br />
hf download unsloth/Qwen3.8-27B-GGUF --include "*Q4_0*"<br />
hf download unsloth/Qwen3.8-27B-GGUF --include "mmproj-BF16*"<br />
hf download unsloth/Qwen3.8-27B-GGUF --include "*mtp-Qwen3.8-27B-Q4_0.gguf"<br />
<br />
# list local tree<br />
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf"<br />
find ~/.cache/huggingface/hub -name "*Qwen3.8-27B-UD-Q8_K_L.gguf" -exec sh -c 'rm -f "$1" "$(readlink -f "$1")"' _ {} \;<br />
tree -h ~/.cache/huggingface/hub/models--unsloth--Qwen3.8-27B-GGUF<br />
</syntaxhighlight></div>
Tacoball