LM/vLLM: Difference between revisions
< LM
| (52 intermediate revisions by the same user not shown) | |||
| Line 1: | Line 1: | ||
== | == compose == | ||
<syntaxhighlight lang="yaml"> | <syntaxhighlight lang="yaml"> | ||
services: | |||
vllm-xpu: | |||
image: vllm/vllm-openai-xpu:v0.30.0 | |||
container_name: vllm-xpu-qwen38-exp | |||
devices: | |||
- /dev/dri:/dev/dri | |||
volumes: | |||
- ${HOME}/.cache/huggingface:/root/.cache/huggingface | |||
environment: | |||
- HF_HOME=/root/.cache/huggingface | |||
- HF_TOKEN=${HF_TOKEN:-} | |||
- no_proxy=localhost,127.0.0.1 | |||
- ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE | |||
- ZE_AFFINITY_MASK=0 | |||
- VLLM_WORKER_MULTIPROC_METHOD=spawn | |||
- PYTORCH_ALLOC_CONF=expandable_segments:True | |||
- VLLM_XPU_ENABLE_XPU_GRAPH=1 | |||
ports: | |||
- "9000:9000" | |||
entrypoint: ["vllm", "serve"] | |||
command: | |||
- "SergiioB/Qwen3.8-27B-GPTQ-Int4-sym-G128-MTP-BF16" | |||
- "--served-model-name=qwen38" | |||
- "--port=9000" | |||
- "--host=0.0.0.0" | |||
- "--quantization=gptq" | |||
- "--dtype=float16" | |||
- "--max-model-len=131072" | |||
- "--gpu-memory-utilization=0.88" | |||
- "--kv-cache-dtype=fp8" | |||
- "--max-num-seqs=1" | |||
- "--max-num-batched-tokens=8192" | |||
- "--trust-remote-code" | |||
- "--enable-auto-tool-choice" | |||
- "--tool-call-parser=qwen3_xml" | |||
- "--reasoning-parser=qwen3" | |||
- "--limit-mm-per-prompt={\"image\":10}" | |||
- "--speculative-config={\"method\":\"mtp\",\"num_speculative_tokens\":4}" | |||
restart: no | |||
</syntaxhighlight> | </syntaxhighlight> | ||
| Line 28: | Line 52: | ||
{| class="wikitable" | {| class="wikitable" | ||
! Command | ! Command || Results in vllm v0.30.0 | ||
| | |||
|- | |- | ||
| valign="top" | | | valign="top" | <syntaxhighlight lang="bash"> | ||
<syntaxhighlight lang="bash"> | docker exec vllm-xpu-qwen38-prod bash -c \ | ||
pip list | "pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'" | ||
</syntaxhighlight> | </syntaxhighlight> | ||
| | | <pre> | ||
Package Version Editable project location | |||
<pre> | ---------------------------------------- --------------- ------------------------------------- | ||
Package Version | |||
---------------------------------------- ------------- ------------------------------------- | |||
dpcpp-cpp-rt 2026.0.0 | dpcpp-cpp-rt 2026.0.0 | ||
intel-cmplr-lib-rt 2026.0.0 | intel-cmplr-lib-rt 2026.0.0 | ||
intel-cmplr-lib-ur 2026.0.0 | intel-cmplr-lib-ur 2026.0.0 | ||
| Line 237: | Line 69: | ||
intel-pti 0.17.0 | intel-pti 0.17.0 | ||
intel-sycl-rt 2026.0.0 | intel-sycl-rt 2026.0.0 | ||
mkl 2026.0.0 | mkl 2026.0.0 | ||
oneccl 2022.0.0 | oneccl 2022.0.0 | ||
oneccl-devel 2022.0.0 | oneccl-devel 2022.0.0 | ||
| Line 292: | Line 78: | ||
onemkl-sycl-rng 2026.0.0 | onemkl-sycl-rng 2026.0.0 | ||
onemkl-sycl-sparse 2026.0.0 | onemkl-sycl-sparse 2026.0.0 | ||
open_clip_torch 2.32.0 | |||
torch 2.13.0+xpu | torch 2.13.0+xpu | ||
torchaudio 2.11.0+xpu | torchaudio 2.11.0+xpu | ||
torchcodec 0.16.0+cpu | torchcodec 0.16.0+cpu | ||
torchvision 0.28.0+xpu | torchvision 0.28.0+xpu | ||
triton 3.7.2+xpu | triton 3.7.2+xpu | ||
triton-xpu 3.7.2 | triton-xpu 3.7.2 | ||
vllm 0.30.0+xpu | |||
vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils | |||
vllm 0. | |||
vllm_test_utils 0.1 | |||
vllm-xpu-kernels 0.1.14.1 | vllm-xpu-kernels 0.1.14.1 | ||
</pre> | </pre> | ||
|- | |||
| | | valign="top" | <syntaxhighlight lang="bash"> | ||
< | docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels' | ||
</syntaxhighlight> | |||
| <pre> | |||
Name: vllm-xpu-kernels | |||
Version: 0.1.14.1 | |||
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs | |||
Home-page: https://github.com/vllm-project/vllm-xpu-kernels | |||
Author: vLLM Team | |||
Author-email: | |||
License-Expression: Apache-2.0 | |||
Location: /opt/venv/lib/python3.12/site-packages | |||
Requires: | |||
Required-by: vllm | |||
</pre> | |||
|- | |||
| valign="top" | <syntaxhighlight lang="bash"> | |||
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a' | |||
</syntaxhighlight> | |||
| <pre> | |||
Distributor ID: Ubuntu | |||
Description: Ubuntu 24.04.5 LTS | |||
Release: 24.04 | |||
Codename: noble | |||
</pre> | </pre> | ||
|- | |- | ||
| docker | | valign="top" | <syntaxhighlight lang="bash"> | ||
docker exec -it vllm-xpu-qwen38-prod bash | |||
</syntaxhighlight> | |||
| | | | ||
|} | |} | ||
== | == About CCL_SYCL_* variables == | ||
<pre> | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0) | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa | |||
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa | |||
ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode | |||
vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode | |||
</pre> | |||
<syntaxhighlight lang=" | == k8v4 == | ||
<syntaxhighlight lang="bash"> | |||
# 1. 建立並激活 Python 虛擬環境 | |||
python3 -m venv k8v4-env | |||
source k8v4-env/bin/activate | |||
# 2. 安裝 Intel oneAPI（若尚未安裝） | |||
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html | |||
source /opt/intel/oneapi/setvars.sh | |||
# 疑似 AI 誤導 | |||
# 3. 安裝 vLLM 0.30（XPU 版本） | |||
pip install vllm==0.30.0 | |||
# 4. 克隆 k8v4-xpu 專案 | |||
git clone https://github.com/Wrapzii/k8v4-xpu.git | |||
cd k8v4-xpu | |||
# 疑似 AI 誤導 | |||
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so） | |||
# 專案包含 C/C++ 核心，需本機編譯 | |||
pip install -e . | |||
# 或手動編譯: | |||
mkdir -p build && cd build | |||
cmake .. -DCMAKE_BUILD_TYPE=Release | |||
make -j$(nproc) | |||
sudo make install | |||
# 6. 設定環境變數 | |||
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH | |||
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮 | |||
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例） | |||
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \ | |||
--tensor-parallel-size 1 \ | |||
--gpu-memory-utilization 0.90 \ | |||
--max-model-len 32768 | |||
</syntaxhighlight> | </syntaxhighlight> | ||
== CCL Optimization == | |||
=== vLLM and oneccl mapping === | |||
= | {| class="wikitable" | ||
! vLLM || oneccl || native | |||
|- | |||
| 0.31.0 || 2022.1.2 || 2021.18.0 | |||
|- | |||
| 0.30.0 || 2022.0.0 || 2021.18.0 | |||
|} | |||
<syntaxhighlight lang="bash"> | <syntaxhighlight lang="bash"> | ||
# pip | |||
sudo docker | sudo docker exec vllm-xpu-test pip show oneccl | ||
docker | # native | ||
sudo docker exec vllm-xpu-test bash -lc ' | |||
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \ | |||
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50 | |||
' | |||
</syntaxhighlight> | </syntaxhighlight> | ||
== TODO == | == TODO == | ||
* https://hub.docker.com/r/ | * [https://hub.docker.com/r/vllm/vllm-openai-xpu/tags Official Images for XPU] | ||
* https:// | * [https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#create-a-new-python-environment Install using Python] | ||
* https://docs.vllm.ai/en/stable/ | * [https://docs.vllm.ai/en/stable/cli/serve/ vllm serve arguments] | ||
* https:// | * [https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables HF_* Environments] | ||
* https://github.com/ | * [https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md ONEAPI_* SYCL_* Environments] | ||
* [https://www.intel.com/content/www/us/en/docs/oneccl/developer-guide-reference/2021-16/environment-variables.html CCL_* Environments] | |||
* https:// | |||
== References == | == References == | ||
<references/> | <references/> | ||
Latest revision as of 09:49, 26 October 2026
compose
services:
vllm-xpu:
image: vllm/vllm-openai-xpu:v0.30.0
container_name: vllm-xpu-qwen38-exp
devices:
- /dev/dri:/dev/dri
volumes:
- ${HOME}/.cache/huggingface:/root/.cache/huggingface
environment:
- HF_HOME=/root/.cache/huggingface
- HF_TOKEN=${HF_TOKEN:-}
- no_proxy=localhost,127.0.0.1
- ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE
- ZE_AFFINITY_MASK=0
- VLLM_WORKER_MULTIPROC_METHOD=spawn
- PYTORCH_ALLOC_CONF=expandable_segments:True
- VLLM_XPU_ENABLE_XPU_GRAPH=1
ports:
- "9000:9000"
entrypoint: ["vllm", "serve"]
command:
- "SergiioB/Qwen3.8-27B-GPTQ-Int4-sym-G128-MTP-BF16"
- "--served-model-name=qwen38"
- "--port=9000"
- "--host=0.0.0.0"
- "--quantization=gptq"
- "--dtype=float16"
- "--max-model-len=131072"
- "--gpu-memory-utilization=0.88"
- "--kv-cache-dtype=fp8"
- "--max-num-seqs=1"
- "--max-num-batched-tokens=8192"
- "--trust-remote-code"
- "--enable-auto-tool-choice"
- "--tool-call-parser=qwen3_xml"
- "--reasoning-parser=qwen3"
- "--limit-mm-per-prompt={\"image\":10}"
- "--speculative-config={\"method\":\"mtp\",\"num_speculative_tokens\":4}"
restart: no
Verify environments in container
| Command | Results in vllm v0.30.0 |
|---|---|
docker exec vllm-xpu-qwen38-prod bash -c \
"pip list | awk 'NR<3 || tolower(\$0) ~ /torch|intel|triton|vllm|oneccl|mkl|dpc|level/'"
|
Package Version Editable project location ---------------------------------------- --------------- ------------------------------------- dpcpp-cpp-rt 2026.0.0 intel-cmplr-lib-rt 2026.0.0 intel-cmplr-lib-ur 2026.0.0 intel-cmplr-lic-rt 2026.0.0 intel-opencl-rt 2026.0.0 intel-openmp 2026.0.0 intel-pti 0.17.0 intel-sycl-rt 2026.0.0 mkl 2026.0.0 oneccl 2022.0.0 oneccl-devel 2022.0.0 onemkl-license 2026.0.0 onemkl-sycl-blas 2026.0.0 onemkl-sycl-dft 2026.0.0 onemkl-sycl-lapack 2026.0.0 onemkl-sycl-rng 2026.0.0 onemkl-sycl-sparse 2026.0.0 open_clip_torch 2.32.0 torch 2.13.0+xpu torchaudio 2.11.0+xpu torchcodec 0.16.0+cpu torchvision 0.28.0+xpu triton 3.7.2+xpu triton-xpu 3.7.2 vllm 0.30.0+xpu vllm_test_utils 0.1 /workspace/vllm/tests/vllm_test_utils vllm-xpu-kernels 0.1.14.1 |
docker exec vllm-xpu-qwen38-prod bash -c 'pip show vllm-xpu-kernels'
|
Name: vllm-xpu-kernels Version: 0.1.14.1 Summary: A high-throughput and memory-efficient inference and serving engine for LLMs Home-page: https://github.com/vllm-project/vllm-xpu-kernels Author: vLLM Team Author-email: License-Expression: Apache-2.0 Location: /opt/venv/lib/python3.12/site-packages Requires: Required-by: vllm |
docker exec vllm-xpu-qwen38-prod bash -c 'lsb_release -a'
|
Distributor ID: Ubuntu Description: Ubuntu 24.04.5 LTS Release: 24.04 Codename: noble |
docker exec -it vllm-xpu-qwen38-prod bash
|
About CCL_SYCL_* variables
vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_ATL_TRANSPORT changed to be ofi (default:mpi) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLREDUCE_SIMPLE_THRESHOLD changed to be 4294967296 (default:4194304) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_REDUCE_SCATTER_SIMPLE_THRESHOLD changed to be 4294967296 (default:8388608) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLGATHERV_SIMPLE_THRESHOLD changed to be 4294967296 (default:1048576) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| value of CCL_SYCL_ALLTOALL_TMP_BUF changed to be 1 (default:0) vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 118 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL vllm-xpu-qwen38-prod | 2026:09:30-07:47:40: 117 |CCL_WARN| could not get local_idx/count from environment variables, trying to get them from ATL vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 223:[1] |CCL_WARN| no membind support for NUMA node 0, skip thread membind vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 224:[0] |CCL_WARN| no membind support for NUMA node 0, skip thread membind vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| topology recognition shows PCIe connection between devices. If this is not correct, you can disa ble topology recognition, with CCL_TOPO_FABRIC_VERTEX_CONNECTION_CHECK=0. This will assume XeLinks across devices vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 118:[1] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode vllm-xpu-qwen38-prod | 2026:09:30-07:47:41: 117:[0] |CCL_WARN| pidfd is not supported, fallbacks to drmfd exchange mode
k8v4
# 1. 建立並激活 Python 虛擬環境
python3 -m venv k8v4-env
source k8v4-env/bin/activate
# 2. 安裝 Intel oneAPI（若尚未安裝）
# 參考: https://www.intel.com/content/www/us/en/developer/tools/oneapi/index.html
source /opt/intel/oneapi/setvars.sh
# 疑似 AI 誤導
# 3. 安裝 vLLM 0.30（XPU 版本）
pip install vllm==0.30.0
# 4. 克隆 k8v4-xpu 專案
git clone https://github.com/Wrapzii/k8v4-xpu.git
cd k8v4-xpu
# 疑似 AI 誤導
# 5. 編譯 KV Cache 核心庫（libxe2_kv.so）
# 專案包含 C/C++ 核心，需本機編譯
pip install -e .
# 或手動編譯:
mkdir -p build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
make -j$(nproc)
sudo make install
# 6. 設定環境變數
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
export VLLM_KV_CACHE_TYPE=k8v4 # 啟用 K8/V4 壓縮
# 7. 啟動 vLLM 服務（以 Swift-1.5 Qwen3.8-27B 模型為例）
vllm serve Wrapzii/Swift-1.5-Qwen3.8-27b-GPTQ-Int4-baked-v1-embed-int8 \
--tensor-parallel-size 1 \
--gpu-memory-utilization 0.90 \
--max-model-len 32768
CCL Optimization
vLLM and oneccl mapping
| vLLM | oneccl | native |
|---|---|---|
| 0.31.0 | 2022.1.2 | 2021.18.0 |
| 0.30.0 | 2022.0.0 | 2021.18.0 |
# pip
sudo docker exec vllm-xpu-test pip show oneccl
# native
sudo docker exec vllm-xpu-test bash -lc '
grep -aoE "oneCCL[^[:cntrl:]]{0,100}|CCL[_ -]?version[^[:cntrl:]]{0,100}|202[0-9]\.[0-9]+(\.[0-9]+)?" \
/opt/venv/lib/libccl.so.2 | sort -u | head -n 50
'
TODO
- Official Images for XPU
- Install using Python
- vllm serve arguments
- HF_* Environments
- ONEAPI_* SYCL_* Environments
- CCL_* Environments