LM/SGLang: Difference between revisions
< LM
Jump to navigation
Jump to search
No edit summary |
|||
| (18 intermediate revisions by the same user not shown) | |||
| Line 1: | Line 1: | ||
'''Engine works fine, but most models cannot loaded in xpu mode.''' | |||
= Rebuild docker image from source = | |||
<syntaxhighlight lang="bash"> | |||
cd sglang | |||
git pull | |||
cd docker | |||
docker build -t sglang-xpu:latest -f xpu.Dockerfile . | |||
</syntaxhighlight> | |||
= Run = | |||
<syntaxhighlight lang="bash"> | |||
docker run \ | |||
-it \ | |||
--privileged \ | |||
--ipc=host \ | |||
--network=host \ | |||
--user root \ | |||
--group-add $(getent group video | cut -d: -f3) \ | |||
--device /dev/dri \ | |||
-v /dev/dri/by-path:/dev/dri/by-path \ | |||
-v /dev/shm:/dev/shm \ | |||
-v ~/.cache/huggingface:/root/.cache/huggingface \ | |||
-p 30000:30000 \ | |||
-e "HF_TOKEN=$HF_TOKEN" \ | |||
sglang-xpu:latest /bin/bash | |||
</syntaxhighlight> | |||
In container ... | |||
<syntaxhighlight lang="bash"> | |||
sglang serve \ | |||
--model-path Intel/gemma-4-12B-it-int4-AutoRound \ | |||
--device xpu \ | |||
--attention-backend intel_xpu | |||
</syntaxhighlight> | |||
= Structure = | = Structure = | ||
| Line 8: | Line 47: | ||
B["SGLang Router & Radix Cache 管理 (CPU)"] | B["SGLang Router & Radix Cache 管理 (CPU)"] | ||
A --> B | A --> B | ||
end | |||
%% SRT 引擎層 | %% SRT 引擎層 | ||
| Line 16: | Line 55: | ||
B --> C | B --> C | ||
C --> D | C --> D | ||
end | |||
%% 轉接層：Intel XPU 平台分支 | %% 轉接層：Intel XPU 平台分支 | ||
| Line 24: | Line 63: | ||
D -- "檢測到 Intel GPU" --> E | D -- "檢測到 Intel GPU" --> E | ||
D --> F | D --> F | ||
end | |||
%% 高效算子與編譯層 | %% 高效算子與編譯層 | ||
| Line 37: | Line 76: | ||
E --> I | E --> I | ||
G & H -- "AOT 或 JIT 編譯" --> J | G & H -- "AOT 或 JIT 編譯" --> J | ||
end | |||
%% 硬體實體層 | %% 硬體實體層 | ||
| Line 45: | Line 84: | ||
I --> K | I --> K | ||
F --> K | F --> K | ||
end | |||
%% 樣式設定 | %% 樣式設定 | ||
classDef frontend fill:#e1f5fe,stroke:#0288d1,stroke-width:2px | %%classDef frontend fill:#e1f5fe,stroke:#0288d1,stroke-width:2px | ||
classDef engine fill:#fff3e0,stroke:#f57c00,stroke-width:2px | %%classDef engine fill:#fff3e0,stroke:#f57c00,stroke-width:2px | ||
classDef xpu fill:#e8f5e9,stroke:#388e3c,stroke-width:2px | %%classDef xpu fill:#e8f5e9,stroke:#388e3c,stroke-width:2px | ||
classDef kernel fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px | %%classDef kernel fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px | ||
classDef hw fill:#eceff1,stroke:#455a64,stroke-width:2px | %%classDef hw fill:#eceff1,stroke:#455a64,stroke-width:2px | ||
%%class A,B frontend; | %%class A,B frontend; | ||
Latest revision as of 09:31, 17 September 2026
Engine works fine, but most models cannot loaded in xpu mode.
Rebuild docker image from source
cd sglang
git pull
cd docker
docker build -t sglang-xpu:latest -f xpu.Dockerfile .
Run
docker run \
-it \
--privileged \
--ipc=host \
--network=host \
--user root \
--group-add $(getent group video | cut -d: -f3) \
--device /dev/dri \
-v /dev/dri/by-path:/dev/dri/by-path \
-v /dev/shm:/dev/shm \
-v ~/.cache/huggingface:/root/.cache/huggingface \
-p 30000:30000 \
-e "HF_TOKEN=$HF_TOKEN" \
sglang-xpu:latest /bin/bash
In container ...
sglang serve \
--model-path Intel/gemma-4-12B-it-int4-AutoRound \
--device xpu \
--attention-backend intel_xpu
Structure
