LLM/Intel Pro Arc B70/2026-07-13

Install on Ubuntu 26.04

sudo apt update
sudo apt install -y openssh-server curl screen

Install driver

# install ppa (the most important)
sudo add-apt-repository -y ppa:kobuk-team/intel-graphics
sudo apt update
sudo apt install -y intel-opencl-icd libze-intel-gpu1 intel-media-va-driver-non-free clinfo intel-gsc

# check again
sudo clinfo

Install OpenVINO archive

sudo mkdir /opt/intel
curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2026.2.1/linux/openvino_toolkit_ubuntu24_2026.2.1.21919.ede283a88e3_x86_64.tgz --output openvino_2026.2.1.tgz
tar -xf openvino_2026.2.1.tgz
sudo mv openvino_toolkit_ubuntu24_2026.2.1.21919.ede283a88e3_x86_64 /opt/intel/openvino_2026.2.1
cd /opt/intel/openvino_2026.2.1
sudo -E ./install_dependencies/install_openvino_dependencies.sh

Apply OpenVINO environment for user

nano ./bashrc

# Setup LD Path
source /opt/intel/openvino_2026/setupvars.sh
# Use GPU first when running llama-cli, llama-server, ...
GGML_OPENVINO_DEVICE=GPU

Install llama.cpp + OpenVINO

See: https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENVINO.md#supported-devices

sudo apt install git libssl-dev
mkdir -p ~/llama.cpp/src
git clone https://github.com/ggerganov/llama.cpp ~/llama.cpp/src
cd ~/llama.cpp/src
sudo apt-get install -y opencl-c-headers ocl-icd-opencl-dev

cmake -B build-openvino \
  -DGGML_OPENVINO=ON \
  -DGGML_OPENVINO_DEVICE=GPU \
  -DCMAKE_BUILD_TYPE=Release \
  -DCMAKE_INSTALL_RPATH="\$ORIGIN/../lib" .

cmake --build build-openvino --parallel

cmake --install build-openvino --prefix ~/llama.cpp/openvino

Install llama.cpp + SYCL

sudo apt update
sudo apt install -y wget gnupg ca-certificates

# 建立存放密鑰的資料夾
sudo mkdir -m 0755 -p /etc/apt/keyrings

# 下載 Intel SW Products 公鑰並轉換為 GPG 格式
wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
| gpg --dearmor | sudo tee /etc/apt/keyrings/oneapi-archive-keyring.gpg > /dev/null

echo "deb [signed-by=/etc/apt/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \
| sudo tee /etc/apt/sources.list.d/oneAPI.list

sudo apt update
sudo apt install -y intel-oneapi-mkl-devel intel-oneapi-compiler-dpcpp-cpp
source /opt/intel/oneapi/setvars.sh

# ...

cmake --install build-sycl --prefix ~/llama.cpp/sycl

Run llama.cpp

Run Gemma 4 12B with 65535 context.

./llama-server \
  -hf google/gemma-4-12B-it-qat-q4_0-gguf \
  -c 65535 \
  -ngl 99 \
  --log-file ../logs/llama.log -lv 3 \
  --flash-attn on \
  --cache-type-k q4_0 \
  --cache-type-v q4_0 \
  --api-key sk-no-such-key \
  --host 0.0.0.0 \
  --port 9012

With tempature 0.3.

./llama-server \
  -hf google/gemma-4-12B-it-qat-q4_0-gguf \
  -c 65535 \
  -ngl 99 \
  --flash-attn on \
  --cache-type-k q4_0 \
  --cache-type-v q4_0 \
  --temp 0.3 \
  --api-key sk-no-such-key \
  --host 0.0.0.0 \
  --port 9012

./llama-server \
  -hf google/gemma-4-E4B-it-qat-q4_0-gguf \
  -c 65535 \
  -ngl 99 \
  --flash-attn on \
  --cache-type-k q4_0 \
  --cache-type-v q4_0 \
  --temp 0.3 \
  --api-key sk-no-such-key \
  --host 0.0.0.0 \
  --port 9004

CPU optimization

taskset -c 0,2,4 ./llama-server \
  -hf google/gemma-4-12B-it-qat-q4_0-gguf \
  -t 3 \
  -tb 3 \
  --mlock \
  -c 65535 \
  -ngl 99 \
  --flash-attn on \
  --cache-type-k q4_0 \
  --cache-type-v q4_0 \
  --temp 0.3 \
  --api-key sk-no-such-key \
  --host 0.0.0.0 \
  --port 9012

taskset -c 6,8,10 ./llama-server \
  -hf google/gemma-4-E4B-it-qat-q4_0-gguf \
  -t 3 \
  -tb 3 \
  --mlock \
  -c 65535 \
  -ngl 99 \
  --flash-attn on \
  --cache-type-k q4_0 \
  --cache-type-v q4_0 \
  --temp 0.3 \
  --api-key sk-no-such-key \
  --host 0.0.0.0 \
  --port 9004

Serve two models in the same tcp port

nano models.ini

# common settings
[*]
mlock = true
flash-attn = on
ctx-size = 32000
gpu-layers = 99
cache-type-k = q4_0
cache-type-v = q4_0
temp = 0.3
threads = 3
threads-batch = 3

[gemma4-12b]
hf = google/gemma-4-12B-it-qat-q4_0-gguf

[gemma4-e4b]
hf = google/gemma-4-E4B-it-qat-q4_0-gguf

./llama-server \
  --models-preset models.ini \
  --host 0.0.0.0 \
  --port 9000

But only one model can be loaded at a time.

llama-swap

wget https://github.com/mostlygeek/llama-swap/releases/download/v240/llama-swap_240_linux_amd64.tar.gz
tar -zxv -C ~/.local/bin -f llama-swap_240_linux_amd64.tar.gz
nano config.yaml

apiKeys:
  - "sk-no-such-key"

# Optional: Global timeout settings
healthCheckTimeout: 120  # Max seconds to wait for model server to boot up
logLevel: info

# Allow both models to run simultaneously without auto-unloading each other
groups:
  parallel:
    swap: false
    members:
      - gemma4-e4b
      - gemma4-12b

# Run warmup requests automatically on llama-swap startup
hooks:
  on_startup:
    preload:
      - gemma4-e4b
      - gemma4-12b

# Map model names to their host execution commands
models:
  gemma4-e4b:
    cmd: >
      taskset -c 0,2,4 ./llama.cpp/sycl/bin/llama-server
      -hf google/gemma-4-E4B-it-qat-q4_0-gguf
      -t 3 -tb 3 --mlock -c 64000 -ngl 99
      --flash-attn on --cache-type-k q4_0 --cache-type-v q4_0
      --temp 0.3
      --port ${PORT}
    ttl: 0

  gemma4-12b:
    # Use multi-line strings for readable shell commands
    cmd: >
      taskset -c 6,8,10 ./llama.cpp/sycl/bin/llama-server
      -hf google/gemma-4-12B-it-qat-q4_0-gguf
      -t 3 -tb 3 --mlock -c 64000 -ngl 99
      --flash-attn on --cache-type-k q4_0 --cache-type-v q4_0
      --temp 0.3
      --port ${PORT}
    ttl: 0

llama-swap --config config.yaml --listen 0.0.0.0:9000

Clear models

sudo apt update
sudo apt install pipx -y
pipx ensurepath
pipx install "huggingface_hub[cli]"

hf download google/gemma-4-12B-it-qat-q4_0-gguf

cd ~/.cache/huggingface
rm -rf *