LM/llama.cpp for TurboQuant

Download source

sudo apt install git cmake
sudo apt install libssl-dev openssl
mkdir -p ~/llama.cpp/src

git clone https://github.com/TheTom/llama-cpp-turboquant ~/llama.cpp/src-tbq

List available options

grep -rn "option(GGML_SYCL" CMakeLists.txt ggml/CMakeLists.txt
grep -rn "SYCL" CMakeLists.txt ggml/CMakeLists.txt

Build

# install required headers
sudo apt install -y \
  intel-oneapi-mkl-devel \
  intel-oneapi-dnnl-devel \
  intel-oneapi-mkl-sycl-distributed-dft-devel \
  intel-oneapi-compiler-dpcpp-cpp \
  libze-dev

# set options
cmake -B build/sycl -G Ninja \
  -DGGML_SYCL=ON \
  -DGGML_SYCL_TARGET=INTEL \
  -DGGML_SYCL_DNN=ON \
  -DGGML_SYCL_F16=ON \
  -DGGML_SYCL_DEVICE_ARCH=bmg-g31 \
  -DMKL_DIR=/opt/intel/oneapi/mkl/latest/lib/cmake/mkl \
  -DCMAKE_C_COMPILER=icx \
  -DCMAKE_CXX_COMPILER=icpx \
  -DCMAKE_BUILD_TYPE=Release \
  -DCMAKE_INSTALL_RPATH="\$ORIGIN/../lib"

# build
# !!! don't use option --parallel
cmake --build build/sycl --parallel

# install 
cmake --install build/sycl --prefix ~/llama.cpp/sycl

# clean
rm -rf build-sycl

# test GPU
cd ~/llama.cpp/sycl/bin
./llama-cli --list-devices

Got It!

Available devices:
  SYCL0: Intel(R) Arc(TM) Pro B70 Graphics (32656 MiB, 32128 MiB free)

TODO: optimized

cmake -B build/sycl -G Ninja \
  -DGGML_SYCL=ON \
  -DGGML_SYCL_TARGET=INTEL \
  -DGGML_SYCL_DNN=ON \
  -DGGML_SYCL_F16=ON \
  -DGGML_SYCL_DEVICE_ARCH=bmg-g31 \
  -DGGML_NATIVE=ON \
  -DMKL_DIR=/opt/intel/oneapi/mkl/latest/lib/cmake/mkl \
  -DCMAKE_C_COMPILER=icx \
  -DCMAKE_CXX_COMPILER=icpx \
  -DCMAKE_BUILD_TYPE=Release \
  -DCMAKE_INSTALL_RPATH="\$ORIGIN/../lib" \
  -DCMAKE_CXX_FLAGS="-ffast-math -O3" \
  -DCMAKE_C_FLAGS="-ffast-math -O3" \
  -DGGML_SYCL_TURBO_QUANT=OFF

# some warnings appeared
cmake -B build/sycl -G Ninja \
  -DGGML_SYCL=ON \
  -DGGML_SYCL_TARGET=INTEL \
  -DGGML_SYCL_DNN=ON \
  -DGGML_SYCL_F16=ON \
  -DGGML_SYCL_DEVICE_ARCH=bmg-g31 \
  -DGGML_LTO=ON \
  -DGGML_NATIVE=ON \
  -DMKL_DIR=/opt/intel/oneapi/mkl/latest/lib/cmake/mkl \
  -DCMAKE_C_COMPILER=icx \
  -DCMAKE_CXX_COMPILER=icpx \
  -DCMAKE_BUILD_TYPE=Release \
  -DCMAKE_INSTALL_RPATH="\$ORIGIN/../lib" \
  -DCMAKE_CXX_FLAGS="-ffast-math -O3" \
  -DCMAKE_C_FLAGS="-ffast-math -O3"