disable

.devops/cublas.Dockerfile

@@ -12,7 +12,7 @@ FROM ${BASE_CUDA_DEV_CONTAINER} as build
 ARG CUDA_DOCKER_ARCH=all
 
 RUN apt-get update && \
-    apt-get install -y build-essential git cmake libsdl2-dev wget
+    apt-get install -y build-essential git cmake libsdl2-dev
 
 WORKDIR /app
 
@@ -23,6 +23,6 @@ ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
 # Enable cuBLAS
 ENV GGML_CUDA=1
 
-RUN make base.en
+RUN make
 
 ENTRYPOINT ["/app/main"]

.devops/main-cuda.Dockerfile

@@ -17,7 +17,7 @@ ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
 ENV GGML_CUDA=1
 
 RUN apt-get update && \
-    apt-get install -y build-essential libsdl2-dev wget cmake \
+    apt-get install -y build-essential libsdl2-dev \
     && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 # Ref: https://stackoverflow.com/a/53464012
@@ -25,7 +25,7 @@ ENV CUDA_MAIN_VERSION=12.3
 ENV LD_LIBRARY_PATH /usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH
 
 COPY .. .
-RUN make base.en
+RUN make
 
 FROM ${BASE_CUDA_RUN_CONTAINER} AS runtime
 ENV CUDA_MAIN_VERSION=12.3
@@ -33,7 +33,7 @@ ENV LD_LIBRARY_PATH /usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y curl ffmpeg wget cmake \
+  apt-get install -y curl ffmpeg \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY --from=build /app /app

.devops/main.Dockerfile

@@ -2,17 +2,17 @@ FROM ubuntu:22.04 AS build
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y build-essential wget cmake \
+  apt-get install -y build-essential \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY .. .
-RUN make base.en
+RUN make
 
 FROM ubuntu:22.04 AS runtime
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y curl ffmpeg libsdl2-dev wget cmake \
+  apt-get install -y curl ffmpeg libsdl2-dev \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY --from=build /app /app

.github/workflows/bindings-ruby.yml

@@ -3,41 +3,61 @@ on:
   push:
     paths:
       - bindings/ruby/**
-      - src/**/*.c
-      - src/**/*.cpp
-      - src/**/*.h
-      - src/**/*.m
-      - src/**/*.metal
-      - include/**/*.c
-      - include/**/*.cpp
-      - include/**/*.h
-      - include/**/*.m
-      - include/**/*.metal
-      - ggml/**/*.c
-      - ggml/**/*.cpp
-      - ggml/**/*.h
-      - ggml/**/*.m
-      - ggml/**/*.metal
+      - src/whisper.cpp
+      - include/whisper.h
+      - ggml/src/ggml.c
+      - ggml/src/ggml-impl.h
+      - ggml/src/ggml-aarch64.h
+      - ggml/src/ggml-aarch64.c
+      - ggml/src/ggml-alloc.c
+      - ggml/src/ggml-backend-impl.h
+      - ggml/src/ggml-backend.cpp
+      - ggml/src/ggml-common.h
+      - ggml/src/ggml-quants.h
+      - ggml/src/ggml-quants.c
+      - ggml/src/ggml-cpu-impl.h
+      - ggml/src/ggml-metal.m
+      - ggml/src/ggml-metal.metal
+      - ggml/src/ggml-blas.cpp
+      - ggml/include/ggml.h
+      - ggml/include/ggml-alloc.h
+      - ggml/include/ggml-backend.h
+      - ggml/include/ggml-cuda.h
+      - ggml/include/ggml-kompute.h
+      - ggml/include/ggml-metal.h
+      - ggml/include/ggml-sycl.h
+      - ggml/include/ggml-vulkan.h
+      - ggml/include/ggml-blas.h
       - scripts/get-flags.mk
       - examples/dr_wav.h
   pull_request:
     paths:
       - bindings/ruby/**
-      - src/**/*.c
-      - src/**/*.cpp
-      - src/**/*.h
-      - src/**/*.m
-      - src/**/*.metal
-      - include/**/*.c
-      - include/**/*.cpp
-      - include/**/*.h
-      - include/**/*.m
-      - include/**/*.metal
-      - ggml/**/*.c
-      - ggml/**/*.cpp
-      - ggml/**/*.h
-      - ggml/**/*.m
-      - ggml/**/*.metal
+      - src/whisper.cpp
+      - include/whisper.h
+      - ggml/src/ggml.c
+      - ggml/src/ggml-impl.h
+      - ggml/src/ggml-aarch64.h
+      - ggml/src/ggml-aarch64.c
+      - ggml/src/ggml-alloc.c
+      - ggml/src/ggml-backend-impl.h
+      - ggml/src/ggml-backend.cpp
+      - ggml/src/ggml-common.h
+      - ggml/src/ggml-quants.h
+      - ggml/src/ggml-quants.c
+      - ggml/src/ggml-cpu-impl.h
+      - ggml/src/ggml-metal.m
+      - ggml/src/ggml-metal.metal
+      - ggml/src/ggml-blas.cpp
+      - ggml/include/ggml.h
+      - ggml/include/ggml-alloc.h
+      - ggml/include/ggml-backend.h
+      - ggml/include/ggml-cuda.h
+      - ggml/include/ggml-kompute.h
+      - ggml/include/ggml-metal.h
+      - ggml/include/ggml-sycl.h
+      - ggml/include/ggml-vulkan.h
+      - ggml/include/ggml-blas.h
       - scripts/get-flags.mk
       - examples/dr_wav.h
 
@@ -50,6 +70,6 @@ jobs:
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.1'
+          ruby-version: '3.0'
       - uses: actions/checkout@v4
       - run: rake test

.github/workflows/build.yml

@@ -28,9 +28,9 @@ jobs:
             -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
             set -e
             apt update
-            apt install -y build-essential libsdl2-dev cmake
-            cmake -B build
-            cmake --build build --config Release -j $(nproc)'
+            apt install -y build-essential libsdl2-dev
+            make
+            make stream'
 
   macOS-latest:
     runs-on: macOS-latest
@@ -42,30 +42,30 @@ jobs:
       - name: Dependencies
         run: |
           brew update
-          brew install sdl2 cmake
+          brew install sdl2
 
       - name: Build
         run: |
-          cmake -B build
-          cmake --build build --config Release
+          make
+          make stream
 
-#  freeBSD-latest:
-#    runs-on: macos-12
-#
-#    steps:
-#      - name: Clone
-#        uses: actions/checkout@v4
-#
-#      - name: Build
-#        uses: cross-platform-actions/action@v0.24.0
-#        with:
-#          operating_system: freebsd
-#          version: '13.3'
-#          run: |
-#            sudo pkg update
-#            sudo pkg install -y gmake sdl2 cmake
-#            cmake -B build
-#            cmake --build build --config Release
+  freeBSD-latest:
+    runs-on: macos-12
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+
+      - name: Build
+        uses: cross-platform-actions/action@v0.24.0
+        with:
+          operating_system: freebsd
+          version: '13.3'
+          run: |
+            sudo pkg update
+            sudo pkg install -y gmake sdl2
+            gmake
+            gmake stream
 
   ubuntu-latest-gcc:
     runs-on: ubuntu-latest
@@ -280,10 +280,25 @@ jobs:
             mingw-w64-${{matrix.env}}-SDL2
             mingw-w64-${{matrix.env}}-openblas
 
+      - name: Build using make
+        shell: msys2 {0}
+        run: |
+            make -j $(nproc)
+
+      - name: Clean after building using make
+        shell: msys2 {0}
+        run: |
+            make clean
+
+      - name: Build using make w/ OpenBLAS
+        shell: msys2 {0}
+        run: |
+            make GGML_OPENBLAS=1 -j $(nproc)
+
       - name: Build using CMake
         shell: msys2 {0}
         run: |
-            cmake -B build -DWHISPER_SDL2=ON
+            cmake -B build
             cmake --build build --config ${{ matrix.build }} -j $(nproc)
 
       - name: Clean after building using CMake
@@ -430,72 +445,71 @@ jobs:
           name: whisper-blas-bin-${{ matrix.arch }}
           path: build/bin/${{ matrix.build }}
 
-# TODO: fix and re-enable
-#  windows-cublas:
-#    runs-on: windows-2019
-#
-#    strategy:
-#      matrix:
-#        build: [Release]
-#        arch: [x64]
-#        cublas: [ON]
-#        sdl2: [ON]
-#        cuda-toolkit: [12.2.0, 11.8.0]
-#        include:
-#          - arch: x64
-#            s2arc: x64
-#          - sdl2: ON
-#            s2ver: 2.28.5
-#
-#    steps:
-#      - name: Clone
-#        uses: actions/checkout@v4
-#
-#      - name: Add msbuild to PATH
-#        uses: microsoft/setup-msbuild@v2
-#
-#      - name: Install CUDA Toolkit
-#        id: cuda-toolkit
-#        uses: Jimver/cuda-toolkit@v0.2.15
-#        with:
-#          cuda: '${{ matrix.cuda-toolkit }}'
-#
-#      - name: Fetch SDL2 and set SDL2_DIR
-#        if: matrix.sdl2 == 'ON'
-#        run: |
-#          C:/msys64/usr/bin/wget.exe -qO sdl2.zip https://github.com/libsdl-org/SDL/releases/download/release-${{ matrix.s2ver }}/SDL2-devel-${{ matrix.s2ver }}-VC.zip
-#          7z x sdl2.zip
-#          echo "SDL2_DIR=$env:GITHUB_WORKSPACE/SDL2-${{ matrix.s2ver }}/cmake" >> $env:GITHUB_ENV
-#
-#      - name: Configure
-#        run: >
-#          cmake -S . -B ./build -A ${{ matrix.arch }}
-#          -DCMAKE_BUILD_TYPE=${{ matrix.build }}
-#          -DGGML_CUDA=${{ matrix.cublas }}
-#          -DWHISPER_SDL2=${{ matrix.sdl2 }}
-#
-#      - name: Build ${{ matrix.cuda-toolkit }}
-#        run: |
-#          cd ./build
-#          cmake --build . --config ${{ matrix.build }}
-#
-#      - name: Copy CUDA DLLs
-#        run: >
-#          Copy-Item -PassThru
-#          -Path "${{ steps.cuda-toolkit.outputs.CUDA_PATH }}/bin/*.dll"
-#          -Include cudart64_*,cublas64_*,cublasLt64_*
-#          -Destination build/bin/${{ matrix.build }}
-#
-#      - name: Copy SDL2.dll
-#        if: matrix.sdl2 == 'ON'
-#        run: copy "$env:SDL2_DIR/../lib/${{ matrix.s2arc }}/SDL2.dll" build/bin/${{ matrix.build }}
-#
-#      - name: Upload binaries
-#        if: matrix.sdl2 == 'ON'
-#        uses: actions/upload-artifact@v4
-#        with:
-#          name: whisper-cublas-${{ matrix.cuda-toolkit }}-bin-${{ matrix.arch }}
-#          path: build/bin/${{ matrix.build }}
+  windows-cublas:
+    runs-on: windows-2019
+
+    strategy:
+      matrix:
+        build: [Release]
+        arch: [x64]
+        cublas: [ON]
+        sdl2: [ON]
+        cuda-toolkit: [12.2.0, 11.8.0]
+        include:
+          - arch: x64
+            s2arc: x64
+          - sdl2: ON
+            s2ver: 2.28.5
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+
+      - name: Add msbuild to PATH
+        uses: microsoft/setup-msbuild@v2
+
+      - name: Install CUDA Toolkit
+        id: cuda-toolkit
+        uses: Jimver/cuda-toolkit@v0.2.15
+        with:
+          cuda: '${{ matrix.cuda-toolkit }}'
+
+      - name: Fetch SDL2 and set SDL2_DIR
+        if: matrix.sdl2 == 'ON'
+        run: |
+          C:/msys64/usr/bin/wget.exe -qO sdl2.zip https://github.com/libsdl-org/SDL/releases/download/release-${{ matrix.s2ver }}/SDL2-devel-${{ matrix.s2ver }}-VC.zip
+          7z x sdl2.zip
+          echo "SDL2_DIR=$env:GITHUB_WORKSPACE/SDL2-${{ matrix.s2ver }}/cmake" >> $env:GITHUB_ENV
+
+      - name: Configure
+        run: >
+          cmake -S . -B ./build -A ${{ matrix.arch }}
+          -DCMAKE_BUILD_TYPE=${{ matrix.build }}
+          -DGGML_CUDA=${{ matrix.cublas }}
+          -DWHISPER_SDL2=${{ matrix.sdl2 }}
+
+      - name: Build ${{ matrix.cuda-toolkit }}
+        run: |
+          cd ./build
+          cmake --build . --config ${{ matrix.build }}
+
+      - name: Copy CUDA DLLs
+        run: >
+          Copy-Item -PassThru
+          -Path "${{ steps.cuda-toolkit.outputs.CUDA_PATH }}/bin/*.dll"
+          -Include cudart64_*,cublas64_*,cublasLt64_*
+          -Destination build/bin/${{ matrix.build }}
+
+      - name: Copy SDL2.dll
+        if: matrix.sdl2 == 'ON'
+        run: copy "$env:SDL2_DIR/../lib/${{ matrix.s2arc }}/SDL2.dll" build/bin/${{ matrix.build }}
+
+      - name: Upload binaries
+        if: matrix.sdl2 == 'ON'
+        uses: actions/upload-artifact@v4
+        with:
+          name: whisper-cublas-${{ matrix.cuda-toolkit }}-bin-${{ matrix.arch }}
+          path: build/bin/${{ matrix.build }}
 
   emscripten:
     runs-on: ubuntu-latest
@@ -519,7 +533,7 @@ jobs:
           emcmake cmake . -DCMAKE_BUILD_TYPE=${{ matrix.build }}
           make
 
-  ios-xcode-build:
+  ios:
     runs-on: macos-latest
 
     strategy:
@@ -527,7 +541,7 @@ jobs:
         build: [Release]
 
     steps:
-      - name: Checkout code
+      - name: Clone
         uses: actions/checkout@v4
 
       - name: Configure
@@ -535,34 +549,12 @@ jobs:
           cp models/for-tests-ggml-base.en.bin models/ggml-base.en.bin
           mkdir models/ggml-base.en-encoder.mlmodelc
 
-      - name: Build
-        id: cmake_build
-        run: |
-          sysctl -a
-          mkdir build
-          cd build
-          cmake -G Xcode .. \
-            -DGGML_METAL_USE_BF16=ON \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DWHISPER_BUILD_EXAMPLES=OFF \
-            -DWHISPER_BUILD_TESTS=OFF \
-            -DWHISPER_BUILD_SERVER=OFF \
-            -DCMAKE_SYSTEM_NAME=iOS \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
-            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
-          cmake --build . --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
-          sudo cmake --install . --config Release
-
-      - name: xcodebuild for swift package
-        id: xcodebuild
-        run: |
-          xcodebuild -scheme whisper-Package -destination 'generic/platform=iOS'
-
-#- name: Build objc example
-#  run: xcodebuild -project examples/whisper.objc/whisper.objc.xcodeproj -scheme whisper.objc -configuration ${{ matrix.build }} -sdk iphoneos build
+# TODO: disabled because it fails for some reason with Github Actions
+#      - name: Build objc example
+#        run: xcodebuild -project examples/whisper.objc/whisper.objc.xcodeproj -scheme whisper.objc -configuration ${{ matrix.build }} -sdk iphonesimulator build
 
       - name: Build swiftui example
-        run: xcodebuild -project examples/whisper.swiftui/whisper.swiftui.xcodeproj -scheme WhisperCppDemo -configuration ${{ matrix.build }} -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' build
+        run: xcodebuild -project examples/whisper.swiftui/whisper.swiftui.xcodeproj -scheme WhisperCppDemo -configuration ${{ matrix.build }} -sdk iphonesimulator build
 
   android:
     runs-on: ubuntu-latest
@@ -673,6 +665,5 @@ jobs:
       - name: Test quantize
         run: |
           ./models/download-ggml-model.sh tiny.en
-          cmake -B build
-          cmake --build build --config Release
-          ./build/bin/quantize models/ggml-tiny.en.bin models/ggml-tiny.en-q4_0.bin q4_0
+          make quantize
+          ./quantize models/ggml-tiny.en.bin models/ggml-tiny.en-q4_0.bin q4_0

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.5) # for add_link_options and implicit target directories.
 project("whisper.cpp" C CXX)
-project("whisper.cpp" VERSION 1.7.3)
+project("whisper.cpp" VERSION 1.7.2)
 include(CheckIncludeFileCXX)
 
 set(SOVERSION 1)

Package.swift

@@ -14,6 +14,55 @@ let package = Package(
         .library(name: "whisper", targets: ["whisper"]),
     ],
     targets: [
-        .systemLibrary(name: "whisper", pkgConfig: "whisper"),
-    ]
+        .target(
+            name: "whisper",
+            path: ".",
+            exclude: [
+               "build",
+               "bindings",
+               "cmake",
+               "examples",
+               "scripts",
+               "models",
+               "samples",
+               "tests",
+               "CMakeLists.txt",
+               "Makefile",
+               "ggml/src/ggml-metal/ggml-metal-embed.metal"
+            ],
+            sources: [
+                "ggml/src/ggml.c",
+                "src/whisper.cpp",
+                "ggml/src/ggml-aarch64.c",
+                "ggml/src/ggml-alloc.c",
+                "ggml/src/ggml-backend.cpp",
+                "ggml/src/ggml-backend-reg.cpp",
+                "ggml/src/ggml-cpu/ggml-cpu.c",
+                "ggml/src/ggml-cpu/ggml-cpu.cpp",
+                "ggml/src/ggml-cpu/ggml-cpu-aarch64.c",
+                "ggml/src/ggml-cpu/ggml-cpu-quants.c",
+                "ggml/src/ggml-quants.c",
+                "ggml/src/ggml-threading.cpp",
+                "ggml/src/ggml-metal/ggml-metal.m"
+            ],
+            resources: [.process("ggml/src/ggml-metal/ggml-metal.metal")],
+            publicHeadersPath: "spm-headers",
+            cSettings: [
+                .unsafeFlags(["-Wno-shorten-64-to-32", "-O3", "-DNDEBUG"]),
+                .unsafeFlags(["-fno-objc-arc"]),
+                .headerSearchPath("ggml/src"),
+                .define("GGML_USE_ACCELERATE"),
+                .define("GGML_USE_METAL")
+                // NOTE: NEW_LAPACK will required iOS version 16.4+
+                // We should consider add this in the future when we drop support for iOS 14
+                // (ref: ref: https://developer.apple.com/documentation/accelerate/1513264-cblas_sgemm?language=objc)
+                // .define("ACCELERATE_NEW_LAPACK"),
+                // .define("ACCELERATE_LAPACK_ILP64")
+            ],
+            linkerSettings: [
+                .linkedFramework("Accelerate")
+            ]
+        )
+    ],
+    cxxLanguageStandard: .cxx11
 )

README.md

@@ -7,7 +7,7 @@
 [![Conan Center](https://shields.io/conan/v/whisper-cpp)](https://conan.io/center/whisper-cpp)
 [![npm](https://img.shields.io/npm/v/whisper.cpp.svg)](https://www.npmjs.com/package/whisper.cpp/)
 
-Stable: [v1.7.3](https://github.com/ggerganov/whisper.cpp/releases/tag/v1.7.3) / [Roadmap | F.A.Q.](https://github.com/ggerganov/whisper.cpp/discussions/126)
+Stable: [v1.7.2](https://github.com/ggerganov/whisper.cpp/releases/tag/v1.7.2) / [Roadmap | F.A.Q.](https://github.com/ggerganov/whisper.cpp/discussions/126)
 
 High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisper) automatic speech recognition (ASR) model:
 
@@ -89,11 +89,10 @@ Now build the [main](examples/main) example and transcribe an audio file like th
 
 ```bash
 # build the main example
-cmake -B build
-cmake --build build --config Release
+make -j
 
 # transcribe an audio file
-./build/bin/main -f samples/jfk.wav
+./main -f samples/jfk.wav
 ```
 
 ---
@@ -266,12 +265,11 @@ Here are the steps for creating and using a quantized model:
 
 ```bash
 # quantize a model with Q5_0 method
-cmake -B build
-cmake --build build --config Release
-./build/bin/quantize models/ggml-base.en.bin models/ggml-base.en-q5_0.bin q5_0
+make -j quantize
+./quantize models/ggml-base.en.bin models/ggml-base.en-q5_0.bin q5_0
 
 # run the examples as usual, specifying the quantized model file
-./build/bin/main -m models/ggml-base.en-q5_0.bin ./samples/gb0.wav
+./main -m models/ggml-base.en-q5_0.bin ./samples/gb0.wav
 ```
 
 ## Core ML support
@@ -305,6 +303,10 @@ speed-up - more than x3 faster compared with CPU-only execution. Here are the in
 - Build `whisper.cpp` with Core ML support:
 
   ```bash
+  # using Makefile
+  make clean
+  WHISPER_COREML=1 make -j
+
   # using CMake
   cmake -B build -DWHISPER_COREML=1
   cmake --build build -j --config Release
@@ -424,8 +426,8 @@ First, make sure you have installed `cuda`: https://developer.nvidia.com/cuda-do
 Now build `whisper.cpp` with CUDA support:
 
 ```
-cmake -B build -DGGML_CUDA=1
-cmake --build build -j --config Release
+make clean
+GGML_CUDA=1 make -j
 ```
 
 ## Vulkan GPU support
@@ -434,8 +436,8 @@ First, make sure your graphics card driver provides support for Vulkan API.
 
 Now build `whisper.cpp` with Vulkan support:
 ```
-cmake -B build -DGGML_VULKAN=1
-cmake --build build -j --config Release
+make clean
+make GGML_VULKAN=1 -j
 ```
 
 ## BLAS CPU support via OpenBLAS
@@ -446,13 +448,28 @@ First, make sure you have installed `openblas`: https://www.openblas.net/
 Now build `whisper.cpp` with OpenBLAS support:
 
 ```
-cmake -B build -DGGML_BLAS=1
-cmake --build build -j --config Release
+make clean
+GGML_OPENBLAS=1 make -j
+```
+
+## BLAS CPU support via Intel MKL
+
+Encoder processing can be accelerated on the CPU via the BLAS compatible interface of Intel's Math Kernel Library.
+First, make sure you have installed Intel's MKL runtime and development packages: https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl-download.html
+
+Now build `whisper.cpp` with Intel MKL BLAS support:
+
+```
+source /opt/intel/oneapi/setvars.sh
+mkdir build
+cd build
+cmake -DWHISPER_MKL=ON ..
+WHISPER_MKL=1 make -j
 ```
 
 ## Ascend NPU support
 
-Ascend NPU provides inference acceleration via [`CANN`](https://www.hiascend.com/en/software/cann) and AI cores.
+Ascend NPU provides inference acceleration via [`CANN`](https://www.hiascend.com/en/software/cann) and AI cores. 
 
 First, check if your Ascend NPU device is supported:
 
@@ -466,8 +483,10 @@ Then, make sure you have installed [`CANN toolkit`](https://www.hiascend.com/en/
 Now build `whisper.cpp` with CANN support:
 
 ```
-cmake -B build -DGGML_CANN=1
-cmake --build build -j --config Release
+mkdir build
+cd build
+cmake .. -D GGML_CANN=on
+make -j
 ```
 
 Run the inference examples as usual, for example:
@@ -617,9 +636,8 @@ The [stream](examples/stream) tool samples the audio every half a second and run
 More info is available in [issue #10](https://github.com/ggerganov/whisper.cpp/issues/10).
 
 ```bash
-cmake -B build
-cmake --build build --config Release
-./build/bin/stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
+make stream -j
+./stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
 ```
 
 https://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a80f-28ba83be7d09.mp4

Sources/whisper/module.modulemap

@@ -1,5 +0,0 @@
-module whisper [system] {
-    header "whisper.h"
-    link "whisper"
-    export *
-}

Sources/whisper/whisper.h

@@ -1,4 +0,0 @@
-#pragma once
-
-#include <whisper.h>
-

bindings/java/README.md

@@ -67,5 +67,5 @@ copy /y ..\..\build\bin\Release\whisper.dll build\generated\resources\main\win32
 
 ## License
 
-The license for the Java bindings is the same as the license for the rest of the whisper.cpp project, which is the MIT License. See the `LICENSE` file for more details.
+The license for the Go bindings is the same as the license for the rest of the whisper.cpp project, which is the MIT License. See the `LICENSE` file for more details.
 

bindings/javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "whisper.cpp",
-  "version": "1.7.3",
+  "version": "1.7.2",
   "description": "Whisper speech recognition",
   "main": "whisper.js",
   "scripts": {

bindings/ruby/.gitignore

@@ -1,5 +1,3 @@
 LICENSE
 pkg/
-lib/whisper.so
-lib/whisper.bundle
-lib/whisper.dll
+lib/whisper.*

bindings/ruby/README.md

@@ -22,7 +22,7 @@ Usage
 ```ruby
 require "whisper"
 
-whisper = Whisper::Context.new("base")
+whisper = Whisper::Context.new("path/to/model.bin")
 
 params = Whisper::Params.new
 params.language = "en"
@@ -41,66 +41,21 @@ end
 
 ### Preparing model ###
 
-Some models are prepared up-front:
+Use script to download model file(s):
 
-```ruby
-base_en = Whisper::Model.pre_converted_models["base.en"]
-whisper = Whisper::Context.new(base_en)
+```bash
+git clone https://github.com/ggerganov/whisper.cpp.git
+cd whisper.cpp
+sh ./models/download-ggml-model.sh base.en
 ```
 
-At first time you use a model, it is downloaded automatically. After that, downloaded cached file is used. To clear cache, call `#clear_cache`:
-
-```ruby
-Whisper::Model.pre_converted_models["base"].clear_cache
-```
-
-You also can use shorthand for pre-converted models:
-
-```ruby
-whisper = Whisper::Context.new("base.en")
-```
-
-You can see the list of prepared model names by `Whisper::Model.preconverted_models.keys`:
-
-```ruby
-puts Whisper::Model.preconverted_model_names
-# tiny
-# tiny.en
-# tiny-q5_1
-# tiny.en-q5_1
-# tiny-q8_0
-# base
-# base.en
-# base-q5_1
-# base.en-q5_1
-# base-q8_0
-#   :
-#   :
-```
-
-You can also use local model files you prepared:
-
-```ruby
-whisper = Whisper::Context.new("path/to/your/model.bin")
-```
-
-Or, you can download model files:
-
-```ruby
-model_uri = Whisper::Model::URI.new("http://example.net/uri/of/your/model.bin")
-whisper = Whisper::Context.new(model_uri)
-```
-
-See [models][] page for details.
+There are some types of models. See [models][] page for details.
 
 ### Preparing audio file ###
 
 Currently, whisper.cpp accepts only 16-bit WAV files.
 
-API
----
-
-### Segments ###
+### API ###
 
 Once `Whisper::Context#transcribe` called, you can retrieve segments by `#each_segment`:
 
@@ -130,6 +85,13 @@ end
 You can also add hook to params called on new segment:
 
 ```ruby
+def format_time(time_ms)
+  sec, decimal_part = time_ms.divmod(1000)
+  min, sec = sec.divmod(60)
+  hour, min = min.divmod(60)
+  "%02d:%02d:%02d.%03d" % [hour, min, sec, decimal_part]
+end
+
 # Add hook before calling #transcribe
 params.on_new_segment do |segment|
   line = "[%{st} --> %{ed}] %{text}" % {
@@ -145,12 +107,10 @@ whisper.transcribe("path/to/audio.wav", params)
 
 ```
 
-### Models ###
-
 You can see model information:
 
 ```ruby
-whisper = Whisper::Context.new("base")
+whisper = Whisper::Context.new("path/to/model.bin")
 model = whisper.model
 
 model.n_vocab # => 51864
@@ -168,8 +128,6 @@ model.type # => "base"
 
 ```
 
-### Logging ###
-
 You can set log callback:
 
 ```ruby
@@ -199,29 +157,9 @@ Using this feature, you are also able to suppress log:
 Whisper.log_set ->(level, buffer, user_data) {
   # do nothing
 }, nil
-Whisper::Context.new("base")
+Whisper::Context.new(MODEL)
 ```
 
-### Low-level API to transcribe ###
-
-You can also call `Whisper::Context#full` and `#full_parallel` with a Ruby array as samples. Although `#transcribe` with audio file path is recommended because it extracts PCM samples in C++ and is fast, `#full` and `#full_parallel` give you flexibility.
-
-```ruby
-require "whisper"
-require "wavefile"
-
-reader = WaveFile::Reader.new("path/to/audio.wav", WaveFile::Format.new(:mono, :float, 16000))
-samples = reader.enum_for(:each_buffer).map(&:samples).flatten
-
-whisper = Whisper::Context.new("base")
-whisper.full(Whisper::Params.new, samples)
-whisper.each_segment do |segment|
-  puts segment.text
-end
-```
-
-The second argument `samples` may be an array, an object with `length` method, or a MemoryView. If you can prepare audio data as C array and export it as a MemoryView, whispercpp accepts and works with it with zero copy.
-
 License
 -------
 

bindings/ruby/Rakefile

@@ -1,30 +1,39 @@
 require 'rake/clean'
 require "bundler/gem_tasks"
+require "pathname"
+require "yaml"
 require "rake/testtask"
-require_relative "extsources"
 
+extsources = YAML.load_file("extsources.yaml")
 SOURCES = FileList[]
-
-EXTSOURCES.each do |src|
+extsources.each do |src|
   basename = src.pathmap("%f")
-  dest = basename == "LICENSE" ? basename : src.pathmap("%{../..,ext}p")
-  dir = dest.pathmap("%d")
+  dest = basename == "LICENSE" ? basename : basename.pathmap("ext/%f")
   file src
-  directory dir
-  file dest => [src, dir] do |t|
+  file dest => src do |t|
     cp t.source, t.name
   end
   SOURCES.include dest
 end
-
 CLEAN.include SOURCES
-CLEAN.include FileList["ext/*.o", "ext/*.metal", "ext/whisper.{so,bundle,dll}"]
+CLEAN.include FileList[
+                "ext/*.o",
+                "ext/*.metal",
+                "ext/whisper.{so,bundle,dll}",
+                "ext/depend"
+              ]
 
-task build: ["ext/Makefile", "ext/ruby_whisper.h", "ext/ruby_whisper.cpp", "whispercpp.gemspec"]
+task build: FileList[
+       "ext/Makefile",
+       "ext/ruby_whisper.h",
+       "ext/ruby_whisper.cpp",
+       "whispercpp.gemspec",
+     ]
 
 directory "pkg"
 CLOBBER.include "pkg"
 
+TEST_MODEL = "../../models/ggml-base.en.bin"
 LIB_NAME = "whisper".ext(RbConfig::CONFIG["DLEXT"])
 SO_FILE = File.join("ext", LIB_NAME)
 LIB_FILE = File.join("lib", LIB_NAME)
@@ -40,25 +49,20 @@ file SO_FILE => "ext/Makefile" do |t|
     sh "make"
   end
 end
-CLEAN.include SO_FILE
+CLEAN.include LIB_FILE
 
 directory "lib"
 file LIB_FILE => [SO_FILE, "lib"] do |t|
   copy t.source, t.name
 end
-CLEAN.include LIB_FILE
 
 Rake::TestTask.new do |t|
   t.test_files = FileList["tests/test_*.rb"]
 end
+task test: [TEST_MODEL, LIB_FILE]
 
-TEST_MEMORY_VIEW = "tests/jfk_reader/jfk_reader.#{RbConfig::CONFIG['DLEXT']}"
-file TEST_MEMORY_VIEW => "tests/jfk_reader/jfk_reader.c" do |t|
-  Dir.chdir "tests/jfk_reader" do
-    ruby "extconf.rb"
-    sh "make"
+file TEST_MODEL do
+  Dir.chdir "../.." do
+    sh "./models/download-ggml-model.sh base.en"
   end
 end
-CLEAN.include "tests/jfk_reader/jfk_reader.{o,#{RbConfig::CONFIG['DLEXT']}}"
-
-task test: [LIB_FILE, TEST_MEMORY_VIEW]

bindings/ruby/ext/.gitignore

@@ -1,13 +1,35 @@
 Makefile
-whisper.so
+ggml.c
+ggml.h
+ggml-alloc.c
+ggml-alloc.h
+ggml-aarch64.c
+ggml-aarch64.h
+ggml-backend.cpp
+ggml-backend-impl.h
+ggml-backend.c
+ggml-backend.h
+ggml-common.h
+ggml-cpu-impl.h
+ggml-metal.m
+ggml-metal.metal
+ggml-metal-embed.metal
+ggml-blas.cpp
+ggml-cuda.h
+ggml-impl.h
+ggml-kompute.h
+ggml-metal.h
+ggml-opencl.h
+ggml-quants.c
+ggml-quants.h
+ggml-sycl.h
+ggml-vulkan.h
+ggml-blas.h
+get-flags.mk
+whisper.cpp
+whisper.h
+dr_wav.h
+depend
 whisper.bundle
+whisper.so
 whisper.dll
-scripts/get-flags.mk
-*.o
-*.c
-*.cpp
-*.h
-*.m
-*.metal
-!ruby_whisper.cpp
-!ruby_whisper.h

bindings/ruby/ext/cpu.mk

@@ -1,9 +0,0 @@
-ggml/src/ggml-cpu/ggml-cpu-cpp.o: \
-	ggml/src/ggml-cpu/ggml-cpu.cpp \
-	ggml/include/ggml-backend.h \
-	ggml/include/ggml.h \
-	ggml/include/ggml-alloc.h \
-	ggml/src/ggml-backend-impl.h \
-	ggml/include/ggml-cpu.h \
-	ggml/src/ggml-impl.h
-	$(CXX) $(CXXFLAGS)   -c $< -o $@

bindings/ruby/ext/extconf.rb

@@ -1,7 +1,7 @@
 require 'mkmf'
 
 # need to use c++ compiler flags
-$CXXFLAGS << ' -std=c++17'
+$CXXFLAGS << ' -std=c++11'
 
 $LDFLAGS << ' -lstdc++'
 
@@ -35,10 +35,10 @@ if $GGML_METAL
   $GGML_METAL_EMBED_LIBRARY = true
 end
 
-$MK_CPPFLAGS = '-Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -Iexamples'
+$MK_CPPFLAGS = ''
 $MK_CFLAGS   = '-std=c11   -fPIC'
-$MK_CXXFLAGS = '-std=c++17 -fPIC'
-$MK_NVCCFLAGS = '-std=c++17'
+$MK_CXXFLAGS = '-std=c++11 -fPIC'
+$MK_NVCCFLAGS = '-std=c++11'
 $MK_LDFLAGS = ''
 
 $OBJ_GGML = []
@@ -111,6 +111,11 @@ unless ENV['RISCV']
     $MK_CFLAGS     << ' -march=native -mtune=native'
     $HOST_CXXFLAGS << ' -march=native -mtune=native'
   end
+
+  if $UNAME_M.match? /aarch64.*/
+    $MK_CFLAGS   << ' -mcpu=native'
+    $MK_CXXFLAGS << ' -mcpu=native'
+  end
 else
   $MK_CFLAGS   << ' -march=rv64gcv -mabi=lp64d'
   $MK_CXXFLAGS << ' -march=rv64gcv -mabi=lp64d'
@@ -118,11 +123,11 @@ end
 
 unless ENV['GGML_NO_ACCELERATE']
   if $UNAME_S == 'Darwin'
-    $MK_CPPFLAGS << ' -DGGML_USE_ACCELERATE -DGGML_USE_BLAS -DGGML_BLAS_USE_ACCELERATE'
+    $MK_CPPFLAGS << ' -DGGML_USE_ACCELERATE -DGGML_USE_BLAS'
     $MK_CPPFLAGS << ' -DACCELERATE_NEW_LAPACK'
     $MK_CPPFLAGS << ' -DACCELERATE_LAPACK_ILP64'
     $MK_LDFLAGS  << ' -framework Accelerate'
-    $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
+    $OBJ_GGML    << 'ggml-blas.o'
   end
 end
 
@@ -130,20 +135,20 @@ if ENV['GGML_OPENBLAS']
   $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas`.chomp}"
   $MK_CFLAGS   << " #{`pkg-config --cflags-only-other openblas)`.chomp}"
   $MK_LDFLAGS  << " #{`pkg-config --libs openblas`}"
-  $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
+  $OBJ_GGML    << 'ggml-blas.o'
 end
 
 if ENV['GGML_OPENBLAS64']
   $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas64`.chomp}"
   $MK_CFLAGS   << " #{`pkg-config --cflags-only-other openblas64)`.chomp}"
   $MK_LDFLAGS  << " #{`pkg-config --libs openblas64`}"
-  $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
+  $OBJ_GGML    << 'ggml-blas.o'
 end
 
 if $GGML_METAL
   $MK_CPPFLAGS << ' -DGGML_USE_METAL'
   $MK_LDFLAGS  << ' -framework Foundation -framework Metal -framework MetalKit'
-  $OBJ_GGML    << 'ggml/src/ggml-metal/ggml-metal.o'
+  $OBJ_GGML    << 'ggml-metal.o'
 
   if ENV['GGML_METAL_NDEBUG']
     $MK_CPPFLAGS << ' -DGGML_METAL_NDEBUG'
@@ -151,27 +156,20 @@ if $GGML_METAL
 
   if $GGML_METAL_EMBED_LIBRARY
     $MK_CPPFLAGS << ' -DGGML_METAL_EMBED_LIBRARY'
-    $OBJ_GGML    << 'ggml/src/ggml-metal/ggml-metal-embed.o'
+    $OBJ_GGML    << 'ggml-metal-embed.o'
   end
 end
 
 $OBJ_GGML <<
-  'ggml/src/ggml.o' <<
-  'ggml/src/ggml-alloc.o' <<
-  'ggml/src/ggml-backend.o' <<
-  'ggml/src/ggml-backend-reg.o' <<
-  'ggml/src/ggml-opt.o' <<
-  'ggml/src/ggml-quants.o' <<
-  'ggml/src/ggml-threading.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-cpp.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-aarch64.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-hbm.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-quants.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-traits.o'
+  'ggml.o' <<
+  'ggml-cpu.o' <<
+  'ggml-alloc.o' <<
+  'ggml-backend.o' <<
+  'ggml-quants.o' <<
+  'ggml-aarch64.o'
 
 $OBJ_WHISPER <<
-  'src/whisper.o'
+  'whisper.o'
 
 $objs = $OBJ_GGML + $OBJ_WHISPER + $OBJ_COMMON + $OBJ_SDL
 $objs << "ruby_whisper.o"
@@ -186,12 +184,9 @@ $LDFLAGS   = "#{$MK_LDFLAGS} #{$LDFLAGS}"
 create_makefile('whisper')
 
 File.open 'Makefile', 'a' do |file|
-  file.puts 'include scripts/get-flags.mk'
-  file.puts 'include cpu.mk'
+  file.puts 'include get-flags.mk'
 
   if $GGML_METAL
-    file.puts 'include metal.mk'
-
     if $GGML_METAL_EMBED_LIBRARY
       file.puts 'include metal-embed.mk'
     end

bindings/ruby/ext/metal-embed.mk

@@ -1,17 +1,14 @@
-ggml/src/ggml-metal/ggml-metal-embed.o: \
-	ggml/src/ggml-metal/ggml-metal.metal \
-	ggml/src/ggml-metal/ggml-metal-impl.h \
-	ggml/src/ggml-common.h
+ggml-metal-embed.o: \
+	ggml-metal.metal \
+	ggml-common.h
 	@echo "Embedding Metal library"
-	@sed -e '/__embed_ggml-common.h__/r      ggml/src/ggml-common.h'                -e '/__embed_ggml-common.h__/d'      < ggml/src/ggml-metal/ggml-metal.metal           > ggml/src/ggml-metal/ggml-metal-embed.metal.tmp
-	@sed -e '/#include "ggml-metal-impl.h"/r ggml/src/ggml-metal/ggml-metal-impl.h' -e '/#include "ggml-metal-impl.h"/d' < ggml/src/ggml-metal/ggml-metal-embed.metal.tmp > ggml/src/ggml-metal/ggml-metal-embed.metal
-	$(eval TEMP_ASSEMBLY=$(shell mktemp -d))
-	@echo ".section __DATA, __ggml_metallib"                       >  $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo ".globl _ggml_metallib_start"                            >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo "_ggml_metallib_start:"                                  >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo ".incbin \"ggml/src/ggml-metal/ggml-metal-embed.metal\"" >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo ".globl _ggml_metallib_end"                              >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo "_ggml_metallib_end:"                                    >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	$(CC) $(CFLAGS) -c $(TEMP_ASSEMBLY)/ggml-metal-embed.s -o $@
-	@rm -f ${TEMP_ASSEMBLY}/ggml-metal-embed.s
-	@rmdir ${TEMP_ASSEMBLY}
+	@sed -e '/#include "ggml-common.h"/r ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml-metal.metal > ggml-metal-embed.metal
+	$(eval TEMP_ASSEMBLY=$(shell mktemp))
+	@echo ".section __DATA, __ggml_metallib"            >  $(TEMP_ASSEMBLY)
+	@echo ".globl _ggml_metallib_start"                 >> $(TEMP_ASSEMBLY)
+	@echo "_ggml_metallib_start:"                       >> $(TEMP_ASSEMBLY)
+	@echo ".incbin \"ggml-metal-embed.metal\""          >> $(TEMP_ASSEMBLY)
+	@echo ".globl _ggml_metallib_end"                   >> $(TEMP_ASSEMBLY)
+	@echo "_ggml_metallib_end:"                         >> $(TEMP_ASSEMBLY)
+	@$(AS) $(TEMP_ASSEMBLY) -o $@
+	@rm -f ${TEMP_ASSEMBLY}

bindings/ruby/ext/metal.mk

@@ -1,6 +0,0 @@
-ggml/src/ggml-metal/ggml-metal.o: \
-	ggml/src/ggml-metal/ggml-metal.m \
-	ggml/src/ggml-metal/ggml-metal-impl.h \
-	ggml/include/ggml-metal.h \
-	ggml/include/ggml.h
-	$(CC) $(CFLAGS) -c $< -o $@

bindings/ruby/ext/ruby_whisper.cpp

@@ -1,5 +1,4 @@
 #include <ruby.h>
-#include <ruby/memory_view.h>
 #include "ruby_whisper.h"
 #define DR_WAV_IMPLEMENTATION
 #include "dr_wav.h"
@@ -36,20 +35,11 @@ extern "C" {
 VALUE mWhisper;
 VALUE cContext;
 VALUE cParams;
-VALUE eError;
-
-VALUE cSegment;
-VALUE cModel;
 
 static ID id_to_s;
 static ID id_call;
 static ID id___method__;
 static ID id_to_enum;
-static ID id_length;
-static ID id_next;
-static ID id_new;
-static ID id_to_path;
-static ID id_pre_converted_models;
 
 static bool is_log_callback_finalized = false;
 
@@ -110,13 +100,13 @@ static VALUE ruby_whisper_s_finalize_log_callback(VALUE self, VALUE id) {
  *   log_set ->(level, buffer, user_data) { ... }, user_data -> nil
  */
 static VALUE ruby_whisper_s_log_set(VALUE self, VALUE log_callback, VALUE user_data) {
-  VALUE old_callback = rb_iv_get(self, "log_callback");
+  VALUE old_callback = rb_iv_get(self, "@log_callback");
   if (!NIL_P(old_callback)) {
     rb_undefine_finalizer(old_callback);
   }
 
-  rb_iv_set(self, "log_callback", log_callback);
-  rb_iv_set(self, "user_data", user_data);
+  rb_iv_set(self, "@log_callback", log_callback);
+  rb_iv_set(self, "@user_data", user_data);
 
   VALUE finalize_log_callback = rb_funcall(mWhisper, rb_intern("method"), 1, rb_str_new2("finalize_log_callback"));
   rb_define_finalizer(log_callback, finalize_log_callback);
@@ -125,8 +115,8 @@ static VALUE ruby_whisper_s_log_set(VALUE self, VALUE log_callback, VALUE user_d
     if (is_log_callback_finalized) {
       return;
     }
-    VALUE log_callback = rb_iv_get(mWhisper, "log_callback");
-    VALUE udata = rb_iv_get(mWhisper, "user_data");
+    VALUE log_callback = rb_iv_get(mWhisper, "@log_callback");
+    VALUE udata = rb_iv_get(mWhisper, "@user_data");
     rb_funcall(log_callback, id_call, 3, INT2NUM(level), rb_str_new2(buffer), udata);
   }, nullptr);
 
@@ -191,7 +181,6 @@ static VALUE ruby_whisper_params_allocate(VALUE klass) {
   ruby_whisper_params *rwp;
   rwp = ALLOC(ruby_whisper_params);
   rwp->params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
-  rwp->diarize = false;
   rwp->new_segment_callback_container = rb_whisper_callback_container_allocate();
   rwp->progress_callback_container = rb_whisper_callback_container_allocate();
   rwp->abort_callback_container = rb_whisper_callback_container_allocate();
@@ -200,9 +189,7 @@ static VALUE ruby_whisper_params_allocate(VALUE klass) {
 
 /*
  * call-seq:
- *   new("base.en") -> Whisper::Context
  *   new("path/to/model.bin") -> Whisper::Context
- *   new(Whisper::Model::URI.new("https://example.net/uri/of/model.bin")) -> Whisper::Context
  */
 static VALUE ruby_whisper_initialize(int argc, VALUE *argv, VALUE self) {
   ruby_whisper *rw;
@@ -212,14 +199,6 @@ static VALUE ruby_whisper_initialize(int argc, VALUE *argv, VALUE self) {
   rb_scan_args(argc, argv, "01", &whisper_model_file_path);
   Data_Get_Struct(self, ruby_whisper, rw);
 
-  VALUE pre_converted_models = rb_funcall(cModel, id_pre_converted_models, 0);
-  VALUE pre_converted_model = rb_hash_aref(pre_converted_models, whisper_model_file_path);
-  if (!NIL_P(pre_converted_model)) {
-    whisper_model_file_path = pre_converted_model;
-  }
-  if (rb_respond_to(whisper_model_file_path, id_to_path)) {
-    whisper_model_file_path = rb_funcall(whisper_model_file_path, id_to_path, 0);
-  }
   if (!rb_respond_to(whisper_model_file_path, id_to_s)) {
     rb_raise(rb_eRuntimeError, "Expected file path to model to initialize Whisper::Context");
   }
@@ -565,168 +544,6 @@ VALUE ruby_whisper_model_type(VALUE self) {
   return rb_str_new2(whisper_model_type_readable(rw->context));
 }
 
-/*
- * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
- * Not thread safe for same context
- * Uses the specified decoding strategy to obtain the text.
- *
- * call-seq:
- *   full(params, samples, n_samples) -> nil
- *   full(params, samples) -> nil
- *
- * The second argument +samples+ must be an array of samples, respond to :length, or be a MemoryView of an array of float. It must be 32 bit float PCM audio data.
- */
-VALUE ruby_whisper_full(int argc, VALUE *argv, VALUE self) {
-  if (argc < 2 || argc > 3) {
-    rb_raise(rb_eArgError, "wrong number of arguments (given %d, expected 2..3)", argc);
-  }
-
-  ruby_whisper *rw;
-  ruby_whisper_params *rwp;
-  Data_Get_Struct(self, ruby_whisper, rw);
-  VALUE params = argv[0];
-  Data_Get_Struct(params, ruby_whisper_params, rwp);
-  VALUE samples = argv[1];
-  int n_samples;
-  rb_memory_view_t view;
-  const bool memory_view_available_p = rb_memory_view_available_p(samples);
-  if (argc == 3) {
-    n_samples = NUM2INT(argv[2]);
-    if (TYPE(samples) == T_ARRAY) {
-      if (RARRAY_LEN(samples) < n_samples) {
-        rb_raise(rb_eArgError, "samples length %ld is less than n_samples %d", RARRAY_LEN(samples), n_samples);
-      }
-    }
-    // Should check when samples.respond_to?(:length)?
-  } else {
-    if (TYPE(samples) == T_ARRAY) {
-      n_samples = RARRAY_LEN(samples);
-    } else if (memory_view_available_p) {
-      if (!rb_memory_view_get(samples, &view, RUBY_MEMORY_VIEW_SIMPLE)) {
-        view.obj = Qnil;
-        rb_raise(rb_eArgError, "unable to get a memory view");
-      }
-      n_samples = view.byte_size / view.item_size;
-    } else if (rb_respond_to(samples, id_length)) {
-      n_samples = NUM2INT(rb_funcall(samples, id_length, 0));
-    } else {
-      rb_raise(rb_eArgError, "samples must respond to :length or be a MemoryView of an array of flaot when n_samples is not given");
-    }
-  }
-  float * c_samples = (float *)malloc(n_samples * sizeof(float));
-  if (memory_view_available_p)  {
-    c_samples = (float *)view.data;
-  } else {
-    if (TYPE(samples) == T_ARRAY) {
-      for (int i = 0; i < n_samples; i++) {
-        c_samples[i] = RFLOAT_VALUE(rb_ary_entry(samples, i));
-      }
-    } else {
-      // TODO: use rb_block_call
-      VALUE iter = rb_funcall(samples, id_to_enum, 1, rb_str_new2("each"));
-      for (int i = 0; i < n_samples; i++) {
-        // TODO: check if iter is exhausted and raise ArgumentError appropriately
-        VALUE sample = rb_funcall(iter, id_next, 0);
-        c_samples[i] = RFLOAT_VALUE(sample);
-      }
-    }
-  }
-  const int result = whisper_full(rw->context, rwp->params, c_samples, n_samples);
-  if (0 == result) {
-    return Qnil;
-  } else {
-    rb_exc_raise(rb_funcall(eError, id_new, 1, result));
-  }
-}
-
-/*
- * Split the input audio in chunks and process each chunk separately using whisper_full_with_state()
- * Result is stored in the default state of the context
- * Not thread safe if executed in parallel on the same context.
- * It seems this approach can offer some speedup in some cases.
- * However, the transcription accuracy can be worse at the beginning and end of each chunk.
- *
- * call-seq:
- *   full_parallel(params, samples) -> nil
- *   full_parallel(params, samples, n_samples) -> nil
- *   full_parallel(params, samples, n_samples, n_processors) -> nil
- *   full_parallel(params, samples, nil, n_processors) -> nil
- */
-static VALUE ruby_whisper_full_parallel(int argc, VALUE *argv,VALUE self) {
-  if (argc < 2 || argc > 4) {
-    rb_raise(rb_eArgError, "wrong number of arguments (given %d, expected 2..3)", argc);
-  }
-
-  ruby_whisper *rw;
-  ruby_whisper_params *rwp;
-  Data_Get_Struct(self, ruby_whisper, rw);
-  VALUE params = argv[0];
-  Data_Get_Struct(params, ruby_whisper_params, rwp);
-  VALUE samples = argv[1];
-  int n_samples;
-  int n_processors;
-  rb_memory_view_t view;
-  const bool memory_view_available_p = rb_memory_view_available_p(samples);
-  switch (argc) {
-  case 2:
-    n_processors = 1;
-    break;
-  case 3:
-    n_processors = 1;
-    break;
-  case 4:
-    n_processors = NUM2INT(argv[3]);
-    break;
-  }
-  if (argc >= 3 && !NIL_P(argv[2])) {
-    n_samples = NUM2INT(argv[2]);
-    if (TYPE(samples) == T_ARRAY) {
-      if (RARRAY_LEN(samples) < n_samples) {
-        rb_raise(rb_eArgError, "samples length %ld is less than n_samples %d", RARRAY_LEN(samples), n_samples);
-      }
-    }
-    // Should check when samples.respond_to?(:length)?
-  } else if (memory_view_available_p) {
-    if (!rb_memory_view_get(samples, &view, RUBY_MEMORY_VIEW_SIMPLE)) {
-      view.obj = Qnil;
-      rb_raise(rb_eArgError, "unable to get a memory view");
-    }
-    n_samples = view.byte_size / view.item_size;
-  } else {
-    if (TYPE(samples) == T_ARRAY) {
-      n_samples = RARRAY_LEN(samples);
-    } else if (rb_respond_to(samples, id_length)) {
-      n_samples = NUM2INT(rb_funcall(samples, id_length, 0));
-    } else {
-      rb_raise(rb_eArgError, "samples must respond to :length or be a MemoryView of an array of flaot when n_samples is not given");
-    }
-  }
-  float * c_samples = (float *)malloc(n_samples * sizeof(float));
-  if (memory_view_available_p) {
-    c_samples = (float *)view.data;
-  } else {
-    if (TYPE(samples) == T_ARRAY) {
-      for (int i = 0; i < n_samples; i++) {
-        c_samples[i] = RFLOAT_VALUE(rb_ary_entry(samples, i));
-      }
-    } else {
-      // FIXME: use rb_block_call
-      VALUE iter = rb_funcall(samples, id_to_enum, 1, rb_str_new2("each"));
-      for (int i = 0; i < n_samples; i++) {
-        // TODO: check if iter is exhausted and raise ArgumentError
-        VALUE sample = rb_funcall(iter, id_next, 0);
-        c_samples[i] = RFLOAT_VALUE(sample);
-      }
-    }
-  }
-  const int result = whisper_full_parallel(rw->context, rwp->params, c_samples, n_samples, n_processors);
-  if (0 == result) {
-    return Qnil;
-  } else {
-    rb_exc_raise(rb_funcall(eError, id_new, 1, result));
-  }
-}
-
 /*
  * Number of segments.
  *
@@ -1261,25 +1078,6 @@ static VALUE ruby_whisper_params_set_logprob_thold(VALUE self, VALUE value) {
   rwp->params.logprob_thold = RFLOAT_VALUE(value);
   return value;
 }
-/*
- * call-seq:
- *   no_speech_thold -> Float
- */
-static VALUE ruby_whisper_params_get_no_speech_thold(VALUE self) {
-  ruby_whisper_params *rwp;
-  Data_Get_Struct(self, ruby_whisper_params, rwp);
-  return DBL2NUM(rwp->params.no_speech_thold);
-}
-/*
- * call-seq:
- *   no_speech_thold = threshold -> threshold
- */
-static VALUE ruby_whisper_params_set_no_speech_thold(VALUE self, VALUE value) {
-  ruby_whisper_params *rwp;
-  Data_Get_Struct(self, ruby_whisper_params, rwp);
-  rwp->params.no_speech_thold = RFLOAT_VALUE(value);
-  return value;
-}
 /*
  * Sets new segment callback, called for every newly generated text segment.
  *
@@ -1376,6 +1174,9 @@ typedef struct {
   VALUE context;
 } ruby_whisper_model;
 
+VALUE cSegment;
+VALUE cModel;
+
 static void rb_whisper_segment_mark(ruby_whisper_segment *rws) {
   rb_gc_mark(rws->context);
 }
@@ -1717,61 +1518,15 @@ static VALUE ruby_whisper_c_model_type(VALUE self) {
   return rb_str_new2(whisper_model_type_readable(rw->context));
 }
 
-static VALUE ruby_whisper_error_initialize(VALUE self, VALUE code) {
-  const int c_code = NUM2INT(code);
-  char *raw_message;
-  switch (c_code) {
-  case -2:
-    raw_message = "failed to compute log mel spectrogram";
-    break;
-  case -3:
-    raw_message = "failed to auto-detect language";
-    break;
-  case -4:
-    raw_message = "too many decoders requested";
-    break;
-  case -5:
-    raw_message = "audio_ctx is larger than the maximum allowed";
-    break;
-  case -6:
-    raw_message = "failed to encode";
-    break;
-  case -7:
-    raw_message = "whisper_kv_cache_init() failed for self-attention cache";
-    break;
-  case -8:
-    raw_message = "failed to decode";
-    break;
-  case -9:
-    raw_message = "failed to decode";
-    break;
-  default:
-    raw_message = "unknown error";
-    break;
-  }
-  const VALUE message = rb_str_new2(raw_message);
-  rb_call_super(1, &message);
-  rb_iv_set(self, "@code", code);
-
-  return self;
-}
-
-
 void Init_whisper() {
   id_to_s = rb_intern("to_s");
   id_call = rb_intern("call");
   id___method__ = rb_intern("__method__");
   id_to_enum = rb_intern("to_enum");
-  id_length = rb_intern("length");
-  id_next = rb_intern("next");
-  id_new = rb_intern("new");
-  id_to_path = rb_intern("to_path");
-  id_pre_converted_models = rb_intern("pre_converted_models");
 
   mWhisper = rb_define_module("Whisper");
   cContext = rb_define_class_under(mWhisper, "Context", rb_cObject);
   cParams  = rb_define_class_under(mWhisper, "Params", rb_cObject);
-  eError   = rb_define_class_under(mWhisper, "Error", rb_eStandardError);
 
   rb_define_const(mWhisper, "LOG_LEVEL_NONE", INT2NUM(GGML_LOG_LEVEL_NONE));
   rb_define_const(mWhisper, "LOG_LEVEL_INFO", INT2NUM(GGML_LOG_LEVEL_INFO));
@@ -1809,8 +1564,6 @@ void Init_whisper() {
   rb_define_method(cContext, "full_get_segment_t1", ruby_whisper_full_get_segment_t1, 1);
   rb_define_method(cContext, "full_get_segment_speaker_turn_next", ruby_whisper_full_get_segment_speaker_turn_next, 1);
   rb_define_method(cContext, "full_get_segment_text", ruby_whisper_full_get_segment_text, 1);
-  rb_define_method(cContext, "full", ruby_whisper_full, -1);
-  rb_define_method(cContext, "full_parallel", ruby_whisper_full_parallel, -1);
 
   rb_define_alloc_func(cParams, ruby_whisper_params_allocate);
 
@@ -1862,8 +1615,6 @@ void Init_whisper() {
   rb_define_method(cParams, "entropy_thold=", ruby_whisper_params_set_entropy_thold, 1);
   rb_define_method(cParams, "logprob_thold", ruby_whisper_params_get_logprob_thold, 0);
   rb_define_method(cParams, "logprob_thold=", ruby_whisper_params_set_logprob_thold, 1);
-  rb_define_method(cParams, "no_speech_thold", ruby_whisper_params_get_no_speech_thold, 0);
-  rb_define_method(cParams, "no_speech_thold=", ruby_whisper_params_set_no_speech_thold, 1);
 
   rb_define_method(cParams, "new_segment_callback=", ruby_whisper_params_set_new_segment_callback, 1);
   rb_define_method(cParams, "new_segment_callback_user_data=", ruby_whisper_params_set_new_segment_callback_user_data, 1);
@@ -1872,9 +1623,6 @@ void Init_whisper() {
   rb_define_method(cParams, "abort_callback=", ruby_whisper_params_set_abort_callback, 1);
   rb_define_method(cParams, "abort_callback_user_data=", ruby_whisper_params_set_abort_callback_user_data, 1);
 
-  rb_define_attr(eError, "code", true, false);
-  rb_define_method(eError, "initialize", ruby_whisper_error_initialize, 1);
-
   // High leve
   cSegment  = rb_define_class_under(mWhisper, "Segment", rb_cObject);
 

bindings/ruby/extsources.rb

@@ -1,6 +0,0 @@
-require "yaml"
-
-sources = `git ls-files -z ../..`.split("\x0")
-paths = YAML.load_file("../../.github/workflows/bindings-ruby.yml")[true]["push"]["paths"]
-paths.delete "bindings/ruby/**"
-EXTSOURCES = (Dir.glob(paths, base: "../..").collect {|path| "../../#{path}"} << "../../LICENSE") & sources

bindings/ruby/extsources.yaml

@@ -0,0 +1,31 @@
+---
+- ../../src/whisper.cpp
+- ../../include/whisper.h
+- ../../ggml/src/ggml.c
+- ../../ggml/src/ggml-cpu.c
+- ../../ggml/src/ggml-impl.h
+- ../../ggml/src/ggml-aarch64.h
+- ../../ggml/src/ggml-aarch64.c
+- ../../ggml/src/ggml-alloc.c
+- ../../ggml/src/ggml-backend-impl.h
+- ../../ggml/src/ggml-backend.cpp
+- ../../ggml/src/ggml-common.h
+- ../../ggml/src/ggml-quants.h
+- ../../ggml/src/ggml-quants.c
+- ../../ggml/src/ggml-cpu-impl.h
+- ../../ggml/src/ggml-metal.m
+- ../../ggml/src/ggml-metal.metal
+- ../../ggml/src/ggml-blas.cpp
+- ../../ggml/include/ggml.h
+- ../../ggml/include/ggml-alloc.h
+- ../../ggml/include/ggml-backend.h
+- ../../ggml/include/ggml-cpu.h
+- ../../ggml/include/ggml-cuda.h
+- ../../ggml/include/ggml-kompute.h
+- ../../ggml/include/ggml-metal.h
+- ../../ggml/include/ggml-sycl.h
+- ../../ggml/include/ggml-vulkan.h
+- ../../ggml/include/ggml-blas.h
+- ../../scripts/get-flags.mk
+- ../../examples/dr_wav.h
+- ../../LICENSE

bindings/ruby/lib/whisper.rb

@@ -1,2 +0,0 @@
-require "whisper.so"
-require "whisper/model/uri"

bindings/ruby/lib/whisper/model/uri.rb

@@ -1,157 +0,0 @@
-require "whisper.so"
-require "uri"
-require "net/http"
-require "time"
-require "pathname"
-require "io/console/size"
-
-class Whisper::Model
-  class URI
-    def initialize(uri)
-      @uri = URI(uri)
-    end
-
-    def to_path
-      cache
-      cache_path.to_path
-    end
-
-    def clear_cache
-      path = cache_path
-      path.delete if path.exist?
-    end
-
-    private
-
-    def cache_path
-      base_cache_dir/@uri.host/@uri.path[1..]
-    end
-
-    def base_cache_dir
-      base = case RUBY_PLATFORM
-             when /mswin|mingw/
-               ENV.key?("LOCALAPPDATA") ? Pathname(ENV["LOCALAPPDATA"]) : Pathname(Dir.home)/"AppData/Local"
-             when /darwin/
-               Pathname(Dir.home)/"Library/Caches"
-             else
-               ENV.key?("XDG_CACHE_HOME") ? ENV["XDG_CACHE_HOME"] : Pathname(Dir.home)/".cache"
-             end
-      base/"whisper.cpp"
-    end
-
-    def cache
-      path = cache_path
-      headers = {}
-      headers["if-modified-since"] = path.mtime.httpdate if path.exist?
-      request @uri, headers
-      path
-    end
-
-    def request(uri, headers)
-      Net::HTTP.start uri.host, uri.port, use_ssl: uri.scheme == "https" do |http|
-        request = Net::HTTP::Get.new(uri, headers)
-        http.request request do |response|
-          case response
-          when Net::HTTPNotModified
-            # noop
-          when Net::HTTPOK
-            download response
-          when Net::HTTPRedirection
-            request URI(response["location"]), headers
-          else
-            return if headers.key?("if-modified-since") # Use cache file
-
-            raise "#{response.code} #{response.message}\n#{response.body}"
-          end
-        end
-      end
-    end
-
-    def download(response)
-      path = cache_path
-      path.dirname.mkpath unless path.dirname.exist?
-      downloading_path = Pathname("#{path}.downloading")
-      size = response.content_length
-      downloading_path.open "wb" do |file|
-        downloaded = 0
-        response.read_body do |chunk|
-          file << chunk
-          downloaded += chunk.bytesize
-          show_progress downloaded, size
-        end
-      end
-      downloading_path.rename path
-    end
-
-    def show_progress(current, size)
-      return unless $stderr.tty?
-      return unless size
-
-      unless @prev
-        @prev = Time.now
-        $stderr.puts "Downloading #{@uri}"
-      end
-
-      now = Time.now
-      return if now - @prev < 1 && current < size
-
-      progress_width = 20
-      progress = current.to_f / size
-      arrow_length = progress * progress_width
-      arrow = "=" * (arrow_length - 1) + ">" + " " * (progress_width - arrow_length)
-      line = "[#{arrow}] (#{format_bytesize(current)} / #{format_bytesize(size)})"
-      padding = ' ' * ($stderr.winsize[1] - line.size)
-      $stderr.print "\r#{line}#{padding}"
-      $stderr.puts if current >= size
-      @prev = now
-    end
-
-    def format_bytesize(bytesize)
-      return "0.0 B" if bytesize.zero?
-
-      units = %w[B KiB MiB GiB TiB]
-      exp = (Math.log(bytesize) / Math.log(1024)).to_i
-      format("%.1f %s", bytesize.to_f / 1024 ** exp, units[exp])
-    end
-  end
-
-  @pre_converted_models = {}
-  %w[
-    tiny
-    tiny.en
-    tiny-q5_1
-    tiny.en-q5_1
-    tiny-q8_0
-    base
-    base.en
-    base-q5_1
-    base.en-q5_1
-    base-q8_0
-    small
-    small.en
-    small.en-tdrz
-    small-q5_1
-    small.en-q5_1
-    small-q8_0
-    medium
-    medium.en
-    medium-q5_0
-    medium.en-q5_0
-    medium-q8_0
-    large-v1
-    large-v2
-    large-v2-q5_0
-    large-v2-q8_0
-    large-v3
-    large-v3-q5_0
-    large-v3-turbo
-    large-v3-turbo-q5_0
-    large-v3-turbo-q8_0
-  ].each do |name|
-    @pre_converted_models[name] = URI.new("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-#{name}.bin")
-  end
-
-  class << self
-    attr_reader :pre_converted_models
-  end
-end

bindings/ruby/tests/helper.rb

@@ -1,7 +1,7 @@
 require "test/unit"
 require "whisper"
-require_relative "jfk_reader/jfk_reader"
 
 class TestBase < Test::Unit::TestCase
+  MODEL = File.join(__dir__, "..", "..", "..", "models", "ggml-base.en.bin")
   AUDIO = File.join(__dir__, "..", "..", "..", "samples", "jfk.wav")
 end

bindings/ruby/tests/jfk_reader/.gitignore

@@ -1,5 +0,0 @@
-Makefile
-jfk_reader.o
-jfk_reader.so
-jfk_reader.bundle
-jfk_reader.dll

bindings/ruby/tests/jfk_reader/extconf.rb

@@ -1,3 +0,0 @@
-require "mkmf"
-
-create_makefile("jfk_reader")

bindings/ruby/tests/jfk_reader/jfk_reader.c

@@ -1,68 +0,0 @@
-#include <ruby.h>
-#include <ruby/memory_view.h>
-#include <ruby/encoding.h>
-
-static VALUE
-jfk_reader_initialize(VALUE self, VALUE audio_path)
-{
-  rb_iv_set(self, "audio_path", audio_path);
-  return Qnil;
-}
-
-static bool
-jfk_reader_get_memory_view(const VALUE obj, rb_memory_view_t *view, int flags)
-{
-  VALUE audio_path = rb_iv_get(obj, "audio_path");
-  const char *audio_path_str = StringValueCStr(audio_path);
-  const int n_samples = 176000;
-  float *data = (float *)malloc(n_samples * sizeof(float));
-  short *samples = (short *)malloc(n_samples * sizeof(short));
-  FILE *file = fopen(audio_path_str, "rb");
-
-  fseek(file, 78, SEEK_SET);
-  fread(samples, sizeof(short), n_samples, file);
-  fclose(file);
-  for (int i = 0; i < n_samples; i++) {
-    data[i] = samples[i]/32768.0;
-  }
-
-  view->obj = obj;
-  view->data = (void *)data;
-  view->byte_size = sizeof(float) * n_samples;
-  view->readonly = true;
-  view->format = "f";
-  view->item_size = sizeof(float);
-  view->item_desc.components = NULL;
-  view->item_desc.length = 0;
-  view->ndim = 1;
-  view->shape = NULL;
-  view->sub_offsets = NULL;
-  view->private_data = NULL;
-
-  return true;
-}
-
-static bool
-jfk_reader_release_memory_view(const VALUE obj, rb_memory_view_t *view)
-{
-  return true;
-}
-
-static bool
-jfk_reader_memory_view_available_p(const VALUE obj)
-{
-  return true;
-}
-
-static const rb_memory_view_entry_t jfk_reader_view_entry = {
-  jfk_reader_get_memory_view,
-  jfk_reader_release_memory_view,
-  jfk_reader_memory_view_available_p
-};
-
-void Init_jfk_reader(void)
-{
-  VALUE cJFKReader = rb_define_class("JFKReader", rb_cObject);
-  rb_memory_view_register(cJFKReader, &jfk_reader_view_entry);
-  rb_define_method(cJFKReader, "initialize", jfk_reader_initialize, 1);
-}

bindings/ruby/tests/test_callback.rb

@@ -1,11 +1,14 @@
-require_relative "helper"
+require "test/unit"
+require "whisper"
+
+class TestCallback < Test::Unit::TestCase
+  TOPDIR = File.expand_path(File.join(File.dirname(__FILE__), '..'))
 
-class TestCallback < TestBase
   def setup
     GC.start
     @params = Whisper::Params.new
-    @whisper = Whisper::Context.new("base.en")
-    @audio = File.join(AUDIO)
+    @whisper = Whisper::Context.new(File.join(TOPDIR, '..', '..', 'models', 'ggml-base.en.bin'))
+    @audio = File.join(TOPDIR, '..', '..', 'samples', 'jfk.wav')
   end
 
   def test_new_segment_callback

bindings/ruby/tests/test_error.rb

@@ -1,20 +0,0 @@
-require_relative "helper"
-
-class TestError < TestBase
-  def test_error
-    error = Whisper::Error.new(-2)
-    assert_equal "failed to compute log mel spectrogram", error.message
-    assert_equal -2, error.code
-  end
-
-  def test_unknown_error
-    error = Whisper::Error.new(-20)
-    assert_equal "unknown error", error.message
-  end
-
-  def test_non_int_code
-    assert_raise TypeError do
-      error = Whisper::Error.new("non int")
-    end
-  end
-end

bindings/ruby/tests/test_model.rb

@@ -1,14 +1,13 @@
 require_relative "helper"
-require "pathname"
 
 class TestModel < TestBase
   def test_model
-    whisper = Whisper::Context.new("base.en")
+    whisper = Whisper::Context.new(MODEL)
     assert_instance_of Whisper::Model, whisper.model
   end
 
   def test_attributes
-    whisper = Whisper::Context.new("base.en")
+    whisper = Whisper::Context.new(MODEL)
     model = whisper.model
 
     assert_equal 51864, model.n_vocab
@@ -26,7 +25,7 @@ class TestModel < TestBase
   end
 
   def test_gc
-    model = Whisper::Context.new("base.en").model
+    model = Whisper::Context.new(MODEL).model
     GC.start
 
     assert_equal 51864, model.n_vocab
@@ -42,30 +41,4 @@ class TestModel < TestBase
     assert_equal 1, model.ftype
     assert_equal "base", model.type
   end
-
-  def test_pathname
-    path = Pathname(Whisper::Model.pre_converted_models["base.en"].to_path)
-    whisper = Whisper::Context.new(path)
-    model = whisper.model
-
-    assert_equal 51864, model.n_vocab
-    assert_equal 1500, model.n_audio_ctx
-    assert_equal 512, model.n_audio_state
-    assert_equal 8, model.n_audio_head
-    assert_equal 6, model.n_audio_layer
-    assert_equal 448, model.n_text_ctx
-    assert_equal 512, model.n_text_state
-    assert_equal 8, model.n_text_head
-    assert_equal 6, model.n_text_layer
-    assert_equal 80, model.n_mels
-    assert_equal 1, model.ftype
-    assert_equal "base", model.type
-  end
-
-  def test_auto_download
-    path = Whisper::Model.pre_converted_models["base.en"].to_path
-
-    assert_path_exist path
-    assert_equal 147964211, File.size(path)
-  end
 end

bindings/ruby/tests/test_params.rb

@@ -151,10 +151,4 @@ class TestParams < TestBase
     @params.logprob_thold = -0.5
     assert_in_delta -0.5, @params.logprob_thold
   end
-
-  def test_no_speech_thold
-    assert_in_delta 0.6, @params.no_speech_thold
-    @params.no_speech_thold = 0.2
-    assert_in_delta 0.2, @params.no_speech_thold
-  end
 end

bindings/ruby/tests/test_segment.rb

@@ -5,7 +5,7 @@ class TestSegment < TestBase
     attr_reader :whisper
 
     def startup
-      @whisper = Whisper::Context.new("base.en")
+      @whisper = Whisper::Context.new(TestBase::MODEL)
       params = Whisper::Params.new
       params.print_timestamps = false
       @whisper.transcribe(TestBase::AUDIO, params)

bindings/ruby/tests/test_whisper.rb

@@ -1,6 +1,5 @@
 require_relative "helper"
 require "stringio"
-require "etc"
 
 # Exists to detect memory-related bug
 Whisper.log_set ->(level, buffer, user_data) {}, nil
@@ -11,7 +10,7 @@ class TestWhisper < TestBase
   end
 
   def test_whisper
-    @whisper = Whisper::Context.new("base.en")
+    @whisper = Whisper::Context.new(MODEL)
     params  = Whisper::Params.new
     params.print_timestamps = false
 
@@ -25,7 +24,7 @@ class TestWhisper < TestBase
       attr_reader :whisper
 
       def startup
-        @whisper = Whisper::Context.new("base.en")
+        @whisper = Whisper::Context.new(TestBase::MODEL)
         params = Whisper::Params.new
         params.print_timestamps = false
         @whisper.transcribe(TestBase::AUDIO, params)
@@ -104,11 +103,11 @@ class TestWhisper < TestBase
       logs << [level, buffer, udata]
     }
     Whisper.log_set log_callback, user_data
-    Whisper::Context.new("base.en")
+    Whisper::Context.new(MODEL)
 
     assert logs.length > 30
     logs.each do |log|
-      assert_include [Whisper::LOG_LEVEL_DEBUG, Whisper::LOG_LEVEL_INFO, Whisper::LOG_LEVEL_WARN], log[0]
+      assert_equal Whisper::LOG_LEVEL_INFO, log[0]
       assert_same user_data, log[2]
     end
   end
@@ -120,107 +119,9 @@ class TestWhisper < TestBase
     }, nil
     dev = StringIO.new("")
     $stderr = dev
-    Whisper::Context.new("base.en")
+    Whisper::Context.new(MODEL)
     assert_empty dev.string
   ensure
     $stderr = stderr
   end
-
-  sub_test_case "full" do
-    def setup
-      super
-      @whisper = Whisper::Context.new("base.en")
-      @samples = File.read(AUDIO, nil, 78).unpack("s<*").collect {|i| i.to_f / 2**15}
-    end
-
-    def test_full
-      @whisper.full(@params, @samples, @samples.length)
-
-      assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
-    end
-
-    def test_full_without_length
-      @whisper.full(@params, @samples)
-
-      assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
-    end
-
-    def test_full_enumerator
-      samples = @samples.each
-      @whisper.full(@params, samples, @samples.length)
-
-      assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
-    end
-
-    def test_full_enumerator_without_length
-      samples = @samples.each
-      assert_raise ArgumentError do
-        @whisper.full(@params, samples)
-      end
-    end
-
-    def test_full_enumerator_with_too_large_length
-      samples = @samples.each.take(10).to_enum
-      assert_raise StopIteration do
-        @whisper.full(@params, samples, 11)
-      end
-    end
-
-    def test_full_with_memory_view
-      samples = JFKReader.new(AUDIO)
-      @whisper.full(@params, samples)
-
-      assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
-    end
-
-    def test_full_parallel
-      @whisper.full_parallel(@params, @samples, @samples.length, Etc.nprocessors)
-
-      assert_equal Etc.nprocessors, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-
-    def test_full_parallel_with_memory_view
-      samples = JFKReader.new(AUDIO)
-      @whisper.full_parallel(@params, samples, nil, Etc.nprocessors)
-
-      assert_equal Etc.nprocessors, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-
-    def test_full_parallel_without_length_and_n_processors
-      @whisper.full_parallel(@params, @samples)
-
-      assert_equal 1, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-
-    def test_full_parallel_without_length
-      @whisper.full_parallel(@params, @samples, nil, Etc.nprocessors)
-
-      assert_equal Etc.nprocessors, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-
-    def test_full_parallel_without_n_processors
-      @whisper.full_parallel(@params, @samples, @samples.length)
-
-      assert_equal 1, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-  end
 end

bindings/ruby/whispercpp.gemspec

@@ -1,4 +1,4 @@
-require_relative "extsources"
+require "yaml"
 
 Gem::Specification.new do |s|
   s.name    = "whispercpp"
@@ -10,24 +10,24 @@ Gem::Specification.new do |s|
   s.extra_rdoc_files = ['LICENSE', 'README.md']
   
   s.files = `git ls-files . -z`.split("\x0") +
-              EXTSOURCES.collect {|file|
+              YAML.load_file("extsources.yaml").collect {|file|
                 basename = File.basename(file)
                 if s.extra_rdoc_files.include?(basename)
                   basename
                 else
-                  file.sub("../..", "ext")
+                  File.join("ext", basename)
                 end
               }
 
   s.summary = %q{Ruby whisper.cpp bindings}
-  s.test_files = s.files.select {|file| file.start_with? "tests/"}
+  s.test_files = ["tests/test_whisper.rb"]
   
   s.extensions << 'ext/extconf.rb'
-  s.required_ruby_version = '>= 3.1.0'
+  
 
   #### Documentation and testing.
   s.homepage = 'https://github.com/ggerganov/whisper.cpp'
-  s.rdoc_options = ['--main', 'README.md']
+  s.rdoc_options = ['--main', '../../README.md']
 
   
     s.platform = Gem::Platform::RUBY

cmake/whisper.pc.in

@@ -1,10 +1,10 @@
 prefix=@CMAKE_INSTALL_PREFIX@
 exec_prefix=${prefix}
-libdir=${exec_prefix}/lib
+libdir=@CMAKE_INSTALL_FULL_LIBDIR@
 includedir=${prefix}/include
 
 Name: whisper
 Description: Port of OpenAI's Whisper model in C/C++
 Version: @PROJECT_VERSION@
-Libs: -L${libdir} -lggml  -lggml-base -lwhisper
+Libs: -L${libdir} -lwhisper
 Cflags: -I${includedir}

examples/common-ggml.cpp

@@ -72,6 +72,9 @@ bool ggml_common_quantize_0(
         case GGML_FTYPE_MOSTLY_IQ4_XS:
         case GGML_FTYPE_MOSTLY_IQ1_M:
         case GGML_FTYPE_MOSTLY_BF16:
+        case GGML_FTYPE_MOSTLY_Q4_0_4_4:
+        case GGML_FTYPE_MOSTLY_Q4_0_4_8:
+        case GGML_FTYPE_MOSTLY_Q4_0_8_8:
                 {
                     fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype);
                     return false;
@@ -209,6 +212,9 @@ bool ggml_common_quantize_0(
                 case GGML_TYPE_IQ4_XS:
                 case GGML_TYPE_IQ1_M:
                 case GGML_TYPE_BF16:
+                case GGML_TYPE_Q4_0_4_4:
+                case GGML_TYPE_Q4_0_4_8:
+                case GGML_TYPE_Q4_0_8_8:
                 case GGML_TYPE_TQ1_0:
                 case GGML_TYPE_TQ2_0:
                 case GGML_TYPE_COUNT:

examples/common-sdl.cpp

@@ -1,7 +1,5 @@
 #include "common-sdl.h"
 
-#include <cstdio>
-
 audio_async::audio_async(int len_ms) {
     m_len_ms = len_ms;
 

examples/stream/README.md

@@ -5,7 +5,7 @@ The `stream` tool samples the audio every half a second and runs the transcripti
 More info is available in [issue #10](https://github.com/ggerganov/whisper.cpp/issues/10).
 
 ```bash
-./build/bin/stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
+./stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
 ```
 
 https://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a80f-28ba83be7d09.mp4
@@ -15,7 +15,7 @@ https://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a
 Setting the `--step` argument to `0` enables the sliding window mode:
 
 ```bash
- ./build/bin/stream -m ./models/ggml-base.en.bin -t 6 --step 0 --length 30000 -vth 0.6
+ ./stream -m ./models/ggml-small.en.bin -t 6 --step 0 --length 30000 -vth 0.6
 ```
 
 In this mode, the tool will transcribe only after some speech activity is detected. A very
@@ -40,10 +40,21 @@ sudo dnf install SDL2 SDL2-devel
 # Install SDL2 on Mac OS
 brew install sdl2
 
-cmake -B build -DWHISPER_SDL2=ON
-cmake --build build --config Release
+make stream
+```
 
-./build/bin/stream
+Ensure you are at the root of the repo when running `make stream`. Not within the `examples/stream` dir
+as the libraries needed like `common-sdl.h` are located within `examples`. Attempting to compile within
+`examples/steam` means your compiler cannot find them and it gives an error it cannot find the file.
+
+```bash
+whisper.cpp/examples/stream$ make stream
+g++     stream.cpp   -o stream
+stream.cpp:6:10: fatal error: common/sdl.h: No such file or directory
+    6 | #include "common/sdl.h"
+      |          ^~~~~~~~~~~~~~
+compilation terminated.
+make: *** [<builtin>: stream] Error 1
 ```
 
 ## Web version

examples/talk-llama/llama-sampling.cpp

@@ -1396,15 +1396,19 @@ struct llama_sampler * llama_sampler_init_grammar_impl(const struct llama_vocab
 // penalties
 
 struct llama_sampler_penalties {
+    const int32_t     n_vocab;
+    const llama_token special_eos_id;
+    const llama_token linefeed_id;
+
     const int32_t penalty_last_n;
     const float   penalty_repeat;
     const float   penalty_freq;
     const float   penalty_present;
 
-    ring_buffer<llama_token> prev;
+    const bool    penalize_nl;
+    const bool    ignore_eos;
 
-    // a frequency map to count token occurrences
-    std::unordered_map<llama_token, int> token_count;
+    ring_buffer<llama_token> prev;
 };
 
 static const char * llama_sampler_penalties_name(const struct llama_sampler * /*smpl*/) {
@@ -1417,50 +1421,76 @@ static void llama_sampler_penalties_accept(struct llama_sampler * smpl, llama_to
         return;
     }
 
-    ctx->token_count[token]++;
-
-    // if the ring buffer is full, remove the oldest token
-    if (ctx->prev.size() >= (size_t) ctx->penalty_last_n) {
-        const auto old = ctx->prev.front();
-
-        ctx->token_count[old]--;
-        if (ctx->token_count[old] == 0) {
-            ctx->token_count.erase(old);
-        }
-    }
-
     ctx->prev.push_back(token);
-
-#if 0
-    // sanity check
-    std::unordered_map<llama_token, int> tmp;
-    for (int i = 0; i < std::min<int>(ctx->penalty_last_n, ctx->prev.size()); ++i) {
-        tmp[ctx->prev.rat(i)]++;
-    }
-
-    assert(ctx->token_count == tmp);
-#endif
 }
 
 static void llama_sampler_penalties_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
     auto * ctx = (llama_sampler_penalties *) smpl->ctx;
 
+    if (ctx->ignore_eos) {
+        assert(ctx->special_eos_id >= 0);
+
+        // optimistically check if the candidates are not yet sorted/shuffled/truncated
+        if (cur_p->size > (size_t) ctx->special_eos_id && cur_p->data[ctx->special_eos_id].id == ctx->special_eos_id) {
+            cur_p->data[ctx->special_eos_id].logit = -INFINITY;
+        } else {
+            // else, search for the special EOS token
+            for (size_t i = 0; i < cur_p->size; ++i) {
+                if (cur_p->data[i].id == ctx->special_eos_id) {
+                    cur_p->data[i].logit = -INFINITY;
+                    break;
+                }
+            }
+        }
+    }
+
     if ((ctx->penalty_last_n == 0) ||
         (ctx->penalty_repeat == 1.0f && ctx->penalty_freq == 0.0f && ctx->penalty_present == 0.0f)) {
         return;
     }
 
+    bool nl_found = false;
+    size_t nl_idx = 0;
+    float nl_logit = -INFINITY;
+    if (!ctx->penalize_nl) {
+        assert(ctx->linefeed_id >= 0);
+
+        // optimistically check if the candidates are not yet sorted/shuffled/truncated
+        if (cur_p->size > (size_t) ctx->linefeed_id && cur_p->data[ctx->linefeed_id].id == ctx->linefeed_id) {
+            nl_found = true;
+            nl_idx = ctx->linefeed_id;
+            nl_logit = cur_p->data[ctx->linefeed_id].logit;
+        } else {
+            // else, search for the linefeed token
+            for (size_t i = 0; i < cur_p->size; ++i) {
+                if (cur_p->data[i].id == ctx->linefeed_id) {
+                    nl_found = true;
+                    nl_idx = i;
+                    nl_logit = cur_p->data[i].logit;
+                    break;
+                }
+            }
+        }
+    }
+
+    // Create a frequency map to count occurrences of each token in last_tokens
+    // TODO: optimize this by maintaining the token count in the sampler context
+    using llama_token_cnt = std::unordered_map<llama_token, int>;
+    llama_token_cnt token_count;
+
+    for (int i = 0; i < std::min<int>(ctx->penalty_last_n, ctx->prev.size()); ++i) {
+        token_count[ctx->prev.rat(i)]++;
+    }
+
     // Apply frequency and presence penalties to the cur_p
     for (size_t i = 0; i < cur_p->size; ++i) {
-        const auto token_iter = ctx->token_count.find(cur_p->data[i].id);
-        if (token_iter == ctx->token_count.end()) {
+        const auto token_iter = token_count.find(cur_p->data[i].id);
+        if (token_iter == token_count.end()) {
             continue;
         }
 
         const int count = token_iter->second;
 
-        assert(count > 0 && count <= ctx->penalty_last_n);
-
         // The academic publication that described this technique actually just only divided, but that would cause tokens with negative logits to become more likely, which is obviously wrong.
         // This is common fix for this problem, which is to multiply by the penalty instead of dividing.
         if (cur_p->data[i].logit <= 0) {
@@ -1473,21 +1503,30 @@ static void llama_sampler_penalties_apply(struct llama_sampler * smpl, llama_tok
     }
 
     cur_p->sorted = false;
+
+    if (!ctx->penalize_nl && nl_found) {
+        // restore the logit of the newline token if it was penalized
+        cur_p->data[nl_idx].logit = nl_logit;
+    }
 }
 
 static void llama_sampler_penalties_reset(struct llama_sampler * smpl) {
     auto * ctx = (llama_sampler_penalties *) smpl->ctx;
     ctx->prev.clear();
-    ctx->token_count.clear();
 }
 
 static struct llama_sampler * llama_sampler_penalties_clone(const struct llama_sampler * smpl) {
     const auto * ctx = (const llama_sampler_penalties *) smpl->ctx;
     auto * result = llama_sampler_init_penalties(
+            ctx->n_vocab,
+            ctx->special_eos_id,
+            ctx->linefeed_id,
             ctx->penalty_last_n,
             ctx->penalty_repeat,
             ctx->penalty_freq,
-            ctx->penalty_present);
+            ctx->penalty_present,
+            ctx->penalize_nl,
+            ctx->ignore_eos);
 
     // copy the state
     {
@@ -1513,21 +1552,38 @@ static struct llama_sampler_i llama_sampler_penalties_i = {
 };
 
 struct llama_sampler * llama_sampler_init_penalties(
+        int32_t n_vocab,
+        llama_token special_eos_id,
+        llama_token linefeed_id,
         int32_t penalty_last_n,
         float penalty_repeat,
         float penalty_freq,
-        float penalty_present) {
+        float penalty_present,
+        bool penalize_nl,
+        bool ignore_eos) {
+    if (linefeed_id == LLAMA_TOKEN_NULL) {
+        penalize_nl = true;
+    }
+
+    if (special_eos_id == LLAMA_TOKEN_NULL) {
+        ignore_eos = false;
+    }
+
     penalty_last_n = std::max(penalty_last_n, 0);
 
     return new llama_sampler {
         /* .iface = */ &llama_sampler_penalties_i,
         /* .ctx   = */ new llama_sampler_penalties {
+            /* .n_vocab         = */ n_vocab,
+            /* .special_eos_id  = */ special_eos_id,
+            /* .linefeed_id     = */ linefeed_id,
             /* .penalty_last_n  = */ penalty_last_n,
             /* .penalty_repeat  = */ penalty_repeat,
             /* .penalty_freq    = */ penalty_freq,
             /* .penalty_present = */ penalty_present,
+            /* .penalize_nl     = */ penalize_nl,
+            /* .ignore_eos      = */ ignore_eos,
             /* .prev            = */ ring_buffer<llama_token>(penalty_last_n),
-            /* .token_count     = */ {},
         },
     };
 }
@@ -1555,8 +1611,7 @@ static void get_overlapping_token_sequences(const llama_vocab & vocab, const std
         if (word.find(str) != std::string::npos) {
             token_sequences.emplace(token_id, std::vector<llama_token>());
         } else {
-            size_t word_len = word.size();
-            size_t str_len = str.size();
+            size_t word_len = word.size(), str_len = str.size();
             size_t pos = -1;
             while ((pos = word.find(str[0], pos + 1)) != std::string::npos) {
                 bool match = true;

examples/talk-llama/llama-vocab.cpp

@@ -418,7 +418,6 @@ struct llm_tokenizer_bpe : llm_tokenizer {
             case LLAMA_VOCAB_PRE_TYPE_SMOLLM:
             case LLAMA_VOCAB_PRE_TYPE_CODESHELL:
             case LLAMA_VOCAB_PRE_TYPE_EXAONE:
-            case LLAMA_VOCAB_PRE_TYPE_MINERVA:
                 regex_exprs = {
                     "\\p{N}",
                     "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
@@ -738,7 +737,7 @@ struct llm_tokenizer_wpm_session {
         std::vector<std::string> words(1, "");
 
         for (const uint32_t cpt : cpts_nfd) {
-            const auto flags = unicode_cpt_flags_from_cpt(cpt);
+            const auto flags = unicode_cpt_flags(cpt);
 
             if (flags.is_whitespace) {
                 if (words.back().size()) {  // finish previous word if any

examples/talk-llama/llama.h

@@ -104,15 +104,12 @@ extern "C" {
         LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH   = 24,
         LLAMA_VOCAB_PRE_TYPE_EXAONE         = 25,
         LLAMA_VOCAB_PRE_TYPE_CHAMELEON      = 26,
-        LLAMA_VOCAB_PRE_TYPE_MINERVA        = 27,
     };
 
     enum llama_rope_type {
-        LLAMA_ROPE_TYPE_NONE   = -1,
-        LLAMA_ROPE_TYPE_NORM   = 0,
-        LLAMA_ROPE_TYPE_NEOX   = GGML_ROPE_TYPE_NEOX,
-        LLAMA_ROPE_TYPE_MROPE  = GGML_ROPE_TYPE_MROPE,
-        LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,
+        LLAMA_ROPE_TYPE_NONE = -1,
+        LLAMA_ROPE_TYPE_NORM = 0,
+        LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,
     };
 
     enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file
@@ -174,9 +171,9 @@ extern "C" {
         LLAMA_FTYPE_MOSTLY_IQ4_XS        = 30, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_IQ1_M         = 31, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_BF16          = 32, // except 1d tensors
-        //LLAMA_FTYPE_MOSTLY_Q4_0_4_4      = 33, // removed from gguf files, use Q4_0 and runtime repack
-        //LLAMA_FTYPE_MOSTLY_Q4_0_4_8      = 34, // removed from gguf files, use Q4_0 and runtime repack
-        //LLAMA_FTYPE_MOSTLY_Q4_0_8_8      = 35, // removed from gguf files, use Q4_0 and runtime repack
+        LLAMA_FTYPE_MOSTLY_Q4_0_4_4      = 33, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_Q4_0_4_8      = 34, // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_Q4_0_8_8      = 35, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_TQ1_0         = 36, // except 1d tensors
         LLAMA_FTYPE_MOSTLY_TQ2_0         = 37, // except 1d tensors
 
@@ -188,8 +185,7 @@ extern "C" {
         LLAMA_ROPE_SCALING_TYPE_NONE        = 0,
         LLAMA_ROPE_SCALING_TYPE_LINEAR      = 1,
         LLAMA_ROPE_SCALING_TYPE_YARN        = 2,
-        LLAMA_ROPE_SCALING_TYPE_LONGROPE    = 3,
-        LLAMA_ROPE_SCALING_TYPE_MAX_VALUE   = LLAMA_ROPE_SCALING_TYPE_LONGROPE,
+        LLAMA_ROPE_SCALING_TYPE_MAX_VALUE   = LLAMA_ROPE_SCALING_TYPE_YARN,
     };
 
     enum llama_pooling_type {
@@ -276,9 +272,6 @@ extern "C" {
     };
 
     struct llama_model_params {
-        // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)
-        ggml_backend_dev_t * devices;
-
         int32_t n_gpu_layers; // number of layers to store in VRAM
         enum llama_split_mode split_mode; // how to split the model across multiple GPUs
 
@@ -458,7 +451,6 @@ extern "C" {
     // Functions to access the model's GGUF metadata scalar values
     // - The functions return the length of the string on success, or -1 on failure
     // - The output string is always null-terminated and cleared on failure
-    // - When retrieving a string, an extra byte must be allocated to account for the null terminator
     // - GGUF array values are not supported by these functions
 
     // Get metadata value as a string by key name
@@ -995,9 +987,6 @@ extern "C" {
                                   char * buf,
                                int32_t   length);
 
-    // Get list of built-in chat templates
-    LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len);
-
     //
     // Sampling API
     //
@@ -1139,12 +1128,16 @@ extern "C" {
                           const char * grammar_str,
                           const char * grammar_root);
 
-    /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.
     LLAMA_API struct llama_sampler * llama_sampler_init_penalties(
-                             int32_t   penalty_last_n,   // last n tokens to penalize (0 = disable penalty, -1 = context size)
-                               float   penalty_repeat,   // 1.0 = disabled
-                               float   penalty_freq,     // 0.0 = disabled
-                               float   penalty_present); // 0.0 = disabled
+                             int32_t   n_vocab,         // llama_n_vocab()
+                         llama_token   special_eos_id,  // llama_token_eos()
+                         llama_token   linefeed_id,     // llama_token_nl()
+                             int32_t   penalty_last_n,  // last n tokens to penalize (0 = disable penalty, -1 = context size)
+                               float   penalty_repeat,  // 1.0 = disabled
+                               float   penalty_freq,    // 0.0 = disabled
+                               float   penalty_present, // 0.0 = disabled
+                                bool   penalize_nl,     // consider newlines as a repeatable token
+                                bool   ignore_eos);     // ignore the end-of-sequence token
 
     ///  @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982
     LLAMA_API struct llama_sampler *    llama_sampler_init_dry(

examples/talk-llama/unicode.cpp

@@ -71,15 +71,15 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
     throw std::invalid_argument("failed to convert utf8 to codepoint");
 }
 
-//static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cpt) {
+//static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cp) {
 //    std::vector<uint16_t> result;
-//    if (/* 0x0000 <= cpt && */ cpt <= 0xffff) {
-//        result.emplace_back(cpt);
+//    if (/* 0x0000 <= cp && */ cp <= 0xffff) {
+//        result.emplace_back(cp);
 //        return result;
 //    }
-//    if (0x10000 <= cpt && cpt <= 0x10ffff) {
-//        result.emplace_back(0xd800 | ((cpt - 0x10000) >> 10));
-//        result.emplace_back(0xdc00 | ((cpt - 0x10000) & 0x03ff));
+//    if (0x10000 <= cp && cp <= 0x10ffff) {
+//        result.emplace_back(0xd800 | ((cp - 0x10000) >> 10));
+//        result.emplace_back(0xdc00 | ((cp - 0x10000) & 0x03ff));
 //        return result;
 //    }
 //    throw std::invalid_argument("failed to convert codepoint to utf16");
@@ -120,8 +120,8 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
 //    return result;
 //}
 
-static std::vector<unicode_cpt_flags> unicode_cpt_flags_array() {
-    std::vector<unicode_cpt_flags> cpt_flags(MAX_CODEPOINTS, unicode_cpt_flags::UNDEFINED);
+static std::vector<codepoint_flags> unicode_cpt_flags_array() {
+    std::vector<codepoint_flags> cpt_flags(MAX_CODEPOINTS, codepoint_flags::UNDEFINED);
 
     assert (unicode_ranges_flags.begin()[0].first == 0);
     assert (unicode_ranges_flags.begin()[unicode_ranges_flags.size()-1].first == MAX_CODEPOINTS);
@@ -201,18 +201,7 @@ static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
 }
 
 static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
-#if defined(__clang__)
-    // disable C++17 deprecation warning for std::codecvt_utf8
-#    pragma clang diagnostic push
-#    pragma clang diagnostic ignored "-Wdeprecated-declarations"
-#endif
-
     std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
-
-#if defined(__clang__)
-#    pragma clang diagnostic pop
-#endif
-
     return conv.from_bytes(s);
 }
 
@@ -253,8 +242,8 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
             return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;
         };
 
-        auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags {
-            return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{};
+        auto _get_flags = [&] (const size_t pos) -> codepoint_flags {
+            return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : codepoint_flags{};
         };
 
         size_t _prev_end = offset_ini;
@@ -371,8 +360,8 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
             return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;
         };
 
-        auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags {
-            return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{};
+        auto _get_flags = [&] (const size_t pos) -> codepoint_flags {
+            return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : codepoint_flags{};
         };
 
         size_t _prev_end = offset_ini;
@@ -572,29 +561,29 @@ static std::vector<size_t> unicode_regex_split_custom(const std::string & text,
 // interface
 //
 
-std::string unicode_cpt_to_utf8(uint32_t cpt) {
+std::string unicode_cpt_to_utf8(uint32_t cp) {
     std::string result;
 
-    if (/* 0x00 <= cpt && */ cpt <= 0x7f) {
-        result.push_back(cpt);
+    if (/* 0x00 <= cp && */ cp <= 0x7f) {
+        result.push_back(cp);
         return result;
     }
-    if (0x80 <= cpt && cpt <= 0x7ff) {
-        result.push_back(0xc0 | ((cpt >> 6) & 0x1f));
-        result.push_back(0x80 | (cpt & 0x3f));
+    if (0x80 <= cp && cp <= 0x7ff) {
+        result.push_back(0xc0 | ((cp >> 6) & 0x1f));
+        result.push_back(0x80 | (cp & 0x3f));
         return result;
     }
-    if (0x800 <= cpt && cpt <= 0xffff) {
-        result.push_back(0xe0 | ((cpt >> 12) & 0x0f));
-        result.push_back(0x80 | ((cpt >> 6) & 0x3f));
-        result.push_back(0x80 | (cpt & 0x3f));
+    if (0x800 <= cp && cp <= 0xffff) {
+        result.push_back(0xe0 | ((cp >> 12) & 0x0f));
+        result.push_back(0x80 | ((cp >> 6) & 0x3f));
+        result.push_back(0x80 | (cp & 0x3f));
         return result;
     }
-    if (0x10000 <= cpt && cpt <= 0x10ffff) {
-        result.push_back(0xf0 | ((cpt >> 18) & 0x07));
-        result.push_back(0x80 | ((cpt >> 12) & 0x3f));
-        result.push_back(0x80 | ((cpt >> 6) & 0x3f));
-        result.push_back(0x80 | (cpt & 0x3f));
+    if (0x10000 <= cp && cp <= 0x10ffff) {
+        result.push_back(0xf0 | ((cp >> 18) & 0x07));
+        result.push_back(0x80 | ((cp >> 12) & 0x3f));
+        result.push_back(0x80 | ((cp >> 6) & 0x3f));
+        result.push_back(0x80 | (cp & 0x3f));
         return result;
     }
 
@@ -624,19 +613,19 @@ std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8) {
     return result;
 }
 
-unicode_cpt_flags unicode_cpt_flags_from_cpt(const uint32_t cpt) {
-    static const unicode_cpt_flags undef(unicode_cpt_flags::UNDEFINED);
+codepoint_flags unicode_cpt_flags(const uint32_t cp) {
+    static const codepoint_flags undef(codepoint_flags::UNDEFINED);
     static const auto cpt_flags = unicode_cpt_flags_array();
-    return cpt < cpt_flags.size() ? cpt_flags[cpt] : undef;
+    return cp < cpt_flags.size() ? cpt_flags[cp] : undef;
 }
 
-unicode_cpt_flags unicode_cpt_flags_from_utf8(const std::string & utf8) {
-    static const unicode_cpt_flags undef(unicode_cpt_flags::UNDEFINED);
+codepoint_flags unicode_cpt_flags(const std::string & utf8) {
+    static const codepoint_flags undef(codepoint_flags::UNDEFINED);
     if (utf8.empty()) {
         return undef;  // undefined
     }
     size_t offset = 0;
-    return unicode_cpt_flags_from_cpt(unicode_cpt_from_utf8(utf8, offset));
+    return unicode_cpt_flags(unicode_cpt_from_utf8(utf8, offset));
 }
 
 std::string unicode_byte_to_utf8(uint8_t byte) {
@@ -649,41 +638,41 @@ uint8_t unicode_utf8_to_byte(const std::string & utf8) {
     return map.at(utf8);
 }
 
-uint32_t unicode_tolower(uint32_t cpt) {
+uint32_t unicode_tolower(uint32_t cp) {
     // binary search
-    auto it = std::lower_bound(unicode_map_lowercase.begin(), unicode_map_lowercase.end(), cpt,
+    auto it = std::lower_bound(unicode_map_lowercase.begin(), unicode_map_lowercase.end(), cp,
         [](const std::pair<uint32_t, uint32_t> & pair, uint32_t value) {
             return pair.first < value;
         });
-    if (it != unicode_map_lowercase.end() && it->first == cpt) {
+    if (it != unicode_map_lowercase.end() && it->first == cp) {
         return it->second;
     }
-    return cpt;  // Return the original code point if no lowercase mapping is found
+    return cp;  // Return the original code point if no lowercase mapping is found
 }
 
 std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) {
     // unicode categories
     static const std::map<std::string, int> k_ucat_enum = {
-        { "\\p{N}", unicode_cpt_flags::NUMBER },
-        { "\\p{L}", unicode_cpt_flags::LETTER },
-        { "\\p{P}", unicode_cpt_flags::PUNCTUATION },
+        { "\\p{N}", codepoint_flags::NUMBER },
+        { "\\p{L}", codepoint_flags::LETTER },
+        { "\\p{P}", codepoint_flags::PUNCTUATION },
     };
 
     static const std::map<int, int> k_ucat_cpt = {
-        { unicode_cpt_flags::NUMBER,      0xD1 },
-        { unicode_cpt_flags::LETTER,      0xD2 },
-        { unicode_cpt_flags::PUNCTUATION, 0xD3 },
+        { codepoint_flags::NUMBER,        0xD1 },
+        { codepoint_flags::LETTER,        0xD2 },
+        { codepoint_flags::PUNCTUATION,   0xD3 },
     };
 
     static const std::map<int, std::string> k_ucat_map = {
-        { unicode_cpt_flags::NUMBER,      "\x30-\x39" }, // 0-9
-        { unicode_cpt_flags::LETTER,      "\x41-\x5A\x61-\x7A" }, // A-Za-z
-        { unicode_cpt_flags::PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\}
+        { codepoint_flags::NUMBER,        "\x30-\x39" }, // 0-9
+        { codepoint_flags::LETTER,        "\x41-\x5A\x61-\x7A" }, // A-Za-z
+        { codepoint_flags::PUNCTUATION,   "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\}
     };
 
     // compute collapsed codepoints only if needed by at least one regex
     bool need_collapse = false;
-    for (const auto & regex_expr : regex_exprs) {
+    for (auto & regex_expr : regex_exprs) {
         // search for unicode categories
         for (const auto & ucat : k_ucat_enum) {
             if (std::string::npos != regex_expr.find(ucat.first)) {
@@ -709,7 +698,7 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
                 continue;
             }
 
-            const auto flags = unicode_cpt_flags_from_cpt(cpts[i]);
+            const auto flags = unicode_cpt_flags(cpts[i]);
 
             if (flags.is_whitespace) {
                 //NOTE: C++ std::regex \s does not mach 0x85, Rust and Python regex does.
@@ -725,7 +714,7 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
 
     std::vector<size_t> bpe_offsets = { cpts.size() };
 
-    for (const auto & regex_expr : regex_exprs) {
+    for (auto & regex_expr : regex_exprs) {
         // first, see if we have an efficient custom regex implementation
         auto tmp = unicode_regex_split_custom(text, regex_expr, bpe_offsets);
 
@@ -739,7 +728,7 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
             // if a unicode category is used in the regex, we use the collapsed text and replace the unicode category
             // with the corresponding collapsed representation
             bool use_collapsed = false;
-            for (const auto & ucat : k_ucat_enum) {
+            for (auto & ucat : k_ucat_enum) {
                 if (std::string::npos != regex_expr.find(ucat.first)) {
                     use_collapsed = true;
                     break;
@@ -805,7 +794,7 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
                 // std::wregex \s does not mach non-ASCII whitespaces, using 0x0B as fallback
                 std::wstring wtext(cpts.begin(), cpts.end());
                 for (size_t i = 0; i < wtext.size(); ++i) {
-                    if (wtext[i] > 0x7F && unicode_cpt_flags_from_cpt(wtext[i]).is_whitespace) {
+                    if (wtext[i] > 0x7F && unicode_cpt_flags(wtext[i]).is_whitespace) {
                         wtext[i] = 0x0B;
                     }
                 }

examples/talk-llama/unicode.h

@@ -4,7 +4,9 @@
 #include <string>
 #include <vector>
 
-struct unicode_cpt_flags {
+// TODO: prefix all symbols with "llama_"
+
+struct codepoint_flags {
     enum {
         UNDEFINED       = 0x0001,
         NUMBER          = 0x0002,  // regex: \p{N}
@@ -33,7 +35,7 @@ struct unicode_cpt_flags {
     uint16_t is_nfd         : 1;
 
     // decode from uint16
-    inline unicode_cpt_flags(const uint16_t flags = 0) {
+    inline codepoint_flags(const uint16_t flags=0) {
         *reinterpret_cast<uint16_t*>(this) = flags;
     }
 
@@ -48,19 +50,18 @@ struct unicode_cpt_flags {
 
 size_t unicode_len_utf8(char src);
 
-std::string unicode_cpt_to_utf8  (uint32_t cpt);
-uint32_t    unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);
-
+std::string unicode_cpt_to_utf8(uint32_t cp);
+uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);
 std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);
 
 std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts);
 
-unicode_cpt_flags unicode_cpt_flags_from_cpt (uint32_t cpt);
-unicode_cpt_flags unicode_cpt_flags_from_utf8(const std::string & utf8);
+codepoint_flags unicode_cpt_flags(const uint32_t cp);
+codepoint_flags unicode_cpt_flags(const std::string & utf8);
 
 std::string unicode_byte_to_utf8(uint8_t byte);
-uint8_t     unicode_utf8_to_byte(const std::string & utf8);
+uint8_t unicode_utf8_to_byte(const std::string & utf8);
 
-uint32_t unicode_tolower(uint32_t cpt);
+uint32_t unicode_tolower(uint32_t cp);
 
 std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs);

examples/whisper.android/lib/src/main/jni/whisper/CMakeLists.txt

@@ -2,11 +2,11 @@ cmake_minimum_required(VERSION 3.10)
 
 project(whisper.cpp)
 
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 11)
 set(WHISPER_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../../../../..)
 
 # Path to external GGML, otherwise uses the copy in whisper.cpp.
-option(GGML_HOME "whisper: Path to external GGML source" OFF)
+option(GGML_HOME       "whisper: Path to external GGML source" OFF)
 
 set(
     SOURCE_FILES
@@ -14,13 +14,12 @@ set(
     ${CMAKE_SOURCE_DIR}/jni.c
     )
 
-# TODO: this needs to be updated to work with the new ggml CMakeLists
-
 if (NOT GGML_HOME)
     set(
         SOURCE_FILES
         ${SOURCE_FILES}
         ${WHISPER_LIB_DIR}/ggml/src/ggml.c
+        ${WHISPER_LIB_DIR}/ggml/src/ggml-aarch64.c
         ${WHISPER_LIB_DIR}/ggml/src/ggml-alloc.c
         ${WHISPER_LIB_DIR}/ggml/src/ggml-backend.cpp
         ${WHISPER_LIB_DIR}/ggml/src/ggml-backend-reg.cpp
@@ -28,10 +27,8 @@ if (NOT GGML_HOME)
         ${WHISPER_LIB_DIR}/ggml/src/ggml-threading.cpp
         ${WHISPER_LIB_DIR}/ggml/src/ggml-cpu/ggml-cpu.c
         ${WHISPER_LIB_DIR}/ggml/src/ggml-cpu/ggml-cpu.cpp
-        ${WHISPER_LIB_DIR}/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp
-        ${WHISPER_LIB_DIR}/ggml/src/ggml-cpu/ggml-cpu-hbm.cpp
+        ${WHISPER_LIB_DIR}/ggml/src/ggml-cpu/ggml-cpu-aarch64.c
         ${WHISPER_LIB_DIR}/ggml/src/ggml-cpu/ggml-cpu-quants.c
-        ${WHISPER_LIB_DIR}/ggml/src/ggml-cpu/ggml-cpu-traits.cpp
         )
 endif()
 
@@ -89,5 +86,3 @@ include_directories(${WHISPER_LIB_DIR}/src)
 include_directories(${WHISPER_LIB_DIR}/include)
 include_directories(${WHISPER_LIB_DIR}/ggml/include)
 include_directories(${WHISPER_LIB_DIR}/ggml/src)
-include_directories(${WHISPER_LIB_DIR}/ggml/src/ggml-cpu)
-

examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj

@@ -363,6 +363,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
+				GENERATE_INFOPLIST_FILE = YES;
 				HEADER_SEARCH_PATHS = ../../../ggml/src/;
 				IPHONEOS_DEPLOYMENT_TARGET = 16.0;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
@@ -417,6 +418,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
+				GENERATE_INFOPLIST_FILE = YES;
 				HEADER_SEARCH_PATHS = ../../../ggml/src/;
 				IPHONEOS_DEPLOYMENT_TARGET = 16.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
@@ -430,66 +432,6 @@
 			};
 			name = Release;
 		};
-		18627C9029052BE000BD2A04 /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
-				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 1;
-				DEVELOPMENT_TEAM = P8JZH34X63;
-				GCC_WARN_64_TO_32_BIT_CONVERSION = NO;
-				GENERATE_INFOPLIST_FILE = YES;
-				HEADER_SEARCH_PATHS = ../../../ggml/src/;
-				INFOPLIST_FILE = whisper.objc/Info.plist;
-				INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
-				INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;
-				INFOPLIST_KEY_UIMainStoryboardFile = Main;
-				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
-				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				MARKETING_VERSION = 1.0;
-				MTL_HEADER_SEARCH_PATHS = "";
-				PRODUCT_BUNDLE_IDENTIFIER = "com.ggerganov.whisper-objc";
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_EMIT_LOC_STRINGS = YES;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Debug;
-		};
-		18627C9129052BE000BD2A04 /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
-				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 1;
-				DEVELOPMENT_TEAM = P8JZH34X63;
-				GCC_WARN_64_TO_32_BIT_CONVERSION = NO;
-				GENERATE_INFOPLIST_FILE = YES;
-				HEADER_SEARCH_PATHS = ../../../ggml/src/;
-				INFOPLIST_FILE = whisper.objc/Info.plist;
-				INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
-				INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;
-				INFOPLIST_KEY_UIMainStoryboardFile = Main;
-				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
-				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				MARKETING_VERSION = 1.0;
-				MTL_HEADER_SEARCH_PATHS = "";
-				PRODUCT_BUNDLE_IDENTIFIER = "com.ggerganov.whisper-objc";
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_EMIT_LOC_STRINGS = YES;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Release;
-		};
 /* End XCBuildConfiguration section */
 
 /* Begin XCConfigurationList section */
@@ -502,15 +444,6 @@
 			defaultConfigurationIsVisible = 0;
 			defaultConfigurationName = Release;
 		};
-		18627C8F29052BE000BD2A04 /* Build configuration list for PBXNativeTarget "whisper.objc" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				18627C9029052BE000BD2A04 /* Debug */,
-				18627C9129052BE000BD2A04 /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
 /* End XCConfigurationList section */
 	};
 	rootObject = 18627C6E29052BDF00BD2A04 /* Project object */;

examples/whisper.swiftui/whisper.cpp.swift/LibWhisper.swift

@@ -66,7 +66,7 @@ actor WhisperContext {
 
     private func systemInfo() -> String {
         var info = ""
-        //if (ggml_cpu_has_neon() != 0) { info += "NEON " }
+        if (ggml_cpu_has_neon() != 0) { info += "NEON " }
         return String(info.dropLast())
     }
 
@@ -75,45 +75,45 @@ actor WhisperContext {
         if (whisper_set_mel(context, nil, 0, nMels) != 0) {
             return "error: failed to set mel"
         }
-
+        
         // heat encoder
         if (whisper_encode(context, 0, nThreads) != 0) {
             return "error: failed to encode"
         }
-
+        
         var tokens = [whisper_token](repeating: 0, count: 512)
-
+        
         // prompt heat
         if (whisper_decode(context, &tokens, 256, 0, nThreads) != 0) {
             return "error: failed to decode"
         }
-
+        
         // text-generation heat
         if (whisper_decode(context, &tokens, 1, 256, nThreads) != 0) {
             return "error: failed to decode"
         }
-
+        
         whisper_reset_timings(context)
-
+        
         // actual run
         if (whisper_encode(context, 0, nThreads) != 0) {
             return "error: failed to encode"
         }
-
+        
         // text-generation
         for i in 0..<256 {
             if (whisper_decode(context, &tokens, 1, Int32(i), nThreads) != 0) {
                 return "error: failed to decode"
             }
         }
-
+        
         // batched decoding
         for _ in 0..<64 {
             if (whisper_decode(context, &tokens, 5, 0, nThreads) != 0) {
                 return "error: failed to decode"
             }
         }
-
+        
         // prompt processing
         for _ in 0..<16 {
             if (whisper_decode(context, &tokens, 256, 0, nThreads) != 0) {

ggml/CMakeLists.txt

@@ -32,15 +32,7 @@ else()
     endif()
 endif()
 
-# remove the lib prefix on win32 mingw
-if (WIN32)
-    set(CMAKE_STATIC_LIBRARY_PREFIX "")
-    set(CMAKE_SHARED_LIBRARY_PREFIX "")
-    set(CMAKE_SHARED_MODULE_PREFIX  "")
-endif()
-
 option(BUILD_SHARED_LIBS "ggml: build shared libraries" ${BUILD_SHARED_LIBS_DEFAULT})
-option(GGML_BACKEND_DL   "ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)" OFF)
 
 #
 # option list
@@ -99,33 +91,28 @@ else()
     set(INS_ENB ON)
 endif()
 
-option(GGML_CPU_HBM          "ggml: use memkind for CPU HBM" OFF)
-option(GGML_CPU_AARCH64      "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON)
-option(GGML_AVX              "ggml: enable AVX"              ${INS_ENB})
-option(GGML_AVX_VNNI         "ggml: enable AVX-VNNI"         OFF)
-option(GGML_AVX2             "ggml: enable AVX2"             ${INS_ENB})
-option(GGML_AVX512           "ggml: enable AVX512F"          OFF)
-option(GGML_AVX512_VBMI      "ggml: enable AVX512-VBMI"      OFF)
-option(GGML_AVX512_VNNI      "ggml: enable AVX512-VNNI"      OFF)
-option(GGML_AVX512_BF16      "ggml: enable AVX512-BF16"      OFF)
-if (NOT MSVC)
-    # in MSVC F16C and FMA is implied with AVX2/AVX512
-    option(GGML_FMA          "ggml: enable FMA"              ${INS_ENB})
-    option(GGML_F16C         "ggml: enable F16C"             ${INS_ENB})
-    # MSVC does not seem to support AMX
-    option(GGML_AMX_TILE     "ggml: enable AMX-TILE"         OFF)
-    option(GGML_AMX_INT8     "ggml: enable AMX-INT8"         OFF)
-    option(GGML_AMX_BF16     "ggml: enable AMX-BF16"         OFF)
-endif()
-option(GGML_LASX             "ggml: enable lasx"             ON)
-option(GGML_LSX              "ggml: enable lsx"              ON)
-option(GGML_RVV              "ggml: enable rvv"              ON)
-option(GGML_SVE              "ggml: enable SVE"              OFF)
-option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF)
+option(GGML_CPU_HBM     "ggml: use memkind for CPU HBM" OFF)
+option(GGML_CPU_AARCH64 "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON)
 
+option(GGML_AVX         "ggml: enable AVX"              ${INS_ENB})
+option(GGML_AVX2        "ggml: enable AVX2"             ${INS_ENB})
+option(GGML_AVX512      "ggml: enable AVX512"           OFF)
+option(GGML_AVX512_VBMI "ggml: enable AVX512-VBMI"      OFF)
+option(GGML_AVX512_VNNI "ggml: enable AVX512-VNNI"      OFF)
+option(GGML_AVX512_BF16 "ggml: enable AVX512-BF16"      OFF)
+option(GGML_AMX_TILE    "ggml: enable AMX-TILE"         OFF)
+option(GGML_AMX_INT8    "ggml: enable AMX-INT8"         OFF)
+option(GGML_AMX_BF16    "ggml: enable AMX-BF16"         OFF)
+option(GGML_FMA         "ggml: enable FMA"              ${INS_ENB})
+if (NOT MSVC)
+    option(GGML_F16C    "ggml: enable F16C"             ${INS_ENB}) # in MSVC F16C is implied with AVX2/AVX512
+endif()
+option(GGML_LASX        "ggml: enable lasx"             ON)
+option(GGML_LSX         "ggml: enable lsx"              ON)
+option(GGML_SVE         "ggml: enable SVE"              OFF)
 
 if (WIN32)
-    set(GGML_WIN_VER "0x602" CACHE STRING   "ggml: Windows version")
+    set(GGML_WIN_VER "0x602" CACHE STRING "ggml: Windows Version")
 endif()
 
 # ggml core
@@ -172,6 +159,7 @@ set   (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING
 set   (GGML_METAL_STD "" CACHE STRING       "ggml: metal standard version (-std flag)")
 option(GGML_OPENMP                          "ggml: use OpenMP"                                ON)
 option(GGML_RPC                             "ggml: use RPC"                                   OFF)
+option(GGML_AMX                             "ggml: use AMX"                                   OFF)
 option(GGML_SYCL                            "ggml: use SYCL"                                  OFF)
 option(GGML_SYCL_F16                        "ggml: use 16 bit floats for sycl calculations"   OFF)
 set   (GGML_SYCL_TARGET "INTEL" CACHE STRING
@@ -179,11 +167,6 @@ set   (GGML_SYCL_TARGET "INTEL" CACHE STRING
 set   (GGML_SYCL_DEVICE_ARCH "" CACHE STRING
                                             "ggml: sycl device architecture")
 
-option(GGML_OPENCL                          "ggml: use OpenCL"                                OFF)
-option(GGML_OPENCL_PROFILING                "ggml: use OpenCL profiling (increases overhead)" OFF)
-option(GGML_OPENCL_EMBED_KERNELS            "ggml: embed kernels"                             ON)
-option(GGML_OPENCL_USE_ADRENO_KERNELS       "ggml: use optimized kernels for Adreno"          ON)
-
 # extra artifacts
 option(GGML_BUILD_TESTS    "ggml: build tests"    ${GGML_STANDALONE})
 option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
@@ -195,7 +178,11 @@ option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
 set(CMAKE_C_STANDARD 11)
 set(CMAKE_C_STANDARD_REQUIRED true)
 
-set(CMAKE_CXX_STANDARD 17)
+if (GGML_SYCL)
+    set(CMAKE_CXX_STANDARD 17)
+else()
+    set(CMAKE_CXX_STANDARD 11)
+endif()
 set(CMAKE_CXX_STANDARD_REQUIRED true)
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)

ggml/include/ggml-amx.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+// buffer_type API
+GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_amx_buffer_type(void);
+
+GGML_BACKEND_API bool ggml_backend_is_amx(ggml_backend_t backend);
+
+// backend API
+GGML_BACKEND_API ggml_backend_t ggml_backend_amx_init(void);
+
+GGML_BACKEND_API void ggml_backend_amx_set_n_threads(ggml_backend_t backend_amx, int n_threads);
+
+GGML_BACKEND_API ggml_backend_reg_t ggml_backend_amx_reg(void);
+
+#ifdef  __cplusplus
+}
+#endif

ggml/include/ggml-backend.h

@@ -190,14 +190,6 @@ extern "C" {
     typedef void                         (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
     // Get additional buffer types provided by the device (returns a NULL-terminated array)
     typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
-    // Set the abort callback for the backend
-    typedef void                         (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
-    // Get a list of feature flags supported by the backend (returns a NULL-terminated array)
-    struct ggml_backend_feature {
-        const char * name;
-        const char * value;
-    };
-    typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);
 
     //
     // Backend registry
@@ -222,14 +214,6 @@ extern "C" {
     // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)
     GGML_API ggml_backend_t ggml_backend_init_best(void);
 
-    // Load a backend from a dynamic library and register it
-    GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);
-    // Unload a backend if loaded dynamically and unregister it
-    GGML_API void               ggml_backend_unload(ggml_backend_reg_t reg);
-    // Load all known backends from dynamic libraries
-    GGML_API void               ggml_backend_load_all(void);
-    GGML_API void               ggml_backend_load_all_from_path(const char * dir_path);
-
     //
     // Backend scheduler
     //

ggml/include/ggml-cpu.h

@@ -7,6 +7,29 @@
 extern "C" {
 #endif
 
+    // Scheduling priorities
+    enum ggml_sched_priority {
+        GGML_SCHED_PRIO_NORMAL,
+        GGML_SCHED_PRIO_MEDIUM,
+        GGML_SCHED_PRIO_HIGH,
+        GGML_SCHED_PRIO_REALTIME
+    };
+
+    // Threadpool params
+    // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults
+    struct ggml_threadpool_params {
+        bool                cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
+        int                 n_threads;                   // number of threads
+        enum ggml_sched_priority prio;                   // thread priority
+        uint32_t            poll;                        // polling level (0 - no polling, 100 - aggressive polling)
+        bool                strict_cpu;                  // strict cpu placement
+        bool                paused;                      // start in paused state
+    };
+
+    struct ggml_threadpool;     // forward declaration, see ggml.c
+
+    typedef struct ggml_threadpool * ggml_threadpool_t;
+
     // the compute plan that needs to be prepared for ggml_graph_compute()
     // since https://github.com/ggerganov/ggml/issues/287
     struct ggml_cplan {
@@ -52,11 +75,14 @@ extern "C" {
     GGML_BACKEND_API float   ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
     GGML_BACKEND_API void    ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
 
-    GGML_BACKEND_API struct ggml_threadpool *      ggml_threadpool_new           (struct ggml_threadpool_params  * params);
-    GGML_BACKEND_API void                          ggml_threadpool_free          (struct ggml_threadpool * threadpool);
-    GGML_BACKEND_API int                           ggml_threadpool_get_n_threads (struct ggml_threadpool * threadpool);
-    GGML_BACKEND_API void                          ggml_threadpool_pause         (struct ggml_threadpool * threadpool);
-    GGML_BACKEND_API void                          ggml_threadpool_resume        (struct ggml_threadpool * threadpool);
+    GGML_BACKEND_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);
+    GGML_BACKEND_API void                          ggml_threadpool_params_init   (struct ggml_threadpool_params * p, int n_threads);
+    GGML_BACKEND_API bool                          ggml_threadpool_params_match  (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);
+    GGML_BACKEND_API struct ggml_threadpool *      ggml_threadpool_new          (struct ggml_threadpool_params  * params);
+    GGML_BACKEND_API void                          ggml_threadpool_free         (struct ggml_threadpool * threadpool);
+    GGML_BACKEND_API int                           ggml_threadpool_get_n_threads(struct ggml_threadpool * threadpool);
+    GGML_BACKEND_API void                          ggml_threadpool_pause        (struct ggml_threadpool * threadpool);
+    GGML_BACKEND_API void                          ggml_threadpool_resume       (struct ggml_threadpool * threadpool);
 
     // ggml_graph_plan() has to be called before ggml_graph_compute()
     // when plan.work_size > 0, caller must allocate memory for plan.work_data
@@ -78,10 +104,10 @@ extern "C" {
     GGML_BACKEND_API int ggml_cpu_has_sse3       (void);
     GGML_BACKEND_API int ggml_cpu_has_ssse3      (void);
     GGML_BACKEND_API int ggml_cpu_has_avx        (void);
-    GGML_BACKEND_API int ggml_cpu_has_avx_vnni   (void);
     GGML_BACKEND_API int ggml_cpu_has_avx2       (void);
     GGML_BACKEND_API int ggml_cpu_has_f16c       (void);
     GGML_BACKEND_API int ggml_cpu_has_fma        (void);
+    GGML_BACKEND_API int ggml_cpu_has_avx_vnni   (void);
     GGML_BACKEND_API int ggml_cpu_has_avx512     (void);
     GGML_BACKEND_API int ggml_cpu_has_avx512_vbmi(void);
     GGML_BACKEND_API int ggml_cpu_has_avx512_vnni(void);
@@ -91,7 +117,6 @@ extern "C" {
     GGML_BACKEND_API int ggml_cpu_has_neon       (void);
     GGML_BACKEND_API int ggml_cpu_has_arm_fma    (void);
     GGML_BACKEND_API int ggml_cpu_has_fp16_va    (void);
-    GGML_BACKEND_API int ggml_cpu_has_dotprod    (void);
     GGML_BACKEND_API int ggml_cpu_has_matmul_int8(void);
     GGML_BACKEND_API int ggml_cpu_has_sve        (void);
     GGML_BACKEND_API int ggml_cpu_get_sve_cnt    (void);  // sve vector length in bytes
@@ -103,14 +128,24 @@ extern "C" {
 
     // Internal types and functions exposed for tests and benchmarks
 
+    typedef void (*ggml_from_float_to_mat_t)
+                                     (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t nr, int64_t k, int64_t bs);
     typedef void (*ggml_vec_dot_t)  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx,
                                        const void * GGML_RESTRICT y, size_t by, int nrc);
+    typedef void (*ggml_gemv_t)     (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x,
+                                       const void * GGML_RESTRICT y, int nr, int nc);
+    typedef void (*ggml_gemm_t)     (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x,
+                                       const void * GGML_RESTRICT y, int nr, int nc);
 
     struct ggml_type_traits_cpu {
         ggml_from_float_t        from_float;
+        ggml_from_float_to_mat_t from_float_to_mat;
         ggml_vec_dot_t           vec_dot;
         enum ggml_type           vec_dot_type;
         int64_t                  nrows; // number of rows to process simultaneously
+        int64_t                  ncols; // number of columns to process simultaneously
+        ggml_gemv_t              gemv;
+        ggml_gemm_t              gemm;
     };
 
     GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type);
@@ -130,6 +165,13 @@ extern "C" {
 
     GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cpu_reg(void);
 
+#ifdef GGML_USE_CPU_HBM
+    GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);
+#endif
+
+    GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cpu_aarch64_buffer_type(void);
+    GGML_BACKEND_API bool ggml_backend_cpu_buft_is_aarch64(ggml_backend_buffer_type_t buft);
+
 #ifdef __cplusplus
 }
 #endif

ggml/include/ggml-opencl.h

@@ -1,26 +0,0 @@
-#ifndef GGML_OPENCL_H
-#define GGML_OPENCL_H
-
-#include "ggml.h"
-#include "ggml-backend.h"
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-//
-// backend API
-//
-GGML_BACKEND_API ggml_backend_t ggml_backend_opencl_init(void);
-GGML_BACKEND_API bool ggml_backend_is_opencl(ggml_backend_t backend);
-
-GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_buffer_type(void);
-GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_host_buffer_type(void);
-
-GGML_BACKEND_API ggml_backend_reg_t ggml_backend_opencl_reg(void);
-
-#ifdef  __cplusplus
-}
-#endif
-
-#endif // GGML_OPENCL_H

ggml/include/ggml.h

@@ -237,9 +237,7 @@
 #define GGML_EXIT_SUCCESS 0
 #define GGML_EXIT_ABORTED 1
 
-#define GGML_ROPE_TYPE_NEOX   2
-#define GGML_ROPE_TYPE_MROPE  8
-#define GGML_ROPE_TYPE_VISION 24
+#define GGML_ROPE_TYPE_NEOX 2
 
 #define GGUF_MAGIC "GGUF"
 
@@ -386,15 +384,12 @@ extern "C" {
         GGML_TYPE_F64     = 28,
         GGML_TYPE_IQ1_M   = 29,
         GGML_TYPE_BF16    = 30,
-        // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
-        // GGML_TYPE_Q4_0_4_8 = 32,
-        // GGML_TYPE_Q4_0_8_8 = 33,
+        GGML_TYPE_Q4_0_4_4 = 31,
+        GGML_TYPE_Q4_0_4_8 = 32,
+        GGML_TYPE_Q4_0_8_8 = 33,
         GGML_TYPE_TQ1_0   = 34,
         GGML_TYPE_TQ2_0   = 35,
-        // GGML_TYPE_IQ4_NL_4_4 = 36,
-        // GGML_TYPE_IQ4_NL_4_8 = 37,
-        // GGML_TYPE_IQ4_NL_8_8 = 38,
-        GGML_TYPE_COUNT   = 39,
+        GGML_TYPE_COUNT,
     };
 
     // precision
@@ -435,6 +430,9 @@ extern "C" {
         GGML_FTYPE_MOSTLY_IQ4_XS  = 22, // except 1d tensors
         GGML_FTYPE_MOSTLY_IQ1_M   = 23, // except 1d tensors
         GGML_FTYPE_MOSTLY_BF16    = 24, // except 1d tensors
+        GGML_FTYPE_MOSTLY_Q4_0_4_4 = 25, // except 1d tensors
+        GGML_FTYPE_MOSTLY_Q4_0_4_8 = 26, // except 1d tensors
+        GGML_FTYPE_MOSTLY_Q4_0_8_8 = 27, // except 1d tensors
     };
 
     // available tensor operations:
@@ -498,7 +496,6 @@ extern "C" {
         GGML_OP_POOL_2D_BACK,
         GGML_OP_UPSCALE, // nearest interpolate
         GGML_OP_PAD,
-        GGML_OP_PAD_REFLECT_1D,
         GGML_OP_ARANGE,
         GGML_OP_TIMESTEP_EMBEDDING,
         GGML_OP_ARGSORT,
@@ -1445,22 +1442,6 @@ extern "C" {
             float                 beta_fast,
             float                 beta_slow);
 
-    GGML_API struct ggml_tensor * ggml_rope_multi(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            struct ggml_tensor  * b,
-            struct ggml_tensor  * c,
-            int                   n_dims,
-            int                   sections[4],
-            int                   mode,
-            int                   n_ctx_orig,
-            float                 freq_base,
-            float                 freq_scale,
-            float                 ext_factor,
-            float                 attn_factor,
-            float                 beta_fast,
-            float                 beta_slow);
-
     // in-place, returns view(a)
     GGML_API struct ggml_tensor * ggml_rope_ext_inplace(
             struct ggml_context * ctx,
@@ -1711,13 +1692,6 @@ extern "C" {
             int                  p2,
             int                  p3);
 
-    // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c]
-    GGML_API struct ggml_tensor * ggml_pad_reflect_1d(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * a,
-            int                   p0,
-            int                   p1);
-
     // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
     // timesteps: [N,]
     // return: [N, dim]
@@ -2220,19 +2194,11 @@ extern "C" {
     GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);
     GGML_API void   gguf_get_meta_data(const struct gguf_context * ctx, void * data);
 
-#ifdef __cplusplus
-    // restrict not standard in C++
-#    if defined(__GNUC__)
-#        define GGML_RESTRICT __restrict__
-#    elif defined(__clang__)
-#        define GGML_RESTRICT __restrict
-#    elif defined(_MSC_VER)
-#        define GGML_RESTRICT __restrict
-#    else
-#        define GGML_RESTRICT
-#    endif
+#ifdef  __cplusplus
+// restrict not standard in C++
+#define GGML_RESTRICT
 #else
-#    define GGML_RESTRICT restrict
+#define GGML_RESTRICT restrict
 #endif
     typedef void (*ggml_to_float_t)  (const void  * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
     typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void  * GGML_RESTRICT y, int64_t k);
@@ -2249,37 +2215,6 @@ extern "C" {
 
     GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type);
 
-    // ggml threadpool
-    // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend
-    // the goal should be to create an API that other backends can use move everything to the ggml base
-
-    // scheduling priorities
-    enum ggml_sched_priority {
-        GGML_SCHED_PRIO_NORMAL,
-        GGML_SCHED_PRIO_MEDIUM,
-        GGML_SCHED_PRIO_HIGH,
-        GGML_SCHED_PRIO_REALTIME
-    };
-
-    // threadpool params
-    // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults
-    struct ggml_threadpool_params {
-        bool                cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
-        int                 n_threads;                   // number of threads
-        enum ggml_sched_priority prio;                   // thread priority
-        uint32_t            poll;                        // polling level (0 - no polling, 100 - aggressive polling)
-        bool                strict_cpu;                  // strict cpu placement
-        bool                paused;                      // start in paused state
-    };
-
-    struct ggml_threadpool;     // forward declaration, see ggml.c
-
-    typedef struct ggml_threadpool * ggml_threadpool_t;
-
-    GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);
-    GGML_API void                          ggml_threadpool_params_init   (struct ggml_threadpool_params * p, int n_threads);
-    GGML_API bool                          ggml_threadpool_params_match  (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);
-
 #ifdef  __cplusplus
 }
 #endif

ggml/src/CMakeLists.txt

@@ -24,7 +24,7 @@ if (NOT MSVC)
     endif()
 endif()
 
-function(ggml_get_flags CCID CCVER)
+function(get_flags CCID CCVER)
     set(C_FLAGS "")
     set(CXX_FLAGS "")
 
@@ -41,7 +41,6 @@ function(ggml_get_flags CCID CCVER)
     elseif (CCID STREQUAL "GNU")
         set(C_FLAGS   -Wdouble-promotion)
         set(CXX_FLAGS -Wno-array-bounds)
-
         if (CCVER VERSION_GREATER_EQUAL 8.1.0)
             list(APPEND CXX_FLAGS -Wextra-semi)
         endif()
@@ -70,7 +69,7 @@ if (GGML_ALL_WARNINGS)
         list(APPEND C_FLAGS   ${WARNING_FLAGS})
         list(APPEND CXX_FLAGS ${WARNING_FLAGS})
 
-        ggml_get_flags(${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})
+        get_flags(${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})
 
         add_compile_options("$<$<COMPILE_LANGUAGE:C>:${C_FLAGS};${GF_C_FLAGS}>"
                             "$<$<COMPILE_LANGUAGE:CXX>:${CXX_FLAGS};${GF_CXX_FLAGS}>")
@@ -194,14 +193,15 @@ endif()
 
 if (WIN32)
     add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
+
+    if (BUILD_SHARED_LIBS)
+        # TODO: should not use this
+        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+    endif()
 endif()
 
 # ggml
 
-if (GGML_BACKEND_DL AND NOT BUILD_SHARED_LIBS)
-    message(FATAL_ERROR "GGML_BACKEND_DL requires BUILD_SHARED_LIBS")
-endif()
-
 add_library(ggml-base
             ../include/ggml.h
             ../include/ggml-alloc.h
@@ -215,7 +215,9 @@ add_library(ggml-base
             ggml-threading.cpp
             ggml-threading.h
             ggml-quants.c
-            ggml-quants.h)
+            ggml-quants.h
+            ggml-aarch64.c
+            ggml-aarch64.h)
 
 target_include_directories(ggml-base PRIVATE .)
 
@@ -224,95 +226,44 @@ add_library(ggml
 
 target_link_libraries(ggml PUBLIC ggml-base)
 
-if (CMAKE_SYSTEM_NAME MATCHES "Linux")
-    target_link_libraries(ggml PRIVATE dl)
-endif()
-
-function(ggml_add_backend_library backend)
-    if (GGML_BACKEND_DL)
-        add_library(${backend} MODULE ${ARGN})
-        # write the shared library to the output directory
-        set_target_properties(${backend} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
-        target_compile_definitions(${backend} PRIVATE GGML_BACKEND_DL)
-    else()
-        add_library(${backend} ${ARGN})
-        target_link_libraries(ggml PUBLIC ${backend})
-        install(TARGETS ${backend} LIBRARY)
-    endif()
-
-    target_link_libraries(${backend} PRIVATE ggml-base)
-    target_include_directories(${backend} PRIVATE ..)
-
-    if (${BUILD_SHARED_LIBS})
-        target_compile_definitions(${backend} PRIVATE GGML_BACKEND_BUILD)
-        target_compile_definitions(${backend} PUBLIC  GGML_BACKEND_SHARED)
-    endif()
-endfunction()
-
 function(ggml_add_backend backend)
     string(TOUPPER "GGML_${backend}" backend_id)
     if (${backend_id})
         string(TOLOWER "ggml-${backend}" backend_target)
         add_subdirectory(${backend_target})
-        message(STATUS "Including ${backend} backend")
-        if (NOT GGML_BACKEND_DL)
+        # check again in case the backend disabled itself
+        # note that this should NOT be the normal behavior, in case of errors the backend should fail the build
+        # however, currently it is necessary for AMX, since it is enabled by default on llama.cpp
+        if (${backend_id})
+            message(STATUS "Including ${backend} backend")
+            if (${BUILD_SHARED_LIBS})
+                target_compile_definitions(${backend_target} PRIVATE GGML_BACKEND_BUILD)
+                target_compile_definitions(${backend_target} PUBLIC  GGML_BACKEND_SHARED)
+            endif()
+            install(TARGETS ${backend_target} LIBRARY)
+            target_link_libraries(ggml PUBLIC ${backend_target})
             string(TOUPPER "GGML_USE_${backend}" backend_use)
             target_compile_definitions(ggml PUBLIC ${backend_use})
         endif()
     endif()
 endfunction()
 
-function(ggml_add_cpu_backend_variant tag_name)
-    set(GGML_CPU_TAG_NAME ${tag_name})
-    # other: OPENMP LLAMAFILE CPU_HBM
-    foreach (feat NATIVE
-                  AVX AVX2 AVX_VNNI FMA F16C
-                  AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16
-                  AMX_TILE AMX_INT8 AMX_BF16)
-        set(GGML_${feat} OFF)
-    endforeach()
-
-    foreach (feat ${ARGN})
-        set(GGML_${feat} ON)
-    endforeach()
-
-    ggml_add_cpu_backend_variant_impl(${tag_name})
-endfunction()
-
 ggml_add_backend(CPU)
-
-if (GGML_CPU_ALL_VARIANTS)
-    if (NOT GGML_BACKEND_DL)
-        message(FATAL_ERROR "GGML_CPU_ALL_VARIANTS requires GGML_BACKEND_DL")
-    endif()
-    ggml_add_cpu_backend_variant(sandybridge    AVX)
-    ggml_add_cpu_backend_variant(haswell        AVX F16C AVX2 FMA)
-    ggml_add_cpu_backend_variant(skylakex       AVX F16C AVX2 FMA AVX512)
-    ggml_add_cpu_backend_variant(icelake        AVX F16C AVX2 FMA AVX512 AVX512_VBMI AVX512_VNNI)
-    if (NOT MSVC)
-        # MSVC doesn't support AVX-VNNI or AMX
-        ggml_add_cpu_backend_variant(alderlake      AVX F16C AVX2 FMA AVX_VNNI)
-        ggml_add_cpu_backend_variant(sapphirerapids AVX F16C AVX2 FMA AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16 AMX_TILE AMX_INT8)
-    endif()
-else ()
-    ggml_add_cpu_backend_variant_impl("")
-endif()
-
+ggml_add_backend(AMX)
 ggml_add_backend(BLAS)
 ggml_add_backend(CANN)
 ggml_add_backend(CUDA)
 ggml_add_backend(HIP)
 ggml_add_backend(Kompute)
 ggml_add_backend(METAL)
-ggml_add_backend(MUSA)
 ggml_add_backend(RPC)
 ggml_add_backend(SYCL)
 ggml_add_backend(Vulkan)
-ggml_add_backend(OpenCL)
+ggml_add_backend(MUSA)
 
 foreach (target ggml-base ggml)
     target_include_directories(${target} PUBLIC    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)
-    target_compile_features   (${target} PRIVATE c_std_11 cxx_std_17) # don't bump
+    target_compile_features   (${target} PRIVATE c_std_11) # don't bump
 endforeach()
 
 target_link_libraries(ggml-base PRIVATE Threads::Threads)

ggml/src/ggml-aarch64.c

@@ -0,0 +1,129 @@
+#define GGML_COMMON_DECL_C
+#include "ggml-common.h"
+
+#include "ggml-aarch64.h"
+#include "ggml-impl.h"
+#include "ggml-quants.h"
+#include <assert.h>
+
+#define UNUSED GGML_UNUSED
+
+static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave) {
+    block_q4_0x4 out;
+
+    for (int i = 0; i < 4; i++) {
+        out.d[i] = in[i].d;
+    }
+
+    const int end = QK4_0 * 2 / blck_size_interleave;
+
+    if (blck_size_interleave == 8) {
+        const uint64_t xor_mask = 0x8888888888888888ULL;
+        for (int i = 0; i < end; ++i) {
+            int src_id = i % 4;
+            int src_offset = (i / 4) * blck_size_interleave;
+            int dst_offset = i * blck_size_interleave;
+
+            uint64_t elems;
+            // Using memcpy to avoid unaligned memory accesses
+            memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t));
+            elems ^= xor_mask;
+            memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t));
+        }
+    } else if (blck_size_interleave == 4) {
+        const uint32_t xor_mask = 0x88888888;
+        for (int i = 0; i < end; ++i) {
+            int src_id = i % 4;
+            int src_offset = (i / 4) * blck_size_interleave;
+            int dst_offset = i * blck_size_interleave;
+
+            uint32_t elems;
+            memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint32_t));
+            elems ^= xor_mask;
+            memcpy(&out.qs[dst_offset], &elems, sizeof(uint32_t));
+        }
+    } else {
+        GGML_ASSERT(false);
+    }
+
+    return out;
+}
+
+// interleave 8 block_q4_0s in blocks of blck_size_interleave
+// returns an interleaved block_q4_0x8
+// in the interleaved block_q4_0x8, place deltas for 8 block_q4_0 blocks
+// first, then interleave quants from 8 block_q4_0s in blocks of blck_size_interleave
+static block_q4_0x8 make_block_q4_0x8(block_q4_0 * in, unsigned int blck_size_interleave) {
+    block_q4_0x8 out;
+
+    for (int i = 0; i < 8; i++) {
+        out.d[i] = in[i].d;
+    }
+
+    const int end = QK4_0 * 4 / blck_size_interleave;
+    const uint64_t xor_mask = 0x8888888888888888ULL;
+
+    for (int i = 0; i < end; ++i) {
+        int src_id = i % 8;
+        int src_offset = (i / 8) * blck_size_interleave;
+        int dst_offset = i * blck_size_interleave;
+
+        uint64_t elems;
+        memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t));
+        elems ^= xor_mask;
+        memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t));
+    }
+
+    return out;
+}
+
+static size_t quantize_q4_0_nr_bl(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, int nrows_interleaved, int blck_size_interleave) {
+    assert(n_per_row % QK4_0 == 0);
+    const int nb = n_per_row / QK4_0;
+
+    void * out_ptr = NULL;
+    if (nrows_interleaved == 8) {
+        out_ptr = (block_q4_0x8 *) dst;
+    }
+    else if (nrows_interleaved == 4) {
+        out_ptr = (block_q4_0x4 *) dst;
+    }
+    assert(nrows_interleaved <= 8);
+    block_q4_0 dst_tmp[8];
+
+    for (int b = 0; b < (nrow * n_per_row); b += nrows_interleaved * n_per_row) {
+
+        for (int64_t x = 0; x < nb; x++) {
+
+            for (int i  = 0; i < nrows_interleaved; i++ ) {
+                quantize_row_q4_0_ref(src + b + i * n_per_row + x * QK4_0, (block_q4_0 *) dst_tmp + i, QK4_0);
+            }
+
+            if (nrows_interleaved == 8) {
+                *(block_q4_0x8 *) out_ptr = make_block_q4_0x8(dst_tmp, blck_size_interleave);
+                out_ptr = (block_q4_0x8 *) out_ptr + 1;
+            }
+            else if (nrows_interleaved == 4) {
+                *(block_q4_0x4 *) out_ptr = make_block_q4_0x4(dst_tmp, blck_size_interleave);
+                out_ptr = (block_q4_0x4 *) out_ptr + 1;
+            }
+        }
+    }
+
+    return ((nrow * n_per_row) / QK4_0 * sizeof(block_q4_0));
+}
+
+size_t quantize_q4_0_4x4(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+    UNUSED(quant_weights);
+    return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 4, 4);
+}
+
+size_t quantize_q4_0_4x8(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+    UNUSED(quant_weights);
+    return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 4, 8);
+}
+
+size_t quantize_q4_0_8x8(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
+    UNUSED(quant_weights);
+    return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 8, 8);
+}

ggml/src/ggml-aarch64.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "ggml.h"
+
+// GGML internal header
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")
+size_t quantize_q4_0_4x4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_0_4x8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_0_8x8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+#ifdef __cplusplus
+}
+#endif
+

ggml/src/ggml-alloc.c

@@ -534,6 +534,7 @@ static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor
         size_t offset = ggml_dyn_tallocr_alloc(alloc, size, node);
         hn->buffer_id = buffer_id;
         hn->offset = offset;
+        return;
     }
 }
 

ggml/src/ggml-backend-impl.h

@@ -8,8 +8,6 @@
 extern "C" {
 #endif
 
-    #define GGML_BACKEND_API_VERSION 1
-
     //
     // Backend buffer type
     //
@@ -65,20 +63,20 @@ extern "C" {
         enum ggml_backend_buffer_usage usage;
     };
 
-    GGML_API ggml_backend_buffer_t ggml_backend_buffer_init(
+    ggml_backend_buffer_t ggml_backend_buffer_init(
                    ggml_backend_buffer_type_t buft,
             struct ggml_backend_buffer_i      iface,
                    void *                     context,
                    size_t                     size);
 
     // do not use directly, use ggml_backend_tensor_copy instead
-    GGML_API bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
+    bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
 
     // multi-buffer
     // buffer that contains a collection of buffers
-    GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
-    GGML_API bool                  ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
-    GGML_API void                  ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
+    ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
+    bool                  ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
+    void                  ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
 
     //
     // Backend (stream)
@@ -201,55 +199,17 @@ extern "C" {
     };
 
     struct ggml_backend_reg {
-        int api_version; // initialize to GGML_BACKEND_API_VERSION
+        // int api_version; // TODO: for dynamic loading
         struct ggml_backend_reg_i iface;
         void * context;
     };
 
+
     // Internal backend registry API
-    GGML_API void ggml_backend_register(ggml_backend_reg_t reg);
-    GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);
-
-    // Add backend dynamic loading support to the backend
-
-    // Initialize the backend
-    typedef ggml_backend_reg_t (*ggml_backend_init_t)(void);
-    // Optional: obtain a score for the backend based on the system configuration
-    // Higher scores are preferred, 0 means the backend is not supported in the current system
-    typedef int                (*ggml_backend_score_t)(void);
-
-#ifdef GGML_BACKEND_DL
-#    ifdef __cplusplus
-#        define GGML_BACKEND_DL_IMPL(reg_fn)                             \
-            extern "C" {                                                 \
-            GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void); \
-            }                                                            \
-            ggml_backend_reg_t ggml_backend_init(void) {                 \
-                return reg_fn();                                         \
-            }
-#        define GGML_BACKEND_DL_SCORE_IMPL(score_fn)       \
-            extern "C" {                                   \
-            GGML_BACKEND_API int ggml_backend_score(void); \
-            }                                              \
-            int ggml_backend_score(void) {                 \
-                return score_fn();                         \
-            }
-#    else
-#        define GGML_BACKEND_DL_IMPL(reg_fn)                              \
-            GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void);  \
-            ggml_backend_reg_t                  ggml_backend_init(void) { \
-                return reg_fn();                                          \
-            }
-#        define GGML_BACKEND_DL_SCORE_IMPL(score_fn)        \
-            GGML_BACKEND_API int ggml_backend_score(void);  \
-            int                  ggml_backend_score(void) { \
-                return score_fn();                          \
-            }
-#    endif
-#else
-#    define GGML_BACKEND_DL_IMPL(reg_fn)
-#    define GGML_BACKEND_DL_SCORE_IMPL(score_fn)
-#endif
+    void ggml_backend_register(ggml_backend_reg_t reg);
+    void ggml_backend_device_register(ggml_backend_dev_t device);
+    // TODO: backends can be loaded as a dynamic library, in which case it needs to export this function
+    // typedef ggml_backend_register_t * (*ggml_backend_init)(void);
 
 #ifdef  __cplusplus
 }

ggml/src/ggml-backend-reg.cpp

@@ -1,34 +1,11 @@
 #include "ggml-backend-impl.h"
 #include "ggml-backend.h"
+#include "ggml-cpu.h"
 #include "ggml-impl.h"
-#include <algorithm>
-#include <codecvt>
 #include <cstring>
-#include <filesystem>
-#include <locale>
-#include <memory>
-#include <string>
-#include <type_traits>
 #include <vector>
 
-#ifdef _WIN32
-#    define WIN32_LEAN_AND_MEAN
-#    ifndef NOMINMAX
-#        define NOMINMAX
-#    endif
-#    include <windows.h>
-#elif defined(__APPLE__)
-#    include <mach-o/dyld.h>
-#    include <dlfcn.h>
-#else
-#    include <dlfcn.h>
-#    include <unistd.h>
-#endif
-
 // Backend registry
-#ifdef GGML_USE_CPU
-#include "ggml-cpu.h"
-#endif
 
 #ifdef GGML_USE_CUDA
 #include "ggml-cuda.h"
@@ -46,10 +23,6 @@
 #include "ggml-vulkan.h"
 #endif
 
-#ifdef GGML_USE_OPENCL
-#include "ggml-opencl.h"
-#endif
-
 #ifdef GGML_USE_BLAS
 #include "ggml-blas.h"
 #endif
@@ -58,6 +31,10 @@
 #include "ggml-rpc.h"
 #endif
 
+#ifdef GGML_USE_AMX
+#  include "ggml-amx.h"
+#endif
+
 #ifdef GGML_USE_CANN
 #include "ggml-cann.h"
 #endif
@@ -66,75 +43,8 @@
 #include "ggml-kompute.h"
 #endif
 
-#ifdef _WIN32
-
-using dl_handle = std::remove_pointer_t<HMODULE>;
-
-struct dl_handle_deleter {
-    void operator()(HMODULE handle) {
-        FreeLibrary(handle);
-    }
-};
-
-static dl_handle * dl_load_library(const std::wstring & path) {
-    // suppress error dialogs for missing DLLs
-    DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
-    SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);
-
-    HMODULE handle = LoadLibraryW(path.c_str());
-
-    SetErrorMode(old_mode);
-
-    return handle;
-}
-
-static dl_handle * dl_load_library(const std::string & path) {
-    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
-    return dl_load_library(converter.from_bytes(path));
-}
-
-static void * dl_get_sym(dl_handle * handle, const char * name) {
-    DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
-    SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);
-
-    void * p = (void *) GetProcAddress(handle, name);
-
-    SetErrorMode(old_mode);
-
-    return p;
-}
-
-#else
-
-using dl_handle = void;
-
-struct dl_handle_deleter {
-    void operator()(void * handle) {
-        dlclose(handle);
-    }
-};
-
-static void * dl_load_library(const std::string & path) {
-    dl_handle * handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL);
-
-    return handle;
-}
-
-static void * dl_get_sym(dl_handle * handle, const char * name) {
-    return dlsym(handle, name);
-}
-
-#endif
-
-using dl_handle_ptr = std::unique_ptr<dl_handle, dl_handle_deleter>;
-
-struct ggml_backend_reg_entry {
-    ggml_backend_reg_t reg;
-    dl_handle_ptr handle;
-};
-
 struct ggml_backend_registry {
-    std::vector<ggml_backend_reg_entry> backends;
+    std::vector<ggml_backend_reg_t> backends;
     std::vector<ggml_backend_dev_t> devices;
 
     ggml_backend_registry() {
@@ -150,9 +60,6 @@ struct ggml_backend_registry {
 #ifdef GGML_USE_VULKAN
         register_backend(ggml_backend_vk_reg());
 #endif
-#ifdef GGML_USE_OPENCL
-        register_backend(ggml_backend_opencl_reg());
-#endif
 #ifdef GGML_USE_CANN
         register_backend(ggml_backend_cann_reg());
 #endif
@@ -162,25 +69,17 @@ struct ggml_backend_registry {
 #ifdef GGML_USE_RPC
         register_backend(ggml_backend_rpc_reg());
 #endif
+#ifdef GGML_USE_AMX
+        register_backend(ggml_backend_amx_reg());
+#endif
 #ifdef GGML_USE_KOMPUTE
         register_backend(ggml_backend_kompute_reg());
 #endif
-#ifdef GGML_USE_CPU
+
         register_backend(ggml_backend_cpu_reg());
-#endif
     }
 
-    ~ggml_backend_registry() {
-        // FIXME: backends cannot be safely unloaded without a function to destroy all the backend resources,
-        // since backend threads may still be running and accessing resources from the dynamic library
-        for (auto & entry : backends) {
-            if (entry.handle) {
-                entry.handle.release(); // NOLINT
-            }
-        }
-    }
-
-    void register_backend(ggml_backend_reg_t reg, dl_handle_ptr handle = nullptr) {
+    void register_backend(ggml_backend_reg_t reg) {
         if (!reg) {
             return;
         }
@@ -189,7 +88,7 @@ struct ggml_backend_registry {
         GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",
             __func__, ggml_backend_reg_name(reg), ggml_backend_reg_dev_count(reg));
 #endif
-        backends.push_back({ reg, std::move(handle) });
+        backends.push_back(reg);
         for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); i++) {
             register_device(ggml_backend_reg_dev_get(reg, i));
         }
@@ -201,76 +100,6 @@ struct ggml_backend_registry {
 #endif
         devices.push_back(device);
     }
-
-    ggml_backend_reg_t load_backend(const char * path, bool silent) {
-        dl_handle_ptr handle { dl_load_library(path) };
-        if (!handle) {
-            if (!silent) {
-                GGML_LOG_ERROR("%s: failed to load %s\n", __func__, path);
-            }
-            return nullptr;
-        }
-
-        auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");
-        if (score_fn && score_fn() == 0) {
-            if (!silent) {
-                GGML_LOG_INFO("%s: backend %s is not supported on this system\n", __func__, path);
-            }
-            return nullptr;
-        }
-
-        auto backend_init_fn = (ggml_backend_init_t) dl_get_sym(handle.get(), "ggml_backend_init");
-        if (!backend_init_fn) {
-            if (!silent) {
-                GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s\n", __func__, path);
-            }
-            return nullptr;
-        }
-
-        ggml_backend_reg_t reg = backend_init_fn();
-        if (!reg || reg->api_version != GGML_BACKEND_API_VERSION) {
-            if (!silent) {
-                if (!reg) {
-                    GGML_LOG_ERROR("%s: failed to initialize backend from %s: ggml_backend_init returned NULL\n", __func__, path);
-                } else {
-                    GGML_LOG_ERROR("%s: failed to initialize backend from %s: incompatible API version (backend: %d, current: %d)\n",
-                        __func__, path, reg->api_version, GGML_BACKEND_API_VERSION);
-                }
-            }
-            return nullptr;
-        }
-
-        GGML_LOG_INFO("%s: loaded %s backend from %s\n", __func__, ggml_backend_reg_name(reg), path);
-
-        register_backend(reg, std::move(handle));
-
-        return reg;
-    }
-
-    void unload_backend(ggml_backend_reg_t reg, bool silent) {
-        auto it = std::find_if(backends.begin(), backends.end(),
-                               [reg](const ggml_backend_reg_entry & entry) { return entry.reg == reg; });
-
-        if (it == backends.end()) {
-            if (!silent) {
-                GGML_LOG_ERROR("%s: backend not found\n", __func__);
-            }
-            return;
-        }
-
-        if (!silent) {
-            GGML_LOG_DEBUG("%s: unloading %s backend\n", __func__, ggml_backend_reg_name(reg));
-        }
-
-        // remove devices
-        devices.erase(
-            std::remove_if(devices.begin(), devices.end(),
-                            [reg](ggml_backend_dev_t dev) { return ggml_backend_dev_backend_reg(dev) == reg; }),
-            devices.end());
-
-        // remove backend
-        backends.erase(it);
-    }
 };
 
 static ggml_backend_registry & get_reg() {
@@ -288,32 +117,23 @@ void ggml_backend_device_register(ggml_backend_dev_t device) {
 }
 
 // Backend (reg) enumeration
-static bool striequals(const char * a, const char * b) {
-    for (; *a && *b; a++, b++) {
-        if (std::tolower(*a) != std::tolower(*b)) {
-            return false;
-        }
-    }
-    return *a == *b;
-}
-
 size_t ggml_backend_reg_count() {
     return get_reg().backends.size();
 }
 
 ggml_backend_reg_t ggml_backend_reg_get(size_t index) {
     GGML_ASSERT(index < ggml_backend_reg_count());
-    return get_reg().backends[index].reg;
+    return get_reg().backends[index];
 }
 
 ggml_backend_reg_t ggml_backend_reg_by_name(const char * name) {
     for (size_t i = 0; i < ggml_backend_reg_count(); i++) {
         ggml_backend_reg_t reg = ggml_backend_reg_get(i);
-        if (striequals(ggml_backend_reg_name(reg), name)) {
+        if (std::strcmp(ggml_backend_reg_name(reg), name) == 0) {
             return reg;
         }
     }
-    return nullptr;
+    return NULL;
 }
 
 // Device enumeration
@@ -329,11 +149,11 @@ ggml_backend_dev_t ggml_backend_dev_get(size_t index) {
 ggml_backend_dev_t ggml_backend_dev_by_name(const char * name) {
     for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
         ggml_backend_dev_t dev = ggml_backend_dev_get(i);
-        if (striequals(ggml_backend_dev_name(dev), name)) {
+        if (strcmp(ggml_backend_dev_name(dev), name) == 0) {
             return dev;
         }
     }
-    return nullptr;
+    return NULL;
 }
 
 ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type) {
@@ -343,14 +163,14 @@ ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type) {
             return dev;
         }
     }
-    return nullptr;
+    return NULL;
 }
 
 // Convenience functions
 ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params) {
     ggml_backend_dev_t dev = ggml_backend_dev_by_name(name);
     if (!dev) {
-        return nullptr;
+        return NULL;
     }
     return ggml_backend_dev_init(dev, params);
 }
@@ -358,7 +178,7 @@ ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params)
 ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params) {
     ggml_backend_dev_t dev = ggml_backend_dev_by_type(type);
     if (!dev) {
-        return nullptr;
+        return NULL;
     }
     return ggml_backend_dev_init(dev, params);
 }
@@ -369,184 +189,7 @@ ggml_backend_t ggml_backend_init_best(void) {
         dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
     }
     if (!dev) {
-        return nullptr;
+        return NULL;
     }
-    return ggml_backend_dev_init(dev, nullptr);
-}
-
-// Dynamic loading
-ggml_backend_reg_t ggml_backend_load(const char * path) {
-    return get_reg().load_backend(path, false);
-}
-
-void ggml_backend_unload(ggml_backend_reg_t reg) {
-    get_reg().unload_backend(reg, true);
-}
-
-static std::string get_executable_path() {
-#if defined(__APPLE__)
-    // get executable path
-    std::vector<char> path;
-    uint32_t size;
-    while (true) {
-        size = path.size();
-        if (_NSGetExecutablePath(path.data(), &size) == 0) {
-            break;
-        }
-        path.resize(size);
-    }
-    std::string base_path(path.data(), size);
-    // remove executable name
-    auto last_slash = base_path.find_last_of('/');
-    if (last_slash != std::string::npos) {
-        base_path = base_path.substr(0, last_slash);
-    }
-    return base_path + "/";
-#elif defined(__linux__)
-    std::string base_path = ".";
-    std::vector<char> path(1024);
-    while (true) {
-        // get executable path
-        ssize_t len = readlink("/proc/self/exe", path.data(), path.size());
-        if (len == -1) {
-            break;
-        }
-        if (len < (ssize_t) path.size()) {
-            base_path = std::string(path.data(), len);
-            // remove executable name
-            auto last_slash = base_path.find_last_of('/');
-            if (last_slash != std::string::npos) {
-                base_path = base_path.substr(0, last_slash);
-            }
-            break;
-        }
-        path.resize(path.size() * 2);
-    }
-
-    return base_path + "/";
-#elif defined(_WIN32)
-    std::vector<char> path(MAX_PATH);
-    DWORD len = GetModuleFileNameA(NULL, path.data(), path.size());
-    if (len == 0) {
-        return "";
-    }
-    std::string base_path(path.data(), len);
-    // remove executable name
-    auto last_slash = base_path.find_last_of('\\');
-    if (last_slash != std::string::npos) {
-        base_path = base_path.substr(0, last_slash);
-    }
-    return base_path + "\\";
-#endif
-}
-
-static std::string backend_filename_prefix() {
-#ifdef _WIN32
-    return "ggml-";
-#else
-    return "libggml-";
-#endif
-}
-
-static std::string backend_filename_suffix() {
-#ifdef _WIN32
-    return ".dll";
-#else
-    return ".so";
-#endif
-}
-
-static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, const char * user_search_path) {
-    // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths
-     // TODO: search system paths
-    std::string file_prefix = backend_filename_prefix() + name + "-";
-    std::vector<std::string> search_paths;
-    if (user_search_path == nullptr) {
-        search_paths.push_back("./");
-        search_paths.push_back(get_executable_path());
-    } else {
-#if defined(_WIN32)
-        search_paths.push_back(std::string(user_search_path) + "\\");
-#else
-        search_paths.push_back(std::string(user_search_path) + "/");
-#endif
-    }
-
-    int best_score = 0;
-    std::string best_path;
-
-    namespace fs = std::filesystem;
-    for (const auto & search_path : search_paths) {
-        if (!fs::exists(search_path)) {
-            continue;
-        }
-        fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);
-        for (const auto & entry : dir_it) {
-            if (entry.is_regular_file()) {
-                std::string filename = entry.path().filename().string();
-                std::string ext = entry.path().extension().string();
-                if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) {
-                    dl_handle_ptr handle { dl_load_library(entry.path().c_str()) };
-                    if (!handle && !silent) {
-                        GGML_LOG_ERROR("%s: failed to load %s\n", __func__, entry.path().string().c_str());
-                    }
-                    if (handle) {
-                        auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");
-                        if (score_fn) {
-                            int s = score_fn();
-#ifndef NDEBUG
-                            GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, entry.path().string().c_str(), s);
-#endif
-                            if (s > best_score) {
-                                best_score = s;
-                                best_path = entry.path().string();
-                            }
-                        } else {
-                            if (!silent) {
-                                GGML_LOG_INFO("%s: failed to find ggml_backend_score in %s\n", __func__, entry.path().string().c_str());
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    if (best_score == 0) {
-        // try to load the base backend
-        for (const auto & search_path : search_paths) {
-            std::string path = search_path + backend_filename_prefix() + name + backend_filename_suffix();
-            if (fs::exists(path)) {
-                return get_reg().load_backend(path.c_str(), silent);
-            }
-        }
-        return nullptr;
-    }
-
-    return get_reg().load_backend(best_path.c_str(), silent);
-}
-
-void ggml_backend_load_all() {
-    ggml_backend_load_all_from_path(nullptr);
-}
-
-void ggml_backend_load_all_from_path(const char * dir_path) {
-#ifdef NDEBUG
-    bool silent = true;
-#else
-    bool silent = false;
-#endif
-
-    ggml_backend_load_best("blas", silent, dir_path);
-    ggml_backend_load_best("cann", silent, dir_path);
-    ggml_backend_load_best("cuda", silent, dir_path);
-    ggml_backend_load_best("hip", silent, dir_path);
-    ggml_backend_load_best("kompute", silent, dir_path);
-    ggml_backend_load_best("metal", silent, dir_path);
-    ggml_backend_load_best("rpc", silent, dir_path);
-    ggml_backend_load_best("sycl", silent, dir_path);
-    ggml_backend_load_best("vulkan", silent, dir_path);
-    ggml_backend_load_best("opencl", silent, dir_path);
-    ggml_backend_load_best("musa", silent, dir_path);
-    ggml_backend_load_best("cpu", silent, dir_path);
+    return ggml_backend_dev_init(dev, NULL);
 }

ggml/src/ggml-backend.cpp

@@ -252,7 +252,6 @@ void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_ten
 }
 
 void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
-    GGML_ASSERT(tensor);
     ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
 
     if (size == 0) {
@@ -267,7 +266,6 @@ void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, siz
 }
 
 void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
-    GGML_ASSERT(tensor);
     ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
 
     if (size == 0) {
@@ -742,8 +740,7 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, st
 
     if (tensor->buffer || (tensor->view_src && tensor->view_src->buffer)) {
         // since the tensor is pre-allocated, it cannot be moved to another backend
-        ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-        GGML_ABORT("pre-allocated tensor (%s) in a buffer (%s) that cannot run the operation (%s)", tensor->name, ggml_backend_buffer_name(buffer), ggml_op_name(tensor->op));
+        GGML_ABORT("pre-allocated tensor (%s) in a backend that cannot run the operation", tensor->name);
     }
 
     // graph input

ggml/src/ggml-blas/CMakeLists.txt

@@ -11,9 +11,12 @@ find_package(BLAS)
 if (BLAS_FOUND)
     message(STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}")
 
-    ggml_add_backend_library(ggml-blas
-                             ggml-blas.cpp
-                            )
+    add_library(ggml-blas
+                ggml-blas.cpp
+                )
+
+    target_link_libraries(ggml-blas PRIVATE ggml-base)
+    target_include_directories(ggml-blas PRIVATE . ..)
 
     if (${GGML_BLAS_VENDOR} MATCHES "Apple")
         add_compile_definitions(ACCELERATE_NEW_LAPACK)

ggml/src/ggml-blas/ggml-blas.cpp

@@ -506,12 +506,9 @@ static const struct ggml_backend_reg_i ggml_backend_blas_reg_i = {
 
 ggml_backend_reg_t ggml_backend_blas_reg(void) {
     static struct ggml_backend_reg ggml_backend_blas_reg = {
-        /* .api_version = */ GGML_BACKEND_API_VERSION,
-        /* .iface       = */ ggml_backend_blas_reg_i,
-        /* .context     = */ NULL,
+        /* .iface   = */ ggml_backend_blas_reg_i,
+        /* .context = */ NULL,
     };
 
     return &ggml_backend_blas_reg;
 }
-
-GGML_BACKEND_DL_IMPL(ggml_backend_blas_reg)

ggml/src/ggml-cann/CMakeLists.txt

@@ -3,34 +3,6 @@ if ("cann${CANN_INSTALL_DIR}" STREQUAL "cann" AND DEFINED ENV{ASCEND_TOOLKIT_HOM
     message(STATUS "CANN: updated CANN_INSTALL_DIR from ASCEND_TOOLKIT_HOME=$ENV{ASCEND_TOOLKIT_HOME}")
 endif()
 
-# Auto-detech Soc type and Soc version, if detect failed, will abort build
-set(SOC_VERSION "")
-function(detect_ascend_soc_type SOC_VERSION)
-    execute_process(
-        COMMAND bash -c "npu-smi info|awk -F' ' 'NF > 0 && NR==7 {print $3}'"
-        OUTPUT_VARIABLE npu_info
-        RESULT_VARIABLE npu_result
-        OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-    if("${npu_info}" STREQUAL "" OR ${npu_result})
-        message(FATAL_ERROR "Auto-detech ascend soc type failed, please specify manually or check ascend device working normally.")
-    endif()
-    set(${SOC_VERSION} "Ascend${npu_info}" PARENT_SCOPE)
-endfunction()
-
-if(NOT SOC_TYPE)
-    detect_ascend_soc_type(SOC_VERSION)
-    set(SOC_TYPE "${SOC_VERSION}")
-    message(STATUS "CANN: SOC_VERSION auto-detected is:${SOC_VERSION}")
-endif()
-
-string(TOLOWER ${SOC_TYPE} SOC_VERSION) # SOC_VERSION need lower
-
-# Construct Soc specify compile option: ASCEND_#Soc_Major_SN. Such as ASCEND_910B, ASCEND_310P.
-string(REGEX MATCH "[0-9]+[a-zA-Z]" SOC_TYPE_MAJOR_SN "${SOC_VERSION}")
-set(SOC_TYPE_COMPILE_OPTION "ASCEND_${SOC_TYPE_MAJOR_SN}")
-string(TOUPPER ${SOC_TYPE_COMPILE_OPTION} SOC_TYPE_COMPILE_OPTION)
-
 if (CANN_INSTALL_DIR)
     # Only Support Linux.
     if (NOT UNIX)
@@ -62,13 +34,11 @@ if (CANN_INSTALL_DIR)
 
     file(GLOB GGML_SOURCES_CANN "*.cpp")
 
-    ggml_add_backend_library(ggml-cann ${GGML_SOURCES_CANN})
-    target_link_libraries(ggml-cann PRIVATE ${CANN_LIBRARIES})
-    target_include_directories(ggml-cann PRIVATE ${CANN_INCLUDE_DIRS})
+    add_library(ggml-cann ${GGML_SOURCES_CANN})
+    target_link_libraries(ggml-cann PRIVATE ggml-base ${CANN_LIBRARIES})
+    target_include_directories(ggml-cann PRIVATE . .. ${CANN_INCLUDE_DIRS})
     target_link_directories(ggml-cann PRIVATE ${CANN_INSTALL_DIR}/lib64)
 
-    target_compile_definitions(ggml-cann PRIVATE "-D${SOC_TYPE_COMPILE_OPTION}")
-
     message(STATUS "CANN: CANN_INCLUDE_DIRS =  ${CANN_INCLUDE_DIRS}")
     message(STATUS "CANN: CANN_LIBRARIES =  ${CANN_LIBRARIES}")
 else()

ggml/src/ggml-cann/aclnn_ops.cpp

@@ -22,14 +22,11 @@
 
 #include "aclnn_ops.h"
 
-#include <aclnnop/aclnn_addcdiv.h>
 #include <aclnnop/aclnn_avgpool2d.h>
-#include <aclnnop/aclnn_batch_matmul.h>
 #include <aclnnop/aclnn_cast.h>
 #include <aclnnop/aclnn_constant_pad_nd.h>
 #include <aclnnop/aclnn_copy.h>
 #include <aclnnop/aclnn_cos.h>
-#include <aclnnop/aclnn_div.h>
 #include <aclnnop/aclnn_exp.h>
 #include <aclnnop/aclnn_fill_scalar.h>
 #include <aclnnop/aclnn_group_norm.h>
@@ -37,7 +34,6 @@
 #include <aclnnop/aclnn_layer_norm.h>
 #include <aclnnop/aclnn_matmul.h>
 #include <aclnnop/aclnn_max_pool.h>
-#include <aclnnop/aclnn_mm.h>
 #include <aclnnop/aclnn_permute.h>
 #include <aclnnop/aclnn_pow_tensor_tensor.h>
 #include <aclnnop/aclnn_reduce_sum.h>
@@ -57,7 +53,6 @@
 #include <exception>
 #include <vector>
 
-#include "ggml-impl.h"
 #include "kernels/ascendc_kernels.h"
 
 #define GGML_COMMON_DECL_C
@@ -246,14 +241,10 @@ void ggml_cann_concat(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
     aclTensor* acl_src1 = ggml_cann_create_tensor(src1);
     aclTensor* acl_dst = ggml_cann_create_tensor(dst);
 
-    const int32_t dim = ggml_get_op_params_i32(dst, 0);
-
-    GGML_ASSERT(dim >= 0 && dim < 4);
-    int32_t acl_dim = 3 - dim;
-
+    int64_t concat_dim = 1;
     aclTensor* tensors[] = {acl_src0, acl_src1};
     aclTensorList* tensorList = aclCreateTensorList(tensors, 2);
-    aclnn_concat(ctx, tensorList, acl_dst, acl_dim);
+    aclnn_concat(ctx, tensorList, acl_dst, concat_dim);
 
     ACL_CHECK(aclDestroyTensorList(tensorList));
     ACL_CHECK(aclDestroyTensor(acl_dst));
@@ -1105,9 +1096,9 @@ static aclTensor* aclnn_zero(ggml_backend_cann_context& ctx, void* buffer,
 }
 
 /**
- * @brief Creates an ACL tensor initialized with value using a provided buffer.
+ * @brief Creates an ACL tensor initialized with ones using a provided buffer.
  *
- * This function initializes a tensor with value using the specified buffer and
+ * This function initializes a tensor with ones using the specified buffer and
  * tensor parameters.
  *
  * @param ctx The context for the CANN backend operations.
@@ -1120,12 +1111,12 @@ static aclTensor* aclnn_zero(ggml_backend_cann_context& ctx, void* buffer,
  * @param type_size The size of each element in the tensor data type.
  * @param value The value to be used for initializing the tensor (default
  * is 1.0).
- * @return An ACL tensor initialized with value.
+ * @return An ACL tensor initialized with ones.
  */
-static aclTensor* aclnn_values(ggml_backend_cann_context& ctx, void* buffer,
-                               size_t n_bytes, int64_t* ne, int64_t dims,
-                               aclDataType type, size_t type_size,
-                               float value = 1.0f) {
+static aclTensor* aclnn_ones(ggml_backend_cann_context& ctx, void* buffer,
+                             size_t n_bytes, int64_t* ne, int64_t dims,
+                             aclDataType type, size_t type_size,
+                             float value = 1.0f) {
     aclTensor* acl_tensor =
         aclnn_zero(ctx, buffer, n_bytes, ne, dims, type, type_size);
     float alpha_host = 1.0f;
@@ -1167,7 +1158,7 @@ void ggml_cann_rms_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
     size_t one_tensor_n_bytes = src->ne[0] * ggml_element_size(src);
     ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), one_tensor_n_bytes);
 
-    aclTensor* acl_gamma = aclnn_values(
+    aclTensor* acl_gamma = aclnn_ones(
         ctx, one_tensor_allocator.get(), one_tensor_n_bytes, src->ne, 1,
         ggml_cann_type_mapping(src->type), ggml_element_size(src));
 
@@ -1211,9 +1202,9 @@ void ggml_cann_diag_mask(ggml_backend_cann_context& ctx, ggml_tensor* dst,
     ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), one_tensor_n_bytes);
 
     aclTensor* mask_tensor =
-        aclnn_values(ctx, one_tensor_allocator.get(), one_tensor_n_bytes,
-                     src->ne, GGML_MAX_DIMS, ggml_cann_type_mapping(src->type),
-                     ggml_element_size(src), value);
+        aclnn_ones(ctx, one_tensor_allocator.get(), one_tensor_n_bytes, src->ne,
+                   GGML_MAX_DIMS, ggml_cann_type_mapping(src->type),
+                   ggml_element_size(src), value);
 
     uint64_t workspaceSize = 0;
     aclOpExecutor* executor;
@@ -1446,6 +1437,10 @@ void ggml_cann_im2col(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
     ggml_tensor* src0 = dst->src[0];  // kernel
     ggml_tensor* src1 = dst->src[1];  // input
 
+    GGML_ASSERT(src0->type == GGML_TYPE_F16);
+    GGML_ASSERT(src1->type == GGML_TYPE_F32);
+    GGML_ASSERT(dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);
+
     GGML_TENSOR_BINARY_OP_LOCALS;
 
     // aclnnIm2col only works on 2D. set s1, p1, d1 to 1 to perform 2D
@@ -1467,6 +1462,9 @@ void ggml_cann_im2col(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
     const int64_t OH = is_2D ? ne2 : 1;
     const int64_t OW = ne1;
 
+    GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+    GGML_ASSERT(nb10 == sizeof(float));
+
     // memory allocated increased to 3x when is_2D == false
     const int64_t n_bytes_factor = is_2D ? 1 : 3;
 
@@ -1770,92 +1768,6 @@ static void aclnn_sin(ggml_backend_cann_context& ctx, aclTensor* acl_src,
     ACL_CHECK(aclnnSin(workspaceAddr, workspaceSize, executor, ctx.stream()));
 }
 
-/**
- * @brief Performs element-wise division of tensor1 by tensor2 , multiplies the
- result by the scalar value and adds it to self .
- *
- * Performs element-wise division of tensor1 by tensor2,
- * multiplies the result by the scalar value and adds it to self .
- * The operation is defined as:
- * \f[
- *     \text{out}_i = \text{selft}_i + \text{value} \times
- \frac{\text{tensor1}_i}{\text{tensor2}_i}
- * \f]
-
- * @param ctx The context for the CANN backend operations.
- * @param acl_self The source tensor on which the addcdiv function will be
- applied.
- * @param tensor1 Numerator tensor.
- * @param tensor2 Denominator tensor.
- * @param value The value to be used for coefficient.
- */
-static void aclnn_inplace_addcdiv(ggml_backend_cann_context& ctx,
-                                  aclTensor* acl_self, aclTensor* tensor1,
-                                  aclTensor* tensor2, float value) {
-    uint64_t workspaceSize = 0;
-    aclOpExecutor* executor;
-    void* workspaceAddr = nullptr;
-    aclScalar* acl_value = aclCreateScalar(&value, aclDataType::ACL_FLOAT);
-
-    ACL_CHECK(aclnnInplaceAddcdivGetWorkspaceSize(
-        acl_self, tensor1, tensor2, acl_value, &workspaceSize, &executor));
-    if (workspaceSize > 0) {
-        ggml_cann_pool_alloc workspace_allocator(ctx.pool(), workspaceSize);
-        workspaceAddr = workspace_allocator.get();
-    }
-
-    ACL_CHECK(aclnnInplaceAddcdiv(workspaceAddr, workspaceSize, executor,
-                                  ctx.stream()));
-}
-
-/**
- * @brief Matrix division, optionally in-place.
- *
- * This function division each element of the source tensor `acl_src` by the
- * tensor `acl_other` and stores the result in the destination tensor `acl_dst`.
- * If `inplace` is true, `acl_dst` will not be used and the operation is
- * performed in-place on `acl_src`. The operation is defined as: \f[
- *     \text{dst}_i = \frac{\text{acl_src}_i}{\text{acl_other}_i}
- * \f]
- *
- * @param ctx The context for the CANN backend operations.
- * @param acl_src Numerator tensor..
- * @param acl_other Denominator tensor.
- * @param acl_dst The destination tensor where the result will be stored if
- * `inplace` is false.
- * @param inplace Flag indicating whether to perform the operation in-place on
- * `acl_src`.
- */
-static void aclnn_div_tensor(ggml_backend_cann_context& ctx, aclTensor* acl_src,
-                             aclTensor* acl_other, aclTensor* acl_dst,
-                             bool inplace) {
-    uint64_t workspaceSize = 0;
-    aclOpExecutor* executor;
-    void* workspaceAddr = nullptr;
-
-    if (inplace) {
-        ACL_CHECK(aclnnInplaceDivGetWorkspaceSize(acl_src, acl_other,
-                                                  &workspaceSize, &executor));
-        if (workspaceSize > 0) {
-            ggml_cann_pool_alloc workspace_allocator(ctx.pool(), workspaceSize);
-            workspaceAddr = workspace_allocator.get();
-        }
-
-        ACL_CHECK(aclnnInplaceDiv(workspaceAddr, workspaceSize, executor,
-                                  ctx.stream()));
-    } else {
-        ACL_CHECK(aclnnDivGetWorkspaceSize(acl_src, acl_other, acl_dst,
-                                           &workspaceSize, &executor));
-        if (workspaceSize > 0) {
-            ggml_cann_pool_alloc workspace_allocator(ctx.pool(), workspaceSize);
-            workspaceAddr = workspace_allocator.get();
-        }
-
-        ACL_CHECK(
-            aclnnDiv(workspaceAddr, workspaceSize, executor, ctx.stream()));
-    }
-}
-
 void ggml_cann_timestep_embedding(ggml_backend_cann_context& ctx,
                                   ggml_tensor* dst) {
     const ggml_tensor* src = dst->src[0];
@@ -2399,16 +2311,7 @@ void ggml_cann_get_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
                                ctx.stream()));
 
     switch (src0->type) {
-        case GGML_TYPE_F32: {
-#ifdef ASCEND_310P
-            // Special operation for get_row_f32 kernel of 310P: clear the
-            // content of dest data buffer when row is not aligned to 32 bytes
-            if ((src0->ne[0] % 8) != 0) {
-                size_t dst_len = src1->ne[0] * src1->ne[1] * src1->ne[2] *
-                                 src0->ne[0] * ggml_type_size(GGML_TYPE_F32);
-                ACL_CHECK(aclrtMemset((char*)dst->data, dst_len, 0, dst_len));
-            }
-#endif
+        case GGML_TYPE_F32:
             aclrtlaunch_ascendc_get_row_f32(
                 24, ctx.stream(), src0->data, src1->data, dst->data,
                 ((ggml_tensor*)src0->extra)->ne,
@@ -2417,19 +2320,7 @@ void ggml_cann_get_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
                 ((ggml_tensor*)src1->extra)->nb, ((ggml_tensor*)dst->extra)->ne,
                 ((ggml_tensor*)dst->extra)->nb);
             break;
-        }
-        case GGML_TYPE_F16: {
-#ifdef ASCEND_310P
-            // Special operation for get_row_f16 kernel of 310P: clear the
-            // content of dest data buffer when row is not aligned to 32 bytes
-            if ((src0->ne[0] % 16) != 0) {
-                size_t dst_len =
-                    src1->ne[0] * src1->ne[1] * src1->ne[2] * src0->ne[0] *
-                    ggml_type_size(
-                        GGML_TYPE_F32);  // out is also f32, even input is f16
-                ACL_CHECK(aclrtMemset((char*)dst->data, dst_len, 0, dst_len));
-            }
-#endif
+        case GGML_TYPE_F16:
             aclrtlaunch_ascendc_get_row_f16(
                 24, ctx.stream(), src0->data, src1->data, dst->data,
                 ((ggml_tensor*)src0->extra)->ne,
@@ -2438,7 +2329,6 @@ void ggml_cann_get_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
                 ((ggml_tensor*)src1->extra)->nb, ((ggml_tensor*)dst->extra)->ne,
                 ((ggml_tensor*)dst->extra)->nb);
             break;
-        }
         case GGML_TYPE_Q4_0:
             aclrtlaunch_ascendc_get_row_q4_0(
                 24, ctx.stream(), src0->data, src1->data, dst->data,
@@ -2517,6 +2407,7 @@ static void aclnn_mat_mul(ggml_backend_cann_context& ctx, aclTensor* acl_input,
                           aclTensor* acl_weight, aclTensor* acl_dst) {
     int8_t cube_math_type = 1;  // ALLOW_FP32_DOWN_PRECISION, when input is
                                 // fp32, atlas a2 will transpose it to HFLOAT32.
+
     uint64_t workspaceSize = 0;
     aclOpExecutor* executor;
     void* workspaceAddr = nullptr;
@@ -2534,81 +2425,6 @@ static void aclnn_mat_mul(ggml_backend_cann_context& ctx, aclTensor* acl_input,
         aclnnMatmul(workspaceAddr, workspaceSize, executor, ctx.stream()));
 }
 
-/**
- * @brief Performs matrix multiplication of two 2D tensors.
- *
- * This function computes the matrix multiplication of the input tensor
- * `acl_input` and the weight tensor `acl_weight`, and stores the result in the
- * destination tensor `acl_dst`.
- * The operation is defined as:
- * \f[
- *     \text {acl_dst}=\text {acl_input@acl_weight}
- * \f]
- *
- * @param ctx The context for the CANN backend operations.
- * @param acl_input The input tensor for the matrix multiplication.
- * @param acl_weight The weight tensor for the matrix multiplication.
- * @param acl_dst The destination tensor where the result of the matrix
- * multiplication will be stored.
- */
-static void aclnn_mat_mul_2d(ggml_backend_cann_context& ctx,
-                             aclTensor* acl_input, aclTensor* acl_weight,
-                             aclTensor* acl_dst) {
-    int8_t cube_math_type = 2;
-    uint64_t workspaceSize = 0;
-    aclOpExecutor* executor;
-    void* workspaceAddr = nullptr;
-
-    ACL_CHECK(aclnnMmGetWorkspaceSize(acl_input, acl_weight, acl_dst,
-                                      cube_math_type, &workspaceSize,
-                                      &executor));
-
-    if (workspaceSize > 0) {
-        ggml_cann_pool_alloc workspace_allocator(ctx.pool(), workspaceSize);
-        workspaceAddr = workspace_allocator.get();
-    }
-
-    ACL_CHECK(aclnnMm(workspaceAddr, workspaceSize, executor, ctx.stream()));
-}
-
-/**
- * @brief Performs matrix multiplication of two 3D tensors.
- *
- * This function computes the matrix multiplication of the input tensor
- * `acl_input` and the weight tensor `acl_weight`, and stores the result in the
- * destination tensor `acl_dst`.
- * The operation is defined as:
- * \f[
- *     \text {acl_dst}=\text {acl_input@acl_weight}
- * \f]
- *
- * @param ctx The context for the CANN backend operations.
- * @param acl_input The input tensor for the matrix multiplication.
- * @param acl_weight The weight tensor for the matrix multiplication.
- * @param acl_dst The destination tensor where the result of the matrix
- * multiplication will be stored.
- */
-static void aclnn_mat_mul_3d(ggml_backend_cann_context& ctx,
-                             aclTensor* acl_input, aclTensor* acl_weight,
-                             aclTensor* acl_dst) {
-    int8_t cube_math_type = 2;
-    uint64_t workspaceSize = 0;
-    aclOpExecutor* executor;
-    void* workspaceAddr = nullptr;
-
-    ACL_CHECK(aclnnBatchMatMulGetWorkspaceSize(acl_input, acl_weight, acl_dst,
-                                               cube_math_type, &workspaceSize,
-                                               &executor));
-
-    if (workspaceSize > 0) {
-        ggml_cann_pool_alloc workspace_allocator(ctx.pool(), workspaceSize);
-        workspaceAddr = workspace_allocator.get();
-    }
-
-    ACL_CHECK(
-        aclnnBatchMatMul(workspaceAddr, workspaceSize, executor, ctx.stream()));
-}
-
 /**
  * @brief Performs matrix multiplication with floating-point precision on
  * tensors using the CANN backend.
@@ -2630,39 +2446,20 @@ static void ggml_cann_mat_mul_fp(ggml_backend_cann_context& ctx,
     // broadcast, when weight ne2 or ne3 is not 1, weight need repeat.
     BCAST_MUL_MAT_SHAPE(input, weight, dst);
 
-    int64_t n_dims = bcast_dims;
-    if (bcast_input_ne[3] == bcast_weight_ne[3] && bcast_input_ne[3] == 1) {
-        if (bcast_input_ne[2] == 1 && bcast_weight_ne[2] == 1) {
-            n_dims = 2;
-        } else if (bcast_input_ne[2] == 1) {
-            n_dims = 3;
-        }
-    }
-
-    aclTensor* acl_input_tensor =
-        ggml_cann_create_tensor(input, bcast_input_ne, bcast_input_nb, n_dims);
+    // transpose weight: [1,2,3,4] -> [1,2,4,3]
     int64_t transpose_ne[] = {bcast_weight_ne[1], bcast_weight_ne[0],
                               bcast_weight_ne[2], bcast_weight_ne[3],
                               bcast_weight_ne[4], bcast_weight_ne[5]};
     size_t transpose_nb[] = {bcast_weight_nb[1], bcast_weight_nb[0],
                              bcast_weight_nb[2], bcast_weight_nb[3],
                              bcast_weight_nb[4], bcast_weight_nb[5]};
-    aclTensor* acl_weight_tensor =
-        ggml_cann_create_tensor(weight, transpose_ne, transpose_nb, n_dims);
-    aclTensor* acl_dst =
-        ggml_cann_create_tensor(dst, bcast_dst_ne, bcast_dst_nb, n_dims);
 
-    switch (n_dims) {
-        case 2:
-            aclnn_mat_mul_2d(ctx, acl_input_tensor, acl_weight_tensor, acl_dst);
-            break;
-        case 3:
-            aclnn_mat_mul_3d(ctx, acl_input_tensor, acl_weight_tensor, acl_dst);
-            break;
-        default:
-            aclnn_mat_mul(ctx, acl_input_tensor, acl_weight_tensor, acl_dst);
-            break;
-    }
+    aclTensor* acl_weight_tensor =
+        ggml_cann_create_tensor(weight, transpose_ne, transpose_nb, bcast_dims);
+    aclTensor* acl_input_tensor =
+        ggml_cann_create_tensor(input, BCAST_MUL_MAT_PARAM(input));
+    aclTensor* acl_dst = ggml_cann_create_tensor(dst, BCAST_MUL_MAT_PARAM(dst));
+    aclnn_mat_mul(ctx, acl_input_tensor, acl_weight_tensor, acl_dst);
 
     ACL_CHECK(aclDestroyTensor(acl_weight_tensor));
     ACL_CHECK(aclDestroyTensor(acl_input_tensor));
@@ -2683,47 +2480,51 @@ static void ggml_cann_mat_mul_fp(ggml_backend_cann_context& ctx,
  * multiplication will be stored.
  */
 static void ggml_cann_mul_mat_quant(ggml_backend_cann_context& ctx,
-                                    ggml_tensor* dst,
-                                    const enum ggml_type type) {
+                                   ggml_tensor* dst,
+                                   const enum ggml_type type) {
     ggml_tensor* src0 = dst->src[0];  // weight
     ggml_tensor* src1 = dst->src[1];  // input
 
-    // The shape of the weight is NCHW.
-    // Matrix multiplication uses HW dims.
-    // HC is regarded as batch.
-    // weight need transpose.
+    // The shape of the weight is NCHW. Matrix multiplication uses HW dims. HC
+    // is regarded as batch. weight need transpose.
+    int64_t weight_ne[] = {src0->ne[1], src0->ne[0]};
     float weight_elem_size;
     if (type == GGML_TYPE_Q4_0) {
         weight_elem_size = float(sizeof(uint8_t)) / 2;
-    } else if (type == GGML_TYPE_Q8_0) {
+    }
+    else if (type == GGML_TYPE_Q8_0) {
         weight_elem_size = float(sizeof(uint8_t));
-    } else {
+    }
+    else {
         GGML_ABORT("Only support Q4_0 and Q8_0 MUL_MAT");
     }
-    float weight_nb[] = {src0->ne[0] * weight_elem_size, weight_elem_size};
-    size_t weight_stride = src0->ne[1] * src0->ne[0] * weight_elem_size;
+    float weight_nb[] = {weight_elem_size * src0->ne[0], weight_elem_size};
+
+    // size of one matrix is element_size * height * width.
+    size_t weight_stride = weight_elem_size * src0->ne[0] * src0->ne[1];
     size_t weight_size = weight_stride * src0->ne[2] * src0->ne[3];
 
     // scale stored at the end of weight. Also need transpose.
+    GGML_ASSERT(QK4_0 == QK8_0);
+    int64_t scale_ne[] = {src0->ne[1], src0->ne[0] / QK8_0};
     size_t scale_elem_size = sizeof(uint16_t);
     size_t scale_nb[] = {src0->ne[0] / QK8_0 * scale_elem_size,
                          scale_elem_size};
-    size_t scale_stride = src0->ne[1] * src0->ne[0] / QK8_0 * scale_elem_size;
+    size_t scale_stride = scale_elem_size * src0->ne[0] * src0->ne[1] / QK8_0;
     char* scale_offset = (char*)src0->data + weight_size;
 
     // input
+    void* input_buffer;
     size_t input_elem_size = sizeof(uint16_t);
     int64_t input_ne[] = {src1->ne[0], src1->ne[1]};
-    size_t input_nb[] = {input_elem_size, input_ne[0] * input_elem_size};
-    size_t input_stride = input_ne[0] * input_ne[1] * input_elem_size;
-    ggml_cann_pool_alloc input_alloctor(ctx.pool());
-    void* input_buffer = src1->data;
+    size_t input_nb[] = {input_elem_size, input_elem_size * src1->ne[0]};
+    size_t input_stride = input_elem_size * src1->ne[0] * src1->ne[1];
 
-    // case in
+    ggml_cann_pool_alloc input_alloctor(ctx.pool());
     if (src1->type != GGML_TYPE_F16) {
         aclTensor* acl_src1_tensor = ggml_cann_create_tensor(src1);
-        input_buffer =
-            input_alloctor.alloc(ggml_nelements(src1) * input_elem_size);
+        input_alloctor.alloc(ggml_nelements(src1) * input_elem_size);
+        input_buffer = input_alloctor.get();
 
         int64_t* input_cast_ne = src1->ne;
         size_t input_cast_nb[GGML_MAX_DIMS];
@@ -2736,136 +2537,85 @@ static void ggml_cann_mul_mat_quant(ggml_backend_cann_context& ctx,
             input_buffer, ACL_FLOAT16, input_elem_size, input_cast_ne,
             input_cast_nb, GGML_MAX_DIMS);
         aclnn_cast(ctx, acl_src1_tensor, acl_input_tensor, ACL_FLOAT16);
-
         ACL_CHECK(aclDestroyTensor(acl_input_tensor));
         ACL_CHECK(aclDestroyTensor(acl_src1_tensor));
+    } else {
+        input_buffer = src1->data;
     }
 
     // output
     size_t output_elem_size = sizeof(uint16_t);
-    size_t output_nb[] = {output_elem_size, dst->ne[0] * output_elem_size};
-    ggml_cann_pool_alloc output_allocator(ctx.pool());
-    void* output_buffer =
-        output_allocator.alloc(ggml_nelements(dst) * output_elem_size);
-    size_t output_stride = dst->ne[0] * dst->ne[1] * output_elem_size;
+    int64_t output_ne[] = {dst->ne[0], dst->ne[1]};
+    size_t output_nb[] = {output_elem_size, output_elem_size * dst->ne[0]};
+    ggml_cann_pool_alloc output_alloctor(
+        ctx.pool(), ggml_nelements(dst) * output_elem_size);
+    void* output_buffer = output_alloctor.get();
+    size_t output_stride = output_elem_size * dst->ne[0] * dst->ne[1];
 
     // aclnn
-    int64_t max_elem_size = 65535;
-    int64_t split_size = (src0->ne[1] / max_elem_size) + 1;
-    ggml_cann_pool_alloc workspace_allocator(ctx.pool());
-    aclOpExecutor* executor = nullptr;
     uint64_t workspaceSize = 0;
+    aclOpExecutor* executor;
     void* workspaceAddr = nullptr;
+
     for (int64_t n1 = 0; n1 < src1->ne[3]; n1++) {
         for (int64_t c1 = 0; c1 < src1->ne[2]; c1++) {
             int64_t n0 = n1 / (src1->ne[3] / src0->ne[3]);
             int64_t c0 = c1 / (src1->ne[2] / src0->ne[2]);
 
-            int64_t batch1 = (n1 * src1->ne[2]) + c1;
-            int64_t batch0 = (n0 * src0->ne[2]) + c0;
+            int64_t batch1 = n1 * src1->ne[2] + c1;
+            int64_t batch0 = n0 * src0->ne[2] + c0;
 
             aclTensor* acl_input_tensor = ggml_cann_create_tensor(
                 (char*)input_buffer + batch1 * input_stride, ACL_FLOAT16,
                 input_elem_size, input_ne, input_nb, 2);
-
-            // first split
-            int64_t weight_ne_offset = 0;
-            int64_t weight_ne[2] = {
-                max_elem_size > src0->ne[1] ? src0->ne[1] : max_elem_size,
-                src0->ne[0]};
-            int64_t scale_ne_offset = 0;
-            int64_t scale_ne[2] = {weight_ne[0], weight_ne[1] / QK8_0};
-            int64_t output_ne_offset = 0;
-            int64_t output_ne[2] = {weight_ne[0], dst->ne[1]};
-
             aclTensor* acl_weight_tensor = ggml_cann_create_tensor(
                 (char*)src0->data + batch0 * weight_stride,
                 ggml_cann_type_mapping(type), weight_elem_size, weight_ne,
-                weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);
+                weight_nb, 2);
             aclTensor* acl_scale_tensor = ggml_cann_create_tensor(
                 scale_offset + batch0 * scale_stride, ACL_FLOAT16,
-                scale_elem_size, scale_ne, scale_nb, 2, ACL_FORMAT_ND,
-                scale_ne_offset);
+                scale_elem_size, scale_ne, scale_nb, 2);
             aclTensor* acl_output_tensor = ggml_cann_create_tensor(
                 (char*)output_buffer + batch1 * output_stride, ACL_FLOAT16,
-                output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND,
-                output_ne_offset);
+                output_elem_size, output_ne, output_nb, 2);
 
             ACL_CHECK(aclnnWeightQuantBatchMatmulV2GetWorkspaceSize(
                 acl_input_tensor, acl_weight_tensor, acl_scale_tensor, nullptr,
                 nullptr, nullptr, nullptr, QK8_0, acl_output_tensor,
                 &workspaceSize, &executor));
-            if (workspaceAddr == nullptr) {
-                workspaceAddr = workspace_allocator.alloc(workspaceSize);
+
+            if (workspaceSize > 0 && workspaceAddr == nullptr) {
+                ggml_cann_pool_alloc workspace_allocator(ctx.pool(),
+                                                         workspaceSize);
+                workspaceAddr = workspace_allocator.get();
             }
+
             ACL_CHECK(aclnnWeightQuantBatchMatmulV2(
                 workspaceAddr, workspaceSize, executor, ctx.stream()));
 
+            ACL_CHECK(aclDestroyTensor(acl_input_tensor));
             ACL_CHECK(aclDestroyTensor(acl_weight_tensor));
             ACL_CHECK(aclDestroyTensor(acl_scale_tensor));
             ACL_CHECK(aclDestroyTensor(acl_output_tensor));
-
-            // other splits
-            for (int64_t split = 1; split < split_size; split++) {
-                weight_ne_offset +=
-                    weight_elem_size * weight_ne[0] * weight_ne[1];
-                weight_ne[0] = max_elem_size * (split + 1) > src0->ne[1]
-                                   ? src0->ne[1] - (max_elem_size * split)
-                                   : max_elem_size;
-                scale_ne_offset += scale_elem_size * scale_ne[0] * scale_ne[1];
-                scale_ne[0] = weight_ne[0];
-                output_ne_offset +=
-                    output_elem_size * output_ne[0] * output_ne[1];
-                output_ne[0] = weight_ne[0];
-
-                acl_weight_tensor = ggml_cann_create_tensor(
-                    (char*)src0->data + batch0 * weight_stride,
-                    ggml_cann_type_mapping(type), weight_elem_size, weight_ne,
-                    weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);
-                acl_scale_tensor = ggml_cann_create_tensor(
-                    scale_offset + batch0 * scale_stride, ACL_FLOAT16,
-                    scale_elem_size, scale_ne, scale_nb, 2, ACL_FORMAT_ND,
-                    scale_ne_offset);
-                acl_output_tensor = ggml_cann_create_tensor(
-                    (char*)output_buffer + batch1 * output_stride, ACL_FLOAT16,
-                    output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND,
-                    output_ne_offset);
-
-                ACL_CHECK(aclnnWeightQuantBatchMatmulV2GetWorkspaceSize(
-                    acl_input_tensor, acl_weight_tensor, acl_scale_tensor,
-                    nullptr, nullptr, nullptr, nullptr, QK8_0,
-                    acl_output_tensor, &workspaceSize, &executor));
-                ACL_CHECK(aclnnWeightQuantBatchMatmulV2(
-                    workspaceAddr, workspaceSize, executor, ctx.stream()));
-
-                ACL_CHECK(aclDestroyTensor(acl_weight_tensor));
-                ACL_CHECK(aclDestroyTensor(acl_scale_tensor));
-                ACL_CHECK(aclDestroyTensor(acl_output_tensor));
-            }
-
-            ACL_CHECK(aclDestroyTensor(acl_input_tensor));
         }
     }
 
     // cast out
-    if (dst->type != GGML_TYPE_F16) {
-        int64_t* output_cast_ne = dst->ne;
-        size_t output_cast_nb[GGML_MAX_DIMS];
-        output_cast_nb[0] = sizeof(uint16_t);
-        for (int i = 1; i < GGML_MAX_DIMS; i++) {
-            output_cast_nb[i] = output_cast_nb[i - 1] * output_cast_ne[i - 1];
-        }
-
-        aclTensor* acl_output_tensor = ggml_cann_create_tensor(
-            output_buffer, ACL_FLOAT16, output_elem_size, output_cast_ne,
-            output_cast_nb, GGML_MAX_DIMS);
-        aclTensor* acl_dst_tensor = ggml_cann_create_tensor(dst);
-        aclnn_cast(ctx, acl_output_tensor, acl_dst_tensor,
-                   ggml_cann_type_mapping(dst->type));
-
-        ACL_CHECK(aclDestroyTensor(acl_output_tensor));
-        ACL_CHECK(aclDestroyTensor(acl_dst_tensor));
+    int64_t* output_cast_ne = dst->ne;
+    size_t output_cast_nb[GGML_MAX_DIMS];
+    output_cast_nb[0] = sizeof(uint16_t);
+    for (int i = 1; i < GGML_MAX_DIMS; i++) {
+        output_cast_nb[i] = output_cast_nb[i - 1] * output_cast_ne[i - 1];
     }
+
+    aclTensor* acl_output_tensor =
+        ggml_cann_create_tensor(output_buffer, ACL_FLOAT16, output_elem_size,
+                                output_cast_ne, output_cast_nb, GGML_MAX_DIMS);
+    aclTensor* acl_dst_tensor = ggml_cann_create_tensor(dst);
+    aclnn_cast(ctx, acl_output_tensor, acl_dst_tensor, ACL_FLOAT);
+
+    ACL_CHECK(aclDestroyTensor(acl_output_tensor));
+    ACL_CHECK(aclDestroyTensor(acl_dst_tensor));
 }
 
 void ggml_cann_mul_mat(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
@@ -2964,14 +2714,12 @@ static void aclnn_index_fill_tensor(ggml_backend_cann_context& ctx,
 static void aclnn_cache_init(ggml_backend_cann_context& ctx, ggml_tensor* dst,
                              aclTensor* acl_cos_repeat_tensor,
                              aclTensor* acl_sin_repeat_tensor,
-                             float theta_scale, float freq_scale,
-                             float attn_factor, bool is_neox) {
+                             float theta_scale, bool is_neox) {
     // int sin/cos cache, cache has different repeat method depond on
     // @param.is_neox
 
     ggml_tensor* src0 = dst->src[0];  // input
     ggml_tensor* src1 = dst->src[1];  // position
-    ggml_tensor* src2 = dst->src[2];  // freq_factors
 
     // arange, [0,1,...,ne0/2]
     int64_t arange_length = src0->ne[0] / 2;
@@ -3000,26 +2748,11 @@ static void aclnn_cache_init(ggml_backend_cann_context& ctx, ggml_tensor* dst,
     ggml_cann_pool_alloc theta_scale_allocator(ctx.pool(),
                                                arange_length * sizeof(float_t));
     void* theta_scale_buffer = theta_scale_allocator.get();
-    aclTensor* acl_theta_scale_tensor = aclnn_values(
+    aclTensor* acl_theta_scale_tensor = aclnn_ones(
         ctx, theta_scale_buffer, arange_length * sizeof(float_t), arange_ne,
         GGML_MAX_DIMS, ACL_FLOAT, sizeof(float_t), theta_scale);
     aclnn_pow_tensor_tensor(ctx, acl_theta_scale_tensor, acl_arange_tensor);
 
-    // freq_scale
-    if (freq_scale != 1) {
-        aclnn_muls(ctx, acl_theta_scale_tensor, freq_scale, nullptr, true);
-    }
-
-    // freq_factors
-    if (src2) {
-        aclTensor* acl_freq_factors_tensor = ggml_cann_create_tensor(
-            src2->data, ggml_cann_type_mapping(src2->type),
-            ggml_type_size(src2->type), arange_ne, arange_nb, GGML_MAX_DIMS);
-        aclnn_div_tensor(ctx, acl_theta_scale_tensor, acl_freq_factors_tensor,
-                         nullptr, true);
-        ACL_CHECK(aclDestroyTensor(acl_freq_factors_tensor));
-    }
-
     // position
     GGML_ASSERT(src1->type == GGML_TYPE_I32);
     int64_t position_length = src1->ne[0];
@@ -3083,12 +2816,6 @@ static void aclnn_cache_init(ggml_backend_cann_context& ctx, ggml_tensor* dst,
         GGML_MAX_DIMS, ACL_FORMAT_ND);
     aclnn_cos(ctx, acl_permute_tensor, acl_cos_tensor);
 
-    // attn_factor
-    if (attn_factor != 1) {
-        aclnn_muls(ctx, acl_sin_tensor, attn_factor, nullptr, true);
-        aclnn_muls(ctx, acl_cos_tensor, attn_factor, nullptr, true);
-    }
-
     // repeat
     if (is_neox) {
         int64_t repeatsArray[] = {1, 1, 1, 2};
@@ -3114,27 +2841,15 @@ static void aclnn_cache_init(ggml_backend_cann_context& ctx, ggml_tensor* dst,
     ACL_CHECK(aclDestroyTensor(acl_cos_tensor));
 }
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-aclnnStatus aclnnRotaryPositionEmbeddingGetWorkspaceSize(
-    const aclTensor* x, const aclTensor* cos, const aclTensor* sin,
-    int64_t mode, const aclTensor* yOut, uint64_t* workspaceSize,
-    aclOpExecutor** executor);
-aclnnStatus aclnnRotaryPositionEmbedding(void* workspace,
-                                         uint64_t workspaceSize,
-                                         aclOpExecutor* executor,
-                                         aclrtStream stream);
-#ifdef __cplusplus
-}
-#endif
-
 void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
     // TODO: use ascendc
     // Only test with LLAMA model.
     ggml_tensor* src0 = dst->src[0];  // input
     ggml_tensor* src2 = dst->src[2];  // freq_factors
 
+    // TODO: with freq_factors
+    GGML_ASSERT(src2 == NULL);
+
     // param
     float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
     // const int n_past     = ((int32_t *) dst->op_params)[0];
@@ -3152,11 +2867,13 @@ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
     memcpy(&beta_fast, (int32_t*)dst->op_params + 9, sizeof(float));
     memcpy(&beta_slow, (int32_t*)dst->op_params + 10, sizeof(float));
 
-    // TODO: n_dims <= ne0
-    GGML_ASSERT(n_dims == ne0);
+    GGML_ASSERT(n_dims <= ne0);
     GGML_ASSERT(n_dims % 2 == 0);
+
     // TODO: ext_factor != 0
     GGML_ASSERT(ext_factor == 0);
+    // TODO: freq_scale != 1
+    GGML_ASSERT(freq_scale == 1);
 
     const float theta_scale = powf(freq_base, -2.0f / n_dims);
 
@@ -3187,13 +2904,7 @@ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
         ggml_cann_create_tensor(cos_buffer, ACL_FLOAT, sizeof(float_t),
                                 sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);
     aclnn_cache_init(ctx, dst, acl_cos_reshape_tensor, acl_sin_reshape_tensor,
-                     theta_scale, freq_scale, attn_factor, is_neox);
-
-    aclTensor* acl_src = ggml_cann_create_tensor(src0);
-    aclTensor* acl_dst = ggml_cann_create_tensor(dst);
-
-#ifdef ASCEND_310P
-    // Special ROPE operation for 310P
+                     theta_scale, is_neox);
 
     // roll input
     void* input_roll_buffer;
@@ -3236,7 +2947,7 @@ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
         for (int i = 1; i < GGML_MAX_DIMS; i++) {
             minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];
         }
-        acl_minus_one_tensor = aclnn_values(
+        acl_minus_one_tensor = aclnn_ones(
             ctx, minus_one_scale_buffer, sizeof(float_t) * src0->ne[0],
             minus_one_ne, GGML_MAX_DIMS, ACL_FLOAT, sizeof(float_t), 1);
         int64_t dim = 3;
@@ -3263,15 +2974,17 @@ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
 
         ACL_CHECK(aclDestroyTensor(acl_input_roll_tensor));
         ACL_CHECK(aclDestroyTensor(acl_input_tensor));
+
         // init [-1, -1, -1, 1, 1，1，...]
         minus_one_scale_buffer = minus_one_scale_allocator.get();
+
         int64_t minus_one_ne[4] = {src0->ne[0], 1, 1, 1};
         size_t minus_one_nb[GGML_MAX_DIMS];
         minus_one_nb[0] = sizeof(float_t);
         for (int i = 1; i < GGML_MAX_DIMS; i++) {
             minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];
         }
-        acl_minus_one_tensor = aclnn_values(
+        acl_minus_one_tensor = aclnn_ones(
             ctx, minus_one_scale_buffer, sizeof(float_t) * src0->ne[0],
             minus_one_ne, GGML_MAX_DIMS, ACL_FLOAT, sizeof(float_t), 1);
         // -1 * first half
@@ -3313,12 +3026,14 @@ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
               acl_input_roll_mul_scale_tensor);
 
     // output
+    aclTensor* acl_src0 = ggml_cann_create_tensor(src0);
+    aclTensor* acl_dst = ggml_cann_create_tensor(dst);
     void* output_fp32_buffer;
     if (src0->type == GGML_TYPE_F32) {
-        aclnn_inplace_mul(ctx, acl_src, acl_cos_reshape_tensor);
+        aclnn_inplace_mul(ctx, acl_src0, acl_cos_reshape_tensor);
         aclnn_inplace_mul(ctx, acl_input_roll_mul_scale_tensor,
                           acl_sin_reshape_tensor);
-        aclnn_add(ctx, acl_src, acl_input_roll_mul_scale_tensor, acl_dst);
+        aclnn_add(ctx, acl_src0, acl_input_roll_mul_scale_tensor, acl_dst);
         // TODO: ne0 != n_dims in mode2
     } else if (src0->type == GGML_TYPE_F16) {
         size_t input_fp32_nb[GGML_MAX_DIMS];
@@ -3345,7 +3060,7 @@ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
         aclTensor* output_fp32_tensor = ggml_cann_create_tensor(
             output_fp32_buffer, ACL_FLOAT, sizeof(float_t), dst->ne,
             input_fp32_nb, GGML_MAX_DIMS);
-        aclnn_mul(ctx, acl_src, acl_cos_reshape_tensor, input_fp32_tensor1);
+        aclnn_mul(ctx, acl_src0, acl_cos_reshape_tensor, input_fp32_tensor1);
         aclnn_mul(ctx, acl_input_roll_mul_scale_tensor, acl_sin_reshape_tensor,
                   input_fp32_tensor2);
         aclnn_add(ctx, input_fp32_tensor1, input_fp32_tensor2,
@@ -3355,73 +3070,13 @@ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
         ACL_CHECK(aclDestroyTensor(input_fp32_tensor1));
         ACL_CHECK(aclDestroyTensor(input_fp32_tensor2));
         ACL_CHECK(aclDestroyTensor(output_fp32_tensor));
-        ACL_CHECK(aclDestroyTensor(acl_sin_reshape_tensor));
-        ACL_CHECK(aclDestroyTensor(acl_minus_one_tensor));
-        ACL_CHECK(aclDestroyTensor(acl_input_roll_mul_scale_tensor));
-        ACL_CHECK(aclDestroyTensor(acl_input_roll_reshape_tensor));
-        ACL_CHECK(aclDestroyTensor(acl_src));
-    }
-    return;
-#endif
-
-    // src0 == GGML_TYPE_F16
-    // TODO: optimization this `if` code
-    if (src0->type == GGML_TYPE_F16) {
-        ggml_cann_pool_alloc sin_final_allocator(
-            ctx.pool(), src0->ne[0] * src0->ne[2] * ggml_type_size(src0->type));
-        ggml_cann_pool_alloc cos_final_allocator(
-            ctx.pool(), src0->ne[0] * src0->ne[2] * ggml_type_size(src0->type));
-        void* sin_final_buffer = sin_final_allocator.get();
-        void* cos_final_buffer = cos_final_allocator.get();
-
-        int64_t sin_final_ne[4] = {src0->ne[0], 1, src0->ne[2], 1};
-        size_t sin_final_nb[GGML_MAX_DIMS];
-        sin_final_nb[0] = ggml_type_size(src0->type);
-        for (int i = 1; i < GGML_MAX_DIMS; i++) {
-            sin_final_nb[i] = sin_final_nb[i - 1] * sin_final_ne[i - 1];
-        }
-        aclTensor* acl_sin_final_tensor = ggml_cann_create_tensor(
-            sin_final_buffer, ggml_cann_type_mapping(src0->type),
-            ggml_type_size(src0->type), sin_final_ne, sin_final_nb,
-            GGML_MAX_DIMS);
-        aclTensor* acl_cos_final_tensor = ggml_cann_create_tensor(
-            cos_final_buffer, ggml_cann_type_mapping(src0->type),
-            ggml_type_size(src0->type), sin_final_ne, sin_final_nb,
-            GGML_MAX_DIMS);
-
-        aclnn_cast(ctx, acl_sin_reshape_tensor, acl_sin_final_tensor,
-                   ggml_cann_type_mapping(src0->type));
-        aclnn_cast(ctx, acl_cos_reshape_tensor, acl_cos_final_tensor,
-                   ggml_cann_type_mapping(src0->type));
-        ACL_CHECK(aclDestroyTensor(acl_cos_reshape_tensor));
-        ACL_CHECK(aclDestroyTensor(acl_sin_reshape_tensor));
-        acl_sin_reshape_tensor = acl_sin_final_tensor;
-        acl_cos_reshape_tensor = acl_cos_final_tensor;
     }
 
-    uint64_t workspaceSize = 0;
-    aclOpExecutor* executor;
-
-    void* workspaceAddr = nullptr;
-
-    int acl_mode = mode;
-    if (mode == 0) {
-        acl_mode = 1;
-    }
-
-    ACL_CHECK(aclnnRotaryPositionEmbeddingGetWorkspaceSize(
-        acl_src, acl_cos_reshape_tensor, acl_sin_reshape_tensor, acl_mode,
-        acl_dst, &workspaceSize, &executor));
-    if (workspaceSize > 0) {
-        ggml_cann_pool_alloc workspace_allocator(ctx.pool(), workspaceSize);
-        workspaceAddr = workspace_allocator.get();
-    }
-
-    ACL_CHECK(aclnnRotaryPositionEmbedding(workspaceAddr, workspaceSize,
-                                           executor, ctx.stream()));
-
-    ACL_CHECK(aclDestroyTensor(acl_src));
-    ACL_CHECK(aclDestroyTensor(acl_cos_reshape_tensor));
     ACL_CHECK(aclDestroyTensor(acl_sin_reshape_tensor));
+    ACL_CHECK(aclDestroyTensor(acl_cos_reshape_tensor));
+    ACL_CHECK(aclDestroyTensor(acl_minus_one_tensor));
+    ACL_CHECK(aclDestroyTensor(acl_input_roll_mul_scale_tensor));
+    ACL_CHECK(aclDestroyTensor(acl_input_roll_reshape_tensor));
+    ACL_CHECK(aclDestroyTensor(acl_src0));
     ACL_CHECK(aclDestroyTensor(acl_dst));
 }

ggml/src/ggml-cann/common.h

@@ -211,20 +211,17 @@ struct ggml_cann_pool_alloc {
 struct ggml_backend_cann_context {
     int32_t device;                  /**< Device ID. */
     std::string name;                /**< Name of the device. */
-    std::string description;         /**< Description of the device. */
     aclrtEvent copy_event = nullptr; /**< Event for managing copy operations. */
 
-    aclrtStream streams[GGML_CANN_MAX_STREAMS] = {nullptr}; /**< Array of streams for the device. */
+    aclrtStream streams[GGML_CANN_MAX_STREAMS] = {
+        {nullptr}}; /**< Array of streams for the device. */
 
     /**
      * @brief Constructor for initializing the context with a given device.
      * @param device Device ID.
      */
     explicit ggml_backend_cann_context(int device)
-        : device(device), name("CANN" + std::to_string(device)) {
-        ggml_cann_set_device(device);
-        description = aclrtGetSocName();
-    }
+        : device(device), name("CANN" + std::to_string(device)) {}
 
     /**
      * @brief Destructor for cleaning up resources.

ggml/src/ggml-cann/ggml-cann.cpp

@@ -122,10 +122,6 @@ static ggml_cann_device_info ggml_cann_init() {
         ACL_CHECK(aclrtMemGetAllocationGranularity(
             &prop, ACL_RT_MEM_ALLOC_GRANULARITY_RECOMMENDED,
             &info.devices[id].vmm_granularity));
-
-        size_t free, total;
-        ggml_backend_cann_get_device_memory(id, &free, &total);
-        info.devices[id].total_vram = free;
     }
 
     // TODO: add more device info later.
@@ -212,11 +208,6 @@ struct ggml_cann_pool_leg : public ggml_cann_pool {
      * @return A pointer to the allocated buffer.
      */
     void* alloc(size_t size, size_t* actual_size) override {
-        const size_t alignment = 128;
-        size = GGML_PAD(size, alignment);
-        if (size == 0) {
-            size = alignment;
-        }
 #ifdef DEBUG_CANN_MALLOC
         int nnz = 0;
         size_t max_size = 0;
@@ -255,11 +246,13 @@ struct ggml_cann_pool_leg : public ggml_cann_pool {
             return ptr;
         }
         void* ptr;
+        size_t look_ahead_size = (size_t)(1.05 * size);
+        look_ahead_size = 256 * ((look_ahead_size + 255) / 256);
         ggml_cann_set_device(device);
         ACL_CHECK(
-            aclrtMalloc(&ptr, size, ACL_MEM_MALLOC_HUGE_FIRST));
-        *actual_size = size;
-        pool_size += size;
+            aclrtMalloc(&ptr, look_ahead_size, ACL_MEM_MALLOC_HUGE_FIRST));
+        *actual_size = look_ahead_size;
+        pool_size += look_ahead_size;
 #ifdef DEBUG_CANN_MALLOC
         GGML_LOG_INFO(
             "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, "
@@ -303,7 +296,7 @@ struct ggml_cann_pool_vmm : public ggml_cann_pool {
     /**
      * @brief The maximum size of the virtual memory pool (32 GB).
      */
-    size_t max_size;
+    static const size_t CANN_POOL_VMM_MAX_SIZE = 1ull << 35;  // 32 GB
 
     /**
      * @brief The device ID associated with this buffer pool.
@@ -348,11 +341,7 @@ struct ggml_cann_pool_vmm : public ggml_cann_pool {
      */
     explicit ggml_cann_pool_vmm(int device)
         : device(device),
-          granularity(ggml_cann_info().devices[device].vmm_granularity) {
-        auto dev = ggml_cann_info().devices[device];
-        granularity = dev.vmm_granularity;
-        max_size = dev.total_vram;
-    }
+          granularity(ggml_cann_info().devices[device].vmm_granularity) {}
 
     /**
      * @brief Destructor to free all buffers in the virtual memory pool.
@@ -381,19 +370,17 @@ struct ggml_cann_pool_vmm : public ggml_cann_pool {
         // round up the allocation size to the alignment to ensure that all
         // allocations are aligned for all data types
         const size_t alignment = 128;
-        size = GGML_PAD(size, alignment);
-        if (size == 0) {
-            size = alignment;
-        }
+        size = alignment * ((size + alignment - 1) / alignment);
 
         size_t avail = pool_size - pool_used;
 
         if (size > avail) {
             // round up to the next multiple of the granularity
             size_t reserve_size = size - avail;
-            reserve_size = GGML_PAD(reserve_size, granularity);
+            reserve_size =
+                granularity * ((reserve_size + granularity - 1) / granularity);
 
-            GGML_ASSERT(pool_size + reserve_size <= max_size);
+            GGML_ASSERT(pool_size + reserve_size <= CANN_POOL_VMM_MAX_SIZE);
 
             // allocate more physical memory
             aclrtPhysicalMemProp prop = {};
@@ -409,7 +396,7 @@ struct ggml_cann_pool_vmm : public ggml_cann_pool {
             // reserve virtual address space (if not already reserved)
             if (pool_addr == 0) {
                 ACL_CHECK(aclrtReserveMemAddress(
-                    &pool_addr, max_size, 0, NULL, 1));
+                    &pool_addr, CANN_POOL_VMM_MAX_SIZE, 0, NULL, 1));
             }
 
             // map at the end of the pool
@@ -422,11 +409,10 @@ struct ggml_cann_pool_vmm : public ggml_cann_pool {
             // add to the pool
             pool_size += reserve_size;
 
-#ifdef DEBUG_CANN_MALLOC
-             GGML_LOG_INFO("cann pool[%d]: size increased to %llu MB (reserved %llu MB)\n",
-                   device, (unsigned long long) (pool_size/1024/1024),
-                   (unsigned long long) (reserve_size/1024/1024));
-#endif
+            // GGML_LOG_INFO("cann pool[%d]: size increased to %llu MB (
+            // reserved %llu MB)\n",
+            //       device, (unsigned long long) (pool_size/1024/1024),
+            //       (unsigned long long) (reserve_size/1024/1024));
         }
 
         GGML_ASSERT(pool_addr != 0);
@@ -471,6 +457,7 @@ struct ggml_cann_pool_vmm : public ggml_cann_pool {
  */
 std::unique_ptr<ggml_cann_pool> ggml_backend_cann_context::new_pool_for_device(
     int device) {
+    // return std::unique_ptr<ggml_cann_pool>(new ggml_cann_pool_leg(device));
     return std::unique_ptr<ggml_cann_pool>(new ggml_cann_pool_vmm(device));
 }
 
@@ -1143,10 +1130,10 @@ ggml_backend_cann_buffer_type(int32_t device) {
     static bool ggml_backend_cann_buffer_type_initialized = false;
 
     if (!ggml_backend_cann_buffer_type_initialized) {
-        for (int32_t i = 0; i < ggml_cann_info().device_count; i++) {
+        for (int32_t i = 0; i < GGML_CANN_MAX_DEVICES; i++) {
             ggml_backend_cann_buffer_types[i] = {
                 /* .iface    = */ ggml_backend_cann_buffer_type_interface,
-                /* .device    = */ ggml_backend_reg_dev_get(ggml_backend_cann_reg(), i),
+                /* .device    = */ ggml_backend_reg_dev_get(ggml_backend_cann_reg(), device),
                 /* .context  = */
                  new ggml_backend_cann_buffer_type_context{
                     i, "CANN" + std::to_string(i)},
@@ -1212,15 +1199,10 @@ static void * ggml_cann_host_malloc(size_t size) {
         return nullptr;
     }
 
-    const size_t alignment = 128;
-    size = GGML_PAD(size, alignment);
-    if (size == 0) {
-        size = alignment;
-    }
-
     void * hostPtr = nullptr;
     aclError err = aclrtMallocHost((void **) &hostPtr, size);
     if (err != ACL_SUCCESS) {
+
         GGML_LOG_WARN("%s: failed to allocate %.2f MiB of pinned memory: %s\n", __func__,
                            size / 1024.0 / 1024.0, aclGetRecentErrMsg());
         return nullptr;
@@ -1687,14 +1669,12 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev,
             }
         case GGML_OP_MUL_MAT: {
             switch (op->src[0]->type) {
-                case GGML_TYPE_Q8_0:
-                    // Current groupsize should not be greater than k-1 in
-                    // aclnnWeightQuantBatchMatmulV2GetWorkspaceSize
-                    if (op->src[0]->ne[0] <= QK8_0) {
-                        return false;
-                    }
                 case GGML_TYPE_F16:
                 case GGML_TYPE_F32:
+                case GGML_TYPE_Q8_0:
+                    // TODO: fix me
+                    // Current groupsize should not be greater than k-1 in
+                    // aclnnWeightQuantBatchMatmulV2GetWorkspaceSize().
                 case GGML_TYPE_Q4_0:
                     return true;
                 default:
@@ -1726,50 +1706,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev,
                     return false;
             }
         }
-        case GGML_OP_CONT: {
-            // TODO: support GGML_TYPE_BF16
-            switch (op->src[0]->type) {
-                case GGML_TYPE_F32:
-                case GGML_TYPE_F16:
-                    return true;
-                default:
-                    return false;
-            }
-        }
-        case GGML_OP_ROPE: {
-            // TODO: with ops-test v == 1
-            float * ext_factor = (float*)((int32_t*)op->op_params + 7);
-            // TODO: n_dims <= ne0
-            if (op->src[0]->ne[0] != op->op_params[1]) {
-                return false;
-            }
-            // TODO: ext_factor != 0
-            if (*ext_factor != 0) {
-                return false;
-            }
-
-            const int mode = ((const int32_t *) op->op_params)[2];
-            if (mode & GGML_ROPE_TYPE_MROPE) {
-                return false;
-            }
-            if (mode & GGML_ROPE_TYPE_VISION) {
-                return false;
-            }
-
-            return true;
-        }
-        case GGML_OP_UPSCALE: {
-            // aclnnUpsampleNearest2dGetWorkspaceSize not support
-            // selfDimN[2]/outDimN[2] or selfDimC[3]/outDimC[3] not equal
-            if (op->src[0]->ne[2] * op->ne[3] != op->src[0]->ne[3] * op->ne[2]) {
-                return false;
-            }
-            return true;
-        }
-        case GGML_OP_IM2COL:
-        case GGML_OP_CONCAT:
         case GGML_OP_DUP:
         case GGML_OP_REPEAT:
+        case GGML_OP_CONCAT:
         case GGML_OP_NONE:
         case GGML_OP_RESHAPE:
         case GGML_OP_VIEW:
@@ -1783,13 +1722,17 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev,
         case GGML_OP_SCALE:
         case GGML_OP_SQR:
         case GGML_OP_CLAMP:
+        case GGML_OP_CONT:
         case GGML_OP_DIAG_MASK_INF:
         case GGML_OP_SOFT_MAX:
+        case GGML_OP_ROPE:
+        case GGML_OP_IM2COL:
         case GGML_OP_POOL_2D:
         case GGML_OP_SUM_ROWS:
         case GGML_OP_ARGSORT:
         case GGML_OP_ACC:
         case GGML_OP_GROUP_NORM:
+        case GGML_OP_UPSCALE:
         case GGML_OP_PAD:
         case GGML_OP_ARANGE:
         case GGML_OP_TIMESTEP_EMBEDDING:
@@ -2098,7 +2041,7 @@ static void * ggml_backend_cann_reg_get_proc_address(ggml_backend_reg_t reg, con
 static const ggml_backend_reg_i ggml_backend_cann_reg_interface = {
     /* .get_name          = */ ggml_backend_cann_reg_get_name,
     /* .get_device_count  = */ ggml_backend_cann_reg_get_device_count,
-    /* .get_device        = */ ggml_backend_cann_reg_get_device,
+    /* .get_device_get    = */ ggml_backend_cann_reg_get_device,
     /* .get_proc_address  = */ ggml_backend_cann_reg_get_proc_address,
 };
 
@@ -2121,17 +2064,16 @@ ggml_backend_reg_t ggml_backend_cann_reg() {
                 dev_ctx->name = GGML_CANN_NAME + std::to_string(i);
                 ggml_cann_set_device(i);
                 ggml_backend_dev_t dev = new ggml_backend_device {
-                    /* .iface   = */ ggml_backend_cann_device_interface,
-                    /* .reg     = */ &reg,
-                    /* .context = */ dev_ctx
+                    /* .interface = */ ggml_backend_cann_device_interface,
+                    /* .reg       = */ &reg,
+                    /* .context   = */ dev_ctx
                 };
                 ctx->devices.push_back(dev);
             }
 
             reg = ggml_backend_reg {
-                /* .api_version = */ GGML_BACKEND_API_VERSION,
-                /* .iface       = */ ggml_backend_cann_reg_interface,
-                /* .context     = */ ctx
+                /* .interface = */ ggml_backend_cann_reg_interface,
+                /* .context   = */ ctx
             };
         }
 
@@ -2184,5 +2126,3 @@ void ggml_backend_cann_get_device_memory(int32_t device, size_t* free,
     ggml_cann_set_device(device);
     ACL_CHECK(aclrtGetMemInfo(ACL_HBM_MEM, free, total));
 }
-
-GGML_BACKEND_DL_IMPL(ggml_backend_cann_reg)

ggml/src/ggml-cann/kernels/CMakeLists.txt

@@ -1,3 +1,7 @@
+if (NOT SOC_TYPE)
+    set (SOC_TYPE "Ascend910B3")
+endif()
+
 file(GLOB SRC_FILES
     get_row_f32.cpp
     get_row_f16.cpp
@@ -9,6 +13,7 @@ file(GLOB SRC_FILES
     dup.cpp
 )
 
+string(TOLOWER ${SOC_TYPE} SOC_VERSION)
 set(ASCEND_CANN_PACKAGE_PATH ${CANN_INSTALL_DIR})
 set(RUN_MODE "npu" CACHE STRING "run mode: npu/sim")
 
@@ -25,6 +30,4 @@ ascendc_library(ascendc_kernels STATIC
     ${SRC_FILES}
 )
 
-message(STATUS "CANN: compile ascend kernels witch SOC_TYPE:${SOC_TYPE}, SOC_VERSION:${SOC_VERSION}, compile macro:-D${SOC_TYPE_COMPILE_OPTION}.")
-ascendc_compile_definitions(ascendc_kernels PRIVATE "-D${SOC_TYPE_COMPILE_OPTION}")
 # ascendc_compile_definitions(ascendc_kernels PRIVATE -DASCENDC_DUMP)

ggml/src/ggml-cann/kernels/dup.cpp

@@ -5,7 +5,6 @@
 using namespace AscendC;
 
 #define BUFFER_NUM 2
-const int64_t SUPPORTED_MAX_DIM = 65535;  // currently the limit of max block dim supportted by dup kernel is 65535template <typename SRC_T, typename DST_T>
 
 template <typename SRC_T, typename DST_T>
 class DupByRows {
@@ -52,36 +51,24 @@ class DupByRows {
 
     __aicore__ inline void copy_in() {
         LocalTensor<SRC_T> src_local = src_queue.AllocTensor<SRC_T>();
-        const size_t elem_per_block = 32 / sizeof(SRC_T);
-        size_t tail = num_elem % elem_per_block;
-        size_t cpy_elements_len = tail > 0 ? num_elem + 1 : num_elem;
-        DataCopy(src_local, src_gm, cpy_elements_len);
+
+        DataCopyExtParams dataCopyParams;
+        dataCopyParams.blockCount = 1;
+        dataCopyParams.blockLen = num_elem * sizeof(SRC_T);
+        DataCopyPadExtParams<SRC_T> padParams;
+        DataCopyPad(src_local, src_gm, dataCopyParams, padParams);
+
         src_queue.EnQue(src_local);
     }
 
     __aicore__ inline void copy_out() {
         LocalTensor<DST_T> dst_local = dst_queue.DeQue<DST_T>();
-#ifdef ASCEND_310P
-        const size_t elem_per_block = 32 / sizeof(DST_T);
-        size_t tail = num_elem % elem_per_block;
-        size_t len = num_elem & ~(elem_per_block - 1);
-        if (len > 0) {
-            DataCopy(dst_gm, dst_local, len);
-        }
-        if(tail != 0) {
-            for (size_t i = tail; i < elem_per_block; i++) {
-                dst_local[len + i].SetValue(0, 0);
-            }
-            SetAtomicAdd<float>();
-            DataCopy(dst_gm[len], dst_local[len], elem_per_block);
-            SetAtomicNone();
-        }
-#else
+
         DataCopyExtParams dataCopyParams;
         dataCopyParams.blockCount = 1;
         dataCopyParams.blockLen = num_elem * sizeof(DST_T);
         DataCopyPad(dst_gm, dst_local, dataCopyParams);
-#endif
+
         dst_queue.FreeTensor(dst_local);
     }
 

ggml/src/ggml-cann/kernels/get_row_f16.cpp

@@ -14,7 +14,7 @@ class GET_ROW_F16 {
                                 int64_t *output_ne_ub, size_t *output_nb_ub) {
         // TODO, use template for F16/f32
         int64_t op_block_num = GetBlockNum();
-        op_block_idx = GetBlockIdx();
+        int64_t op_block_idx = GetBlockIdx();
 
         for (int i = 0; i < 4; i++) {
             input_ne[i] = input_ne_ub[i];
@@ -59,42 +59,32 @@ class GET_ROW_F16 {
     }
 
     __aicore__ inline void copy_in(uint32_t offset, size_t len) {
-        size_t origin_len = len;
         LocalTensor<half> input_local = input_queue.AllocTensor<half>();
-        const size_t elem_per_block = 32 / sizeof(half);
-        size_t tail = len % elem_per_block;
-        len = len & ~(elem_per_block - 1);
-        if(tail != 0) {
-            len += elem_per_block;
-        }
+        size_t tail = len % 32;
+        len = len & ~31;
         DataCopy(input_local, input_gm[offset], len);
+        if(tail != 0) {
+            DataCopyExtParams dataCopyParams;
+            dataCopyParams.blockCount = 1;
+            dataCopyParams.blockLen = tail * sizeof(half);
+            DataCopyPadExtParams<half> padParams;
+            DataCopyPad(input_local[len], input_gm[offset + len],
+                        dataCopyParams, padParams);
+        }
         input_queue.EnQue(input_local);
     }
 
     __aicore__ inline void copy_out(uint32_t offset, size_t len) {
         LocalTensor<float> output_local = output_queue.DeQue<float>();
-        const size_t elem_per_block = 32 / sizeof(float);
-        size_t tail = len % elem_per_block;
-        len = len & ~(elem_per_block - 1);
-        if (len > 0) {
-            DataCopy(output_gm[offset], output_local, len);
-        }
-
+        size_t tail = len % 32;
+        len = len & ~31;
+        DataCopy(output_gm[offset], output_local, len);
         if(tail != 0) {
-#ifdef ASCEND_310P
-            for (size_t i = tail; i < elem_per_block; i++) {
-                output_local[len + i].SetValue(0, 0);
-            }
-            SetAtomicAdd<float>();
-            DataCopy(output_gm[offset + len], output_local[len], elem_per_block);
-            SetAtomicNone();
-#else
             DataCopyExtParams dataCopyParams;
             dataCopyParams.blockCount = 1;
             dataCopyParams.blockLen = tail * sizeof(float);
             DataCopyPad(output_gm[offset + len], output_local[len],
                         dataCopyParams);
-#endif
         }
         output_queue.FreeTensor(output_local);
     }
@@ -160,7 +150,6 @@ class GET_ROW_F16 {
     GlobalTensor<float> output_gm;
     TQue<QuePosition::VECIN, BUFFER_NUM> input_queue;
     TQue<QuePosition::VECOUT, BUFFER_NUM> output_queue;
-    int64_t op_block_idx;
 };
 
 template <typename T>

ggml/src/ggml-cann/kernels/get_row_f32.cpp

@@ -13,7 +13,7 @@ class GET_ROW_F32 {
                                 int64_t *indices_ne_ub, size_t *indices_nb_ub,
                                 int64_t *output_ne_ub, size_t *output_nb_ub) {
         int64_t op_block_num = GetBlockNum();
-        op_block_idx = GetBlockIdx();
+        int64_t op_block_idx = GetBlockIdx();
 
         for (int i = 0; i < 4; i++) {
             input_ne[i] = input_ne_ub[i];
@@ -55,40 +55,31 @@ class GET_ROW_F32 {
 
     __aicore__ inline void copy_in(uint32_t offset, size_t len) {
         LocalTensor<float> input_local = input_queue.AllocTensor<float>();
-        const size_t elem_per_block = 32 / sizeof(float);
-        size_t tail = len % elem_per_block;
-        len = len & ~(elem_per_block - 1);
-        if(tail != 0) {
-            len += elem_per_block;
-        }
+        size_t tail = len % 32;
+        len = len & ~31;
         DataCopy(input_local, input_gm[offset], len);
+        if(tail != 0) {
+            DataCopyExtParams dataCopyParams;
+            dataCopyParams.blockCount = 1;
+            dataCopyParams.blockLen = tail * sizeof(float);
+            DataCopyPadExtParams<float> padParams;
+            DataCopyPad(input_local[len], input_gm[offset + len],
+                        dataCopyParams, padParams);
+        }
         input_queue.EnQue(input_local);
     }
 
     __aicore__ inline void copy_out(uint32_t offset, size_t len) {
         LocalTensor<float> output_local = output_queue.DeQue<float>();
-        const size_t elem_per_block = 32 / sizeof(float);
-        size_t tail = len % elem_per_block;
-        len = len & ~(elem_per_block - 1);
-        if (len > 0) {
-            DataCopy(output_gm[offset], output_local, len);
-        }
-
+        size_t tail = len % 32;
+        len = len & ~31;
+        DataCopy(output_gm[offset], output_local, len);
         if(tail != 0) {
-#ifdef ASCEND_310P
-            for (size_t i = tail; i < elem_per_block; i++) {
-                output_local[len + i].SetValue(0, 0);
-            }
-            SetAtomicAdd<float>();
-            DataCopy(output_gm[offset + len], output_local[len], elem_per_block);
-            SetAtomicNone();
-#else
             DataCopyExtParams dataCopyParams;
             dataCopyParams.blockCount = 1;
             dataCopyParams.blockLen = tail * sizeof(float);
             DataCopyPad(output_gm[offset + len], output_local[len],
                         dataCopyParams);
-#endif
         }
         output_queue.FreeTensor(output_local);
     }
@@ -153,7 +144,6 @@ class GET_ROW_F32 {
     GlobalTensor<float> output_gm;
     TQue<QuePosition::VECIN, BUFFER_NUM> input_queue;
     TQue<QuePosition::VECOUT, BUFFER_NUM> output_queue;
-    int64_t op_block_idx;
 };
 
 template <typename T>

ggml/src/ggml-cann/kernels/get_row_q4_0.cpp

@@ -2,15 +2,6 @@
 
 // optimize me. Use template to avoid copy code.
 using namespace AscendC;
-#ifdef ASCEND_310P // 310P not support 4bit get row
-    extern "C" __global__ __aicore__ void ascendc_get_row_q4_0(
-        GM_ADDR input_gm, GM_ADDR indices_gm, GM_ADDR output_gm,
-        GM_ADDR input_ne_gm, GM_ADDR indices_ne_gm, GM_ADDR indices_nb_gm,
-        GM_ADDR output_ne_gm, GM_ADDR output_nb_gm) {
-        // let following test cases can continue run, here just print error information. Of Cource the test case that call this operator is failed.
-        printf("Ascend310P not support 4bit get row.\n");
-    }
-#else
 
 #define BUFFER_NUM 2
 
@@ -200,5 +191,3 @@ extern "C" __global__ __aicore__ void ascendc_get_row_q4_0(
             indices_nb_ub, output_ne_ub, output_nb_ub);
     op.calculate();
 }
-
-#endif // #ifdef ASCEND_310P

ggml/src/ggml-cann/kernels/quantize_f16_q8_0.cpp

@@ -1,14 +1,6 @@
 #include "kernel_operator.h"
 
 using namespace AscendC;
-#ifdef ASCEND_310P
-    extern "C" __global__ __aicore__ void ascendc_quantize_f16_q8_0(
-        GM_ADDR input_gm, GM_ADDR output_gm, GM_ADDR input_ne_gm,
-        GM_ADDR input_nb_gm, GM_ADDR output_ne_gm) {
-        // let following test cases can continue run, here just print error information. Of Cource the test case that call this operator is failed.
-        printf("Ascend310P not support f16->8bit quantization.\n");
-    }
-#else
 
 #define BUFFER_NUM 2
 #define QK8_0 32
@@ -214,5 +206,3 @@ extern "C" __global__ __aicore__ void ascendc_quantize_f16_q8_0(
     op.init(input_gm, output_gm, input_ne_ub, input_nb_ub, output_ne_ub);
     op.calculate();
 }
-
-#endif // #ifdef ASCEND_310P

ggml/src/ggml-cann/kernels/quantize_f32_q8_0.cpp

@@ -1,14 +1,6 @@
 #include "kernel_operator.h"
 
 using namespace AscendC;
-#ifdef ASCEND_310P // 310P not support f32->8bit quantization
-    extern "C" __global__ __aicore__ void ascendc_quantize_f32_q8_0(
-        GM_ADDR input_gm, GM_ADDR output_gm, GM_ADDR input_ne_gm,
-        GM_ADDR input_nb_gm, GM_ADDR output_ne_gm) {
-        // let following test cases can continue run, here just print error information. Of Cource the test case that call this operator is failed.
-        printf("Ascend310P not support f32->8bit quantization.\n");
-    }
-#else
 
 #define BUFFER_NUM 2
 #define QK8_0 32
@@ -212,5 +204,3 @@ extern "C" __global__ __aicore__ void ascendc_quantize_f32_q8_0(
     op.init(input_gm, output_gm, input_ne_ub, input_nb_ub, output_ne_ub);
     op.calculate();
 }
-
-#endif // #ifdef ASCEND_310P

ggml/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp

@@ -1,21 +1,6 @@
 #include "kernel_operator.h"
 
 using namespace AscendC;
-#ifdef ASCEND_310P // 310P not support float->4bit quantization
-    extern "C" __global__ __aicore__ void ascendc_quantize_f32_to_q4_0(
-        GM_ADDR input_gm, GM_ADDR output_gm, GM_ADDR input_ne_gm,
-        GM_ADDR input_nb_gm, GM_ADDR output_ne_gm) {
-        // let following test cases can continue run, here just print error information. Of Cource the test case that call this operator is failed.
-        printf("Ascend310P not support f32->4bit quantization.\n");
-    }
-
-    extern "C" __global__ __aicore__ void ascendc_quantize_f16_to_q4_0(
-        GM_ADDR input_gm, GM_ADDR output_gm, GM_ADDR input_ne_gm,
-        GM_ADDR input_nb_gm, GM_ADDR output_ne_gm) {
-        // let following test cases can continue run, here just print error information. Of Cource the test case that call this operator is failed.
-        printf("Ascend310P not support f16->4bit quantization.\n");
-    }
-#else
 
 #define BUFFER_NUM 2
 #define Group_Size 32
@@ -291,5 +276,3 @@ extern "C" __global__ __aicore__ void ascendc_quantize_f32_to_q4_0(
     op.init(input_gm, output_gm, input_ne_ub, input_nb_ub, output_ne_ub);
     op.calculate();
 }
-
-#endif // #ifdef ASCEND_310P

ggml/src/ggml-common.h

@@ -6,20 +6,7 @@
 typedef uint16_t ggml_half;
 typedef uint32_t ggml_half2;
 
-#define GGML_COMMON_AGGR_U
-#define GGML_COMMON_AGGR_S
-
-#define GGML_COMMON_DECL
-#elif defined(GGML_COMMON_DECL_CPP)
-#include <cstdint>
-
-typedef uint16_t ggml_half;
-typedef uint32_t ggml_half2;
-
-// std-c++ allow anonymous unions but some compiler warn on it
-#define GGML_COMMON_AGGR_U data
-// std-c++ do not allow it.
-#define GGML_COMMON_AGGR_S data
+#define GGML_COMMON_AGGR
 
 #define GGML_COMMON_DECL
 #elif defined(GGML_COMMON_DECL_METAL)
@@ -28,8 +15,7 @@ typedef uint32_t ggml_half2;
 typedef half  ggml_half;
 typedef half2 ggml_half2;
 
-#define GGML_COMMON_AGGR_U
-#define GGML_COMMON_AGGR_S
+#define GGML_COMMON_AGGR
 
 #define GGML_COMMON_DECL
 #elif defined(GGML_COMMON_DECL_CUDA)
@@ -43,8 +29,7 @@ typedef half2 ggml_half2;
 typedef half  ggml_half;
 typedef half2 ggml_half2;
 
-#define GGML_COMMON_AGGR_U
-#define GGML_COMMON_AGGR_S data
+#define GGML_COMMON_AGGR data
 
 #define GGML_COMMON_DECL
 #elif defined(GGML_COMMON_DECL_HIP)
@@ -54,8 +39,7 @@ typedef half2 ggml_half2;
 typedef half  ggml_half;
 typedef half2 ggml_half2;
 
-#define GGML_COMMON_AGGR_U
-#define GGML_COMMON_AGGR_S data
+#define GGML_COMMON_AGGR data
 
 #define GGML_COMMON_DECL
 #elif defined(GGML_COMMON_DECL_SYCL)
@@ -65,8 +49,7 @@ typedef half2 ggml_half2;
 typedef sycl::half  ggml_half;
 typedef sycl::half2 ggml_half2;
 
-#define GGML_COMMON_AGGR_U
-#define GGML_COMMON_AGGR_S data
+#define GGML_COMMON_AGGR data
 
 #define GGML_COMMON_DECL
 #endif
@@ -171,9 +154,9 @@ typedef struct {
         struct {
             ggml_half d; // delta
             ggml_half m; // min
-        } GGML_COMMON_AGGR_S;
+        } GGML_COMMON_AGGR;
         ggml_half2 dm;
-    } GGML_COMMON_AGGR_U;
+    };
     uint8_t qs[QK4_1 / 2]; // nibbles / quants
 } block_q4_1;
 static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding");
@@ -192,9 +175,9 @@ typedef struct {
         struct {
             ggml_half d; // delta
             ggml_half m; // min
-        } GGML_COMMON_AGGR_S;
+        } GGML_COMMON_AGGR;
         ggml_half2 dm;
-    } GGML_COMMON_AGGR_U;
+    };
     uint8_t qh[4];         // 5-th bit of quants
     uint8_t qs[QK5_1 / 2]; // nibbles / quants
 } block_q5_1;
@@ -213,13 +196,37 @@ typedef struct {
         struct {
             ggml_half d; // delta
             ggml_half s; // d * sum(qs[i])
-        } GGML_COMMON_AGGR_S;
+        } GGML_COMMON_AGGR;
         ggml_half2 ds;
-    } GGML_COMMON_AGGR_U;
+    };
     int8_t qs[QK8_1]; // quants
 } block_q8_1;
 static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, "wrong q8_1 block size/padding");
 
+typedef struct {
+    ggml_half d[4];        // deltas for 4 q4_0 blocks
+    uint8_t qs[QK4_0 * 2]; // nibbles / quants for 4 q4_0 blocks
+} block_q4_0x4;
+static_assert(sizeof(block_q4_0x4) == 4 * sizeof(ggml_half) + QK4_0 * 2, "wrong q4_0x4 block size/padding");
+
+typedef struct {
+    ggml_half d[8];        // deltas for 8 q4_0 blocks
+    uint8_t qs[QK4_0 * 4]; // nibbles / quants for 8 q4_0 blocks
+} block_q4_0x8;
+static_assert(sizeof(block_q4_0x8) == 8 * sizeof(ggml_half) + QK4_0 * 4, "wrong q4_0x8 block size/padding");
+
+typedef struct {
+    ggml_half d[4];        // deltas for 4 q8_0 blocks
+    int8_t qs[QK8_0 * 4];  // quants for 4 q8_0 blocks
+} block_q8_0x4;
+static_assert(sizeof(block_q8_0x4) == 4 * sizeof(ggml_half) + QK8_0 * 4, "wrong q8_0x4 block size/padding");
+
+typedef struct {
+    ggml_half d[8];        // deltas for 8 q8_0 blocks
+    int8_t qs[QK8_0 * 8];  // quants for 8 q8_0 blocks
+} block_q8_0x8;
+static_assert(sizeof(block_q8_0x8) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong q8_0x8 block size/padding");
+
 //
 // Ternary quantization
 //
@@ -254,9 +261,9 @@ typedef struct {
         struct {
             ggml_half d;    // super-block scale for quantized scales
             ggml_half dmin; // super-block scale for quantized mins
-        } GGML_COMMON_AGGR_S;
+        } GGML_COMMON_AGGR;
         ggml_half2 dm;
-    } GGML_COMMON_AGGR_U;
+    };
 } block_q2_K;
 static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");
 
@@ -281,9 +288,9 @@ typedef struct {
         struct {
             ggml_half d;    // super-block scale for quantized scales
             ggml_half dmin; // super-block scale for quantized mins
-        } GGML_COMMON_AGGR_S;
+        } GGML_COMMON_AGGR;
         ggml_half2 dm;
-    } GGML_COMMON_AGGR_U;
+    };
     uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
     uint8_t qs[QK_K/2];           // 4--bit quants
 } block_q4_K;
@@ -298,9 +305,9 @@ typedef struct {
         struct {
             ggml_half d;    // super-block scale for quantized scales
             ggml_half dmin; // super-block scale for quantized mins
-        } GGML_COMMON_AGGR_S;
+        } GGML_COMMON_AGGR;
         ggml_half2 dm;
-    } GGML_COMMON_AGGR_U;
+    };
     uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
     uint8_t qh[QK_K/8];           // quants, high bit
     uint8_t qs[QK_K/2];           // quants, low 4 bits
@@ -424,13 +431,6 @@ static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_
 #define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
 #define GGML_TABLE_END() };
 
-#define GGML_COMMON_IMPL
-#elif defined(GGML_COMMON_IMPL_CPP)
-#include <cstdint>
-
-#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
-#define GGML_TABLE_END() };
-
 #define GGML_COMMON_IMPL
 #elif defined(GGML_COMMON_IMPL_METAL)
 #include <metal_stdlib>
@@ -473,7 +473,7 @@ GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)
     240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,
 GGML_TABLE_END()
 
-//#if __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A // lowest compute capability for integer intrinsics
+//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics
 GGML_TABLE_BEGIN(uint64_t, ksigns64, 128)
     0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,
     0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,

ggml/src/ggml-cpu/CMakeLists.txt

@@ -1,358 +1,261 @@
-function(ggml_add_cpu_backend_variant_impl tag_name)
-    if (tag_name)
-        set(GGML_CPU_NAME ggml-cpu-${tag_name})
+add_library(ggml-cpu
+            ggml-cpu.c
+            ggml-cpu.cpp
+            ggml-cpu-aarch64.c
+            ggml-cpu-aarch64.h
+            ggml-cpu-quants.c
+            ggml-cpu-quants.h
+            )
+
+target_link_libraries(ggml-cpu PRIVATE ggml-base)
+target_include_directories(ggml-cpu PRIVATE . ..)
+
+if (APPLE AND GGML_ACCELERATE)
+    find_library(ACCELERATE_FRAMEWORK Accelerate)
+    if (ACCELERATE_FRAMEWORK)
+        message(STATUS "Accelerate framework found")
+
+        add_compile_definitions(GGML_USE_ACCELERATE)
+        add_compile_definitions(ACCELERATE_NEW_LAPACK)
+        add_compile_definitions(ACCELERATE_LAPACK_ILP64)
+
+        target_link_libraries(ggml-cpu PRIVATE ${ACCELERATE_FRAMEWORK})
     else()
-        set(GGML_CPU_NAME ggml-cpu)
+        message(WARNING "Accelerate framework not found")
     endif()
+endif()
 
-    ggml_add_backend_library(${GGML_CPU_NAME})
+if (GGML_OPENMP)
+    find_package(OpenMP)
+    if (OpenMP_FOUND)
+        message(STATUS "OpenMP found")
 
-    list (APPEND GGML_CPU_SOURCES
-        ggml-cpu/ggml-cpu.c
-        ggml-cpu/ggml-cpu.cpp
-        ggml-cpu/ggml-cpu-aarch64.cpp
-        ggml-cpu/ggml-cpu-aarch64.h
-        ggml-cpu/ggml-cpu-hbm.cpp
-        ggml-cpu/ggml-cpu-hbm.h
-        ggml-cpu/ggml-cpu-quants.c
-        ggml-cpu/ggml-cpu-quants.h
-        ggml-cpu/ggml-cpu-traits.cpp
-        ggml-cpu/ggml-cpu-traits.h
-        ggml-cpu/amx/amx.cpp
-        ggml-cpu/amx/amx.h
-        ggml-cpu/amx/mmq.cpp
-        ggml-cpu/amx/mmq.h
-        ggml-cpu/ggml-cpu-impl.h
-        )
+        add_compile_definitions(GGML_USE_OPENMP)
 
-    target_compile_features(${GGML_CPU_NAME} PRIVATE c_std_11 cxx_std_17)
-    target_include_directories(${GGML_CPU_NAME} PRIVATE . ggml-cpu)
+        target_link_libraries(ggml-cpu PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX)
 
-    if (APPLE AND GGML_ACCELERATE)
-        find_library(ACCELERATE_FRAMEWORK Accelerate)
-        if (ACCELERATE_FRAMEWORK)
-            message(STATUS "Accelerate framework found")
+        # FIXME: should be replaced with a compiler id check
+        #if (GGML_MUSA)
+        #    list(APPEND GGML_CPU_EXTRA_INCLUDES     "/usr/lib/llvm-14/lib/clang/14.0.0/include")
+        #    list(APPEND GGML_CPU_EXTRA_LIBS_PRIVATE "/usr/lib/llvm-14/lib/libomp.so")
+        #endif()
+    else()
+        message(WARNING "OpenMP not found")
+    endif()
+endif()
 
-            target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_ACCELERATE)
-            target_compile_definitions(${GGML_CPU_NAME} PRIVATE ACCELERATE_NEW_LAPACK)
-            target_compile_definitions(${GGML_CPU_NAME} PRIVATE ACCELERATE_LAPACK_ILP64)
+if (GGML_LLAMAFILE)
+    message(STATUS "Using llamafile")
 
-            target_link_libraries(${GGML_CPU_NAME} PRIVATE ${ACCELERATE_FRAMEWORK})
-        else()
-            message(WARNING "Accelerate framework not found")
+    add_compile_definitions(GGML_USE_LLAMAFILE)
+
+    target_sources(ggml-cpu PRIVATE
+                    llamafile/sgemm.cpp
+                    llamafile/sgemm.h)
+endif()
+
+if (GGML_CPU_HBM)
+    find_library(memkind memkind REQUIRED)
+
+    message(STATUS "Using memkind for CPU HBM")
+
+    add_compile_definitions(GGML_USE_CPU_HBM)
+
+    target_link_libraries(ggml-cpu PUBLIC memkind)
+endif()
+
+if (CMAKE_OSX_ARCHITECTURES      STREQUAL "arm64" OR
+    CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
+    (NOT CMAKE_OSX_ARCHITECTURES      AND
+     NOT CMAKE_GENERATOR_PLATFORM_LWR AND
+         CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
+
+    message(STATUS "ARM detected")
+
+    if (MSVC)
+        add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead
+        add_compile_definitions(__ARM_NEON)
+        add_compile_definitions(__ARM_FEATURE_FMA)
+
+        set(CMAKE_REQUIRED_FLAGS_PREV ${CMAKE_REQUIRED_FLAGS})
+        string(JOIN " " CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} "/arch:armv8.2")
+
+        check_cxx_source_compiles("#include <arm_neon.h>\nint main() { int8x16_t _a, _b; int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_DOTPROD)
+        if (GGML_COMPILER_SUPPORT_DOTPROD)
+            add_compile_definitions(__ARM_FEATURE_DOTPROD)
+        endif ()
+
+        check_cxx_source_compiles("#include <arm_neon.h>\nint main() { int8x16_t _a, _b; int32x4_t _s = vmlaq_f32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_MATMUL_INT8)
+
+        if (GGML_COMPILER_SUPPORT_MATMUL_INT8)
+            add_compile_definitions(__ARM_FEATURE_MATMUL_INT8)
+        endif ()
+
+        check_cxx_source_compiles("#include <arm_neon.h>\nint main() { float16_t _a; float16x8_t _s = vdupq_n_f16(_a); return 0; }" GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC)
+        if (GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC)
+            add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+        endif ()
+
+        set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_PREV})
+    else()
+        check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E)
+        if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
+            list(APPEND ARCH_FLAGS -mfp16-format=ieee)
         endif()
-    endif()
-
-    if (GGML_OPENMP)
-        find_package(OpenMP)
-        if (OpenMP_FOUND)
-            target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP)
-
-            target_link_libraries(${GGML_CPU_NAME} PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX)
-        else()
-            message(WARNING "OpenMP not found")
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6")
+            # Raspberry Pi 1, Zero
+            list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access)
         endif()
-    endif()
-
-    if (GGML_LLAMAFILE)
-        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_LLAMAFILE)
-
-        list(APPEND GGML_CPU_SOURCES
-                    ggml-cpu/llamafile/sgemm.cpp
-                    ggml-cpu/llamafile/sgemm.h)
-    endif()
-
-    if (GGML_CPU_HBM)
-        find_library(memkind memkind REQUIRED)
-
-        message(STATUS "Using memkind for CPU HBM")
-
-        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_HBM)
-
-        target_link_libraries(${GGML_CPU_NAME} PUBLIC memkind)
-    endif()
-
-    if (CMAKE_OSX_ARCHITECTURES      STREQUAL "arm64" OR
-        CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
-        (NOT CMAKE_OSX_ARCHITECTURES      AND
-        NOT CMAKE_GENERATOR_PLATFORM_LWR AND
-            CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
-
-        message(STATUS "ARM detected")
-
-        if (MSVC)
-            list(APPEND ARCH_DEFINITIONS __aarch64__) # MSVC defines _M_ARM64 instead
-            list(APPEND ARCH_DEFINITIONS __ARM_NEON)
-            list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_FMA)
-
-            set(CMAKE_REQUIRED_FLAGS_PREV ${CMAKE_REQUIRED_FLAGS})
-            string(JOIN " " CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} "/arch:armv8.2")
-
-            check_cxx_source_compiles("#include <arm_neon.h>\nint main() { int8x16_t _a, _b; int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_DOTPROD)
-            if (GGML_COMPILER_SUPPORT_DOTPROD)
-                list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_DOTPROD)
-
-                message(STATUS "ARM feature DOTPROD enabled")
-            endif ()
-
-            check_cxx_source_compiles("#include <arm_neon.h>\nint main() { int8x16_t _a, _b; int32x4_t _s = vmmlaq_f32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_MATMUL_INT8)
-
-            if (GGML_COMPILER_SUPPORT_MATMUL_INT8)
-                list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_MATMUL_INT8)
-
-                message(STATUS "ARM feature MATMUL_INT8 enabled")
-            endif ()
-
-            check_cxx_source_compiles("#include <arm_neon.h>\nint main() { float16_t _a; float16x8_t _s = vdupq_n_f16(_a); return 0; }" GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC)
-            if (GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC)
-                list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
-
-                message(STATUS "ARM feature FP16_VECTOR_ARITHMETIC enabled")
-            endif ()
-
-            set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_PREV})
-        elseif (APPLE)
-            if (GGML_NATIVE)
-                set(USER_PROVIDED_MARCH FALSE)
-                foreach(flag_var IN ITEMS CMAKE_C_FLAGS CMAKE_CXX_FLAGS CMAKE_REQUIRED_FLAGS)
-                    if ("${${flag_var}}" MATCHES "-march=[a-zA-Z0-9+._-]+")
-                        set(USER_PROVIDED_MARCH TRUE)
-                        break()
-                    endif()
-                endforeach()
-
-                if (NOT USER_PROVIDED_MARCH)
-                    set(MARCH_FLAGS "-march=armv8.2a")
-
-                    check_cxx_source_compiles("#include <arm_neon.h>\nint main() { int8x16_t _a, _b; int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_DOTPROD)
-                    if (GGML_COMPILER_SUPPORT_DOTPROD)
-                        set(MARCH_FLAGS "${MARCH_FLAGS}+dotprod")
-                        list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_DOTPROD)
-
-                        message(STATUS "ARM feature DOTPROD enabled")
-                    endif ()
-
-                    set(TEST_I8MM_FLAGS "-march=armv8.2a+i8mm")
-
-                    set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})
-                    set(CMAKE_REQUIRED_FLAGS     "${CMAKE_REQUIRED_FLAGS} ${TEST_I8MM_FLAGS}")
-
-                    check_cxx_source_compiles("#include <arm_neon.h>\nint main() { int8x16_t _a, _b; int32x4_t _s = vmmlaq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_MATMUL_INT8)
-                    if (GGML_COMPILER_SUPPORT_MATMUL_INT8)
-                        set(MARCH_FLAGS "${MARCH_FLAGS}+i8mm")
-                        list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_MATMUL_INT8)
-
-                        message(STATUS "ARM feature MATMUL_INT8 enabled")
-                    endif ()
-
-                    set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})
-
-                    list(APPEND ARCH_FLAGS "${MARCH_FLAGS}")
-                endif ()
-            endif ()
-        else()
-            check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E)
-            if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
-                list(APPEND ARCH_FLAGS -mfp16-format=ieee)
-            endif()
-            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6")
-                # Raspberry Pi 1, Zero
-                list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access)
-            endif()
-            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7")
-                if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android")
-                    # Android armeabi-v7a
-                    list(APPEND ARCH_FLAGS -mfpu=neon-vfpv4 -mno-unaligned-access -funsafe-math-optimizations)
-                else()
-                    # Raspberry Pi 2
-                    list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations)
-                endif()
-            endif()
-            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8")
-                # Android arm64-v8a
-                # Raspberry Pi 3, 4, Zero 2 (32-bit)
-                list(APPEND ARCH_FLAGS -mno-unaligned-access)
-            endif()
-            if (GGML_SVE)
-                list(APPEND ARCH_FLAGS -march=armv8.6-a+sve)
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7")
+            if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android")
+                # Android armeabi-v7a
+                list(APPEND ARCH_FLAGS -mfpu=neon-vfpv4 -mno-unaligned-access -funsafe-math-optimizations)
+            else()
+                # Raspberry Pi 2
+                list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations)
             endif()
         endif()
-    elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
-            (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
-            CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64|amd64)$"))
-        if (MSVC)
-            # instruction set detection for MSVC only
-            if (GGML_NATIVE)
-                include(ggml-cpu/cmake/FindSIMD.cmake)
-            endif ()
-            if (GGML_AVX512)
-                list(APPEND ARCH_FLAGS /arch:AVX512)
-                # /arch:AVX512 includes: __AVX512F__, __AVX512CD__, __AVX512BW__, __AVX512DQ__, and __AVX512VL__
-                # MSVC has no compile-time flags enabling specific
-                # AVX512 extensions, neither it defines the
-                # macros corresponding to the extensions.
-                # Do it manually.
-                list(APPEND ARCH_DEFINITIONS GGML_AVX512)
-                if (GGML_AVX512_VBMI)
-                    list(APPEND ARCH_DEFINITIONS __AVX512VBMI__)
-                    if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
-                        list(APPEND ARCH_FLAGS -mavx512vbmi)
-                    endif()
-                endif()
-                if (GGML_AVX512_VNNI)
-                    list(APPEND ARCH_DEFINITIONS __AVX512VNNI__ GGML_AVX512_VNNI)
-                    if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
-                        list(APPEND ARCH_FLAGS -mavx512vnni)
-                    endif()
-                endif()
-                if (GGML_AVX512_BF16)
-                    list(APPEND ARCH_DEFINITIONS __AVX512BF16__ GGML_AVX512_BF16)
-                    if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
-                        list(APPEND ARCH_FLAGS -mavx512bf16)
-                    endif()
-                endif()
-                if (GGML_AMX_TILE)
-                    list(APPEND ARCH_DEFINITIONS __AMX_TILE__ GGML_AMX_TILE)
-                endif()
-                if (GGML_AMX_INT8)
-                    list(APPEND ARCH_DEFINITIONS __AMX_INT8__ GGML_AMX_INT8)
-                endif()
-                if (GGML_AMX_BF16)
-                    list(APPEND ARCH_DEFINITIONS __AMX_BF16__ GGML_AMX_BF16)
-                endif()
-            elseif (GGML_AVX2)
-                list(APPEND ARCH_FLAGS /arch:AVX2)
-                list(APPEND ARCH_DEFINITIONS GGML_AVX2 GGML_FMA GGML_F16C)
-            elseif (GGML_AVX)
-                list(APPEND ARCH_FLAGS /arch:AVX)
-                list(APPEND ARCH_DEFINITIONS GGML_AVX)
-            else ()
-                list(APPEND ARCH_FLAGS /arch:SSE4.2)
-                list(APPEND ARCH_DEFINITIONS GGML_SSE42)
-            endif()
-            if (GGML_AVX_VNNI)
-                # MSVC generates AVX512 with AVX-VNNI intrinsics even with /arch:AVX2
-                #list(APPEND ARCH_DEFINITIONS __AVXVNNI__ GGML_AVX_VNNI)
-            endif()
-        else ()
-            if (GGML_NATIVE)
-                list(APPEND ARCH_FLAGS -march=native)
-            else ()
-                list(APPEND ARCH_FLAGS -msse4.2)
-                list(APPEND ARCH_DEFINITIONS GGML_SSE42)
-                if (GGML_F16C)
-                    list(APPEND ARCH_FLAGS -mf16c)
-                    list(APPEND ARCH_DEFINITIONS GGML_F16C)
-                endif()
-                if (GGML_FMA)
-                    list(APPEND ARCH_FLAGS -mfma)
-                    list(APPEND ARCH_DEFINITIONS GGML_FMA)
-                endif()
-                if (GGML_AVX)
-                    list(APPEND ARCH_FLAGS -mavx)
-                    list(APPEND ARCH_DEFINITIONS GGML_AVX)
-                endif()
-                if (GGML_AVX2)
-                    list(APPEND ARCH_FLAGS -mavx2)
-                    list(APPEND ARCH_DEFINITIONS GGML_AVX2)
-                endif()
-                if (GGML_AVX_VNNI)
-                    list(APPEND ARCH_FLAGS -mavxvnni)
-                    list(APPEND ARCH_DEFINITIONS GGML_AVX_VNNI)
-                endif()
-                if (GGML_AVX512)
-                    list(APPEND ARCH_FLAGS -mavx512f)
-                    list(APPEND ARCH_FLAGS -mavx512cd)
-                    list(APPEND ARCH_FLAGS -mavx512vl)
-                    list(APPEND ARCH_FLAGS -mavx512dq)
-                    list(APPEND ARCH_FLAGS -mavx512bw)
-                    list(APPEND ARCH_DEFINITIONS GGML_AVX512)
-                endif()
-                if (GGML_AVX512_VBMI)
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8")
+            # Android arm64-v8a
+            # Raspberry Pi 3, 4, Zero 2 (32-bit)
+            list(APPEND ARCH_FLAGS -mno-unaligned-access)
+        endif()
+        if (GGML_SVE)
+            list(APPEND ARCH_FLAGS -march=armv8.6-a+sve)
+        endif()
+    endif()
+elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
+        (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
+         CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64)$"))
+    message(STATUS "x86 detected")
+    if (MSVC)
+        # instruction set detection for MSVC only
+        if (GGML_NATIVE)
+            # TODO: improve, should not reference files from the parent folder
+            include(cmake/FindSIMD.cmake)
+        endif ()
+        if (GGML_AVX512)
+            list(APPEND ARCH_FLAGS /arch:AVX512)
+            # MSVC has no compile-time flags enabling specific
+            # AVX512 extensions, neither it defines the
+            # macros corresponding to the extensions.
+            # Do it manually.
+            if (GGML_AVX512_VBMI)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
+                if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
                     list(APPEND ARCH_FLAGS -mavx512vbmi)
-                    list(APPEND ARCH_DEFINITIONS GGML_AVX512_VBMI)
-                endif()
-                if (GGML_AVX512_VNNI)
-                    list(APPEND ARCH_FLAGS -mavx512vnni)
-                    list(APPEND ARCH_DEFINITIONS GGML_AVX512_VNNI)
-                endif()
-                if (GGML_AVX512_BF16)
-                    list(APPEND ARCH_FLAGS -mavx512bf16)
-                    list(APPEND ARCH_DEFINITIONS GGML_AVX512_BF16)
-                endif()
-                if (GGML_AMX_TILE)
-                    list(APPEND ARCH_FLAGS -mamx-tile)
-                    list(APPEND ARCH_DEFINITIONS GGML_AMX_TILE)
-                endif()
-                if (GGML_AMX_INT8)
-                    list(APPEND ARCH_FLAGS -mamx-int8)
-                    list(APPEND ARCH_DEFINITIONS GGML_AMX_INT8)
-                endif()
-                if (GGML_AMX_BF16)
-                    list(APPEND ARCH_FLAGS -mamx-bf16)
-                    list(APPEND ARCH_DEFINITIONS GGML_AMX_BF16)
                 endif()
             endif()
-        endif()
-    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
-        message(STATUS "PowerPC detected")
-        execute_process(COMMAND bash -c "grep POWER10 /proc/cpuinfo | head -n 1" OUTPUT_VARIABLE POWER10_M)
-        string(FIND "${POWER10_M}" "POWER10" substring_index)
-        if (NOT DEFINED substring_index OR "${substring_index}" STREQUAL "")
-            set(substring_index -1)
-        endif()
-
-        if (${substring_index} GREATER_EQUAL 0)
-        list(APPEND ARCH_FLAGS -mcpu=power10)
-        elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le")
-        list(APPEND ARCH_FLAGS -mcpu=powerpc64le)
-        else()
-            list(APPEND ARCH_FLAGS -mcpu=native -mtune=native)
-            # TODO: Add  targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
-        endif()
-    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
-        message(STATUS "loongarch64 detected")
-
-        list(APPEND ARCH_FLAGS -march=loongarch64)
-        if (GGML_LASX)
-            list(APPEND ARCH_FLAGS -mlasx)
-        endif()
-        if (GGML_LSX)
-            list(APPEND ARCH_FLAGS -mlsx)
-        endif()
-    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64")
-        message(STATUS "RISC-V detected")
-        if (GGML_RVV)
-            list(APPEND ARCH_FLAGS -march=rv64gcv -mabi=lp64d)
+            if (GGML_AVX512_VNNI)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
+                if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
+                    list(APPEND ARCH_FLAGS -mavx512vnni)
+                endif()
+            endif()
+            if (GGML_AVX512_BF16)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512BF16__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512BF16__>)
+                if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
+                    list(APPEND ARCH_FLAGS -mavx512bf16)
+                endif()
+            endif()
+            if (GGML_AMX_TILE)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AMX_TILE__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AMX_TILE__>)
+            endif()
+            if (GGML_AMX_INT8)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AMX_INT8__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AMX_INT8__>)
+            endif()
+            if (GGML_AMX_BF16)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AMX_BF16__>)
+                add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AMX_BF16__>)
+            endif()
+        elseif (GGML_AVX2)
+            list(APPEND ARCH_FLAGS /arch:AVX2)
+        elseif (GGML_AVX)
+            list(APPEND ARCH_FLAGS /arch:AVX)
         endif()
     else()
-        message(STATUS "Unknown architecture")
+        if (GGML_NATIVE)
+            list(APPEND ARCH_FLAGS -march=native)
+        endif()
+        if (GGML_F16C)
+            list(APPEND ARCH_FLAGS -mf16c)
+        endif()
+        if (GGML_FMA)
+            list(APPEND ARCH_FLAGS -mfma)
+        endif()
+        if (GGML_AVX)
+            list(APPEND ARCH_FLAGS -mavx)
+        endif()
+        if (GGML_AVX2)
+            list(APPEND ARCH_FLAGS -mavx2)
+        endif()
+        if (GGML_AVX512)
+            list(APPEND ARCH_FLAGS -mavx512f)
+            list(APPEND ARCH_FLAGS -mavx512dq)
+            list(APPEND ARCH_FLAGS -mavx512bw)
+        endif()
+        if (GGML_AVX512_VBMI)
+            list(APPEND ARCH_FLAGS -mavx512vbmi)
+        endif()
+        if (GGML_AVX512_VNNI)
+            list(APPEND ARCH_FLAGS -mavx512vnni)
+        endif()
+        if (GGML_AVX512_BF16)
+            list(APPEND ARCH_FLAGS -mavx512bf16)
+        endif()
+        if (GGML_AMX_TILE)
+            list(APPEND ARCH_FLAGS -mamx-tile)
+        endif()
+        if (GGML_AMX_INT8)
+            list(APPEND ARCH_FLAGS -mamx-int8)
+        endif()
+        if (GGML_AMX_BF16)
+            list(APPEND ARCH_FLAGS -mamx-bf16)
+        endif()
+    endif()
+elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
+    message(STATUS "PowerPC detected")
+    execute_process(COMMAND bash -c "grep POWER10 /proc/cpuinfo | head -n 1" OUTPUT_VARIABLE POWER10_M)
+    string(FIND "${POWER10_M}" "POWER10" substring_index)
+    if (NOT DEFINED substring_index OR "${substring_index}" STREQUAL "")
+        set(substring_index -1)
     endif()
 
-    if (GGML_CPU_AARCH64)
-        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_AARCH64)
+    if (${substring_index} GREATER_EQUAL 0)
+       list(APPEND ARCH_FLAGS -mcpu=power10)
+    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le")
+       list(APPEND ARCH_FLAGS -mcpu=powerpc64le)
+    else()
+        list(APPEND ARCH_FLAGS -mcpu=native -mtune=native)
+        #TODO: Add  targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
     endif()
+elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
+    message(STATUS "loongarch64 detected")
 
-    message(STATUS "Adding CPU backend variant ${GGML_CPU_NAME}: ${ARCH_FLAGS} ${ARCH_DEFINITIONS}")
-    target_sources(${GGML_CPU_NAME} PRIVATE ${GGML_CPU_SOURCES})
-    target_compile_options(${GGML_CPU_NAME} PRIVATE ${ARCH_FLAGS})
-    target_compile_definitions(${GGML_CPU_NAME} PRIVATE ${ARCH_DEFINITIONS})
-
-    if (GGML_BACKEND_DL)
-        # The feature detection code is compiled as a separate target so that
-        # it can be built without the architecture flags
-        # Since multiple variants of the CPU backend may be included in the same
-        # build, using set_source_files_properties() to set the arch flags is not possible
-        set(GGML_CPU_FEATS_NAME ${GGML_CPU_NAME}-feats)
-        add_library(${GGML_CPU_FEATS_NAME} OBJECT ggml-cpu/cpu-feats-x86.cpp)
-        target_include_directories(${GGML_CPU_FEATS_NAME} PRIVATE . .. ../include)
-        target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE ${ARCH_DEFINITIONS})
-        target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE GGML_BACKEND_DL GGML_BACKEND_BUILD GGML_BACKEND_SHARED)
-        set_target_properties(${GGML_CPU_FEATS_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
-        target_link_libraries(${GGML_CPU_NAME} PRIVATE ${GGML_CPU_FEATS_NAME})
+    list(APPEND ARCH_FLAGS -march=loongarch64)
+    if (GGML_LASX)
+        list(APPEND ARCH_FLAGS -mlasx)
     endif()
-
-    if (EMSCRIPTEN)
-        set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS "-msimd128")
+    if (GGML_LSX)
+        list(APPEND ARCH_FLAGS -mlsx)
     endif()
-endfunction()
+else()
+    message(STATUS "Unknown architecture")
+endif()
+
+if (GGML_CPU_AARCH64)
+    message(STATUS "Using runtime weight conversion of Q4_0 to Q4_0_x_x to enable optimized GEMM/GEMV kernels")
+    add_compile_definitions(GGML_USE_CPU_AARCH64)
+endif()
+
+target_compile_options(ggml-cpu PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:${ARCH_FLAGS}>")
+target_compile_options(ggml-cpu PRIVATE "$<$<COMPILE_LANGUAGE:C>:${ARCH_FLAGS}>")
+
+if (EMSCRIPTEN)
+    set_target_properties(ggml-cpu PROPERTIES COMPILE_FLAGS "-msimd128")
+endif()

ggml/src/ggml-cpu/amx/amx.cpp

@@ -1,220 +0,0 @@
-#include "amx.h"
-#include "common.h"
-#include "mmq.h"
-#include "ggml-backend-impl.h"
-#include "ggml-backend.h"
-#include "ggml-impl.h"
-#include "ggml-cpu.h"
-#include "ggml-cpu-traits.h"
-
-#if defined(__gnu_linux__)
-#include <sys/syscall.h>
-#include <unistd.h>
-#endif
-
-#include <cstdlib>
-#include <cstring>
-#include <memory>
-
-#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)
-
-// AMX type_trais
-namespace ggml::cpu::amx {
-class tensor_traits : public ggml::cpu::tensor_traits {
-    bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {
-        size = ggml_backend_amx_desired_wsize(op);
-        return true;
-    }
-
-    bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override {
-        if (op->op == GGML_OP_MUL_MAT) {
-            ggml_backend_amx_mul_mat(params, op);
-            return true;
-        }
-        return false;
-    }
-};
-
-static ggml::cpu::tensor_traits * get_tensor_traits(ggml_backend_buffer_t, struct ggml_tensor *) {
-    static tensor_traits traits;
-    return &traits;
-}
-}  // namespace ggml::cpu::amx
-
-// AMX buffer interface
-static void ggml_backend_amx_buffer_free_buffer(ggml_backend_buffer_t buffer) {
-    free(buffer->context);
-}
-
-static void * ggml_backend_amx_buffer_get_base(ggml_backend_buffer_t buffer) {
-    return (void *) (buffer->context);
-}
-
-static void ggml_backend_amx_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {
-    tensor->extra = (void *) ggml::cpu::amx::get_tensor_traits(buffer, tensor);
-
-    GGML_UNUSED(buffer);
-}
-
-static void ggml_backend_amx_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,
-                                                  uint8_t value, size_t offset, size_t size) {
-    memset((char *) tensor->data + offset, value, size);
-
-    GGML_UNUSED(buffer);
-}
-
-static void ggml_backend_amx_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,
-                                               const void * data, size_t offset, size_t size) {
-    if (qtype_has_amx_kernels(tensor->type)) {
-        GGML_LOG_DEBUG("%s: amx repack tensor %s of type %s\n", __func__, tensor->name, ggml_type_name(tensor->type));
-        ggml_backend_amx_convert_weight(tensor, data, offset, size);
-    } else {
-        memcpy((char *) tensor->data + offset, data, size);
-    }
-
-    GGML_UNUSED(buffer);
-}
-
-/*
-// need to figure what we need to do with buffer->extra.
-static void ggml_backend_amx_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
-    GGML_ASSERT(!qtype_has_amx_kernels(tensor->type));
-    memcpy(data, (const char *)tensor->data + offset, size);
-
-    GGML_UNUSED(buffer);
-}
-
-static bool ggml_backend_amx_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
-    if (ggml_backend_buffer_is_host(src->buffer)) {
-        if (qtype_has_amx_kernels(src->type)) {
-            ggml_backend_amx_convert_weight(dst, src->data, 0, ggml_nbytes(dst));
-        } else {
-            memcpy(dst->data, src->data, ggml_nbytes(src));
-        }
-        return true;
-    }
-    return false;
-
-    GGML_UNUSED(buffer);
-}
-*/
-
-static void ggml_backend_amx_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
-    memset(buffer->context, value, buffer->size);
-}
-
-static ggml_backend_buffer_i ggml_backend_amx_buffer_interface = {
-    /* .free_buffer     = */ ggml_backend_amx_buffer_free_buffer,
-    /* .get_base        = */ ggml_backend_amx_buffer_get_base,
-    /* .init_tensor     = */ ggml_backend_amx_buffer_init_tensor,
-    /* .memset_tensor   = */ ggml_backend_amx_buffer_memset_tensor,
-    /* .set_tensor      = */ ggml_backend_amx_buffer_set_tensor,
-    /* .get_tensor      = */ nullptr,
-    /* .cpy_tensor      = */ nullptr,
-    /* .clear           = */ ggml_backend_amx_buffer_clear,
-    /* .reset           = */ nullptr,
-};
-
-static const char * ggml_backend_amx_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
-    return "AMX";
-
-    GGML_UNUSED(buft);
-}
-
-static ggml_backend_buffer_t ggml_backend_amx_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
-    void * data = ggml_aligned_malloc(size);
-    if (data == NULL) {
-        fprintf(stderr, "%s: failed to allocate buffer of size %zu\n", __func__, size);
-        return NULL;
-    }
-
-    return ggml_backend_buffer_init(buft, ggml_backend_amx_buffer_interface, data, size);
-}
-
-static size_t ggml_backend_amx_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
-    return TENSOR_ALIGNMENT;
-
-    GGML_UNUSED(buft);
-}
-
-namespace ggml::cpu::amx {
-class extra_buffer_type : ggml::cpu::extra_buffer_type {
-    bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {
-        // handle only 2d gemm for now
-        auto is_contiguous_2d = [](const struct ggml_tensor * t) {
-            return ggml_is_contiguous(t) && t->ne[3] == 1 && t->ne[2] == 1;
-        };
-
-        if (op->op == GGML_OP_MUL_MAT && is_contiguous_2d(op->src[0]) &&  // src0 must be contiguous
-            is_contiguous_2d(op->src[1]) &&                               // src1 must be contiguous
-            op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_amx_buffer_type() &&
-            op->ne[0] % (TILE_N * 2) == 0 &&                              // out_features is 32x
-            (qtype_has_amx_kernels(op->src[0]->type) || (op->src[0]->type == GGML_TYPE_F16))) {
-            // src1 must be host buffer
-            if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {
-                return false;
-            }
-            // src1 must be float32
-            if (op->src[1]->type == GGML_TYPE_F32) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {
-        if (op->op == GGML_OP_MUL_MAT && op->src[0]->buffer &&
-            op->src[0]->buffer->buft == ggml_backend_amx_buffer_type()) {
-            return (ggml::cpu::tensor_traits *) op->src[0]->extra;
-        }
-
-        return nullptr;
-    }
-};
-}  // namespace ggml::cpu::amx
-
-static size_t ggml_backend_amx_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
-    return ggml_backend_amx_get_alloc_size(tensor);
-
-    GGML_UNUSED(buft);
-}
-
-#define ARCH_GET_XCOMP_PERM     0x1022
-#define ARCH_REQ_XCOMP_PERM     0x1023
-#define XFEATURE_XTILECFG       17
-#define XFEATURE_XTILEDATA      18
-
-static bool ggml_amx_init() {
-#if defined(__gnu_linux__)
-    if (syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA)) {
-        fprintf(stderr, "AMX is not ready to be used!\n");
-        return false;
-    }
-    return true;
-#elif defined(_WIN32)
-    return true;
-#endif
-}
-
-ggml_backend_buffer_type_t ggml_backend_amx_buffer_type() {
-    static struct ggml_backend_buffer_type ggml_backend_buffer_type_amx = {
-        /* .iface = */ {
-                        /* .get_name         = */ ggml_backend_amx_buffer_type_get_name,
-                        /* .alloc_buffer     = */ ggml_backend_amx_buffer_type_alloc_buffer,
-                        /* .get_alignment    = */ ggml_backend_amx_buffer_type_get_alignment,
-                        /* .get_max_size     = */ nullptr,  // defaults to SIZE_MAX
-                        /* .get_alloc_size   = */ ggml_backend_amx_buffer_type_get_alloc_size,
-                        /* .is_host          = */ nullptr,
-                        },
-        /* .device  = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),
-        /* .context = */ new ggml::cpu::amx::extra_buffer_type(),
-    };
-
-    if (!ggml_amx_init()) {
-        return nullptr;
-    }
-
-    return &ggml_backend_buffer_type_amx;
-}
-
-#endif  // defined(__AMX_INT8__) && defined(__AVX512VNNI__)

ggml/src/ggml-cpu/amx/amx.h

@@ -1,8 +0,0 @@
-#include "ggml-backend.h"
-#include "ggml-cpu-impl.h"
-
-// GGML internal header
-
-#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)
-ggml_backend_buffer_type_t ggml_backend_amx_buffer_type(void);
-#endif

ggml/src/ggml-cpu/amx/common.h

@@ -1,91 +0,0 @@
-#pragma once
-
-#include "ggml.h"
-#include "ggml-cpu-impl.h"
-
-#include <algorithm>
-#include <memory>
-#include <type_traits>
-
-#if defined(GGML_USE_OPENMP)
-#include <omp.h>
-#endif
-
-#define TILE_M 16
-#define TILE_N 16
-#define TILE_K 32
-#define VNNI_BLK 4
-
-#define AMX_BLK_SIZE 32
-
-#define TMM0 0
-#define TMM1 1
-#define TMM2 2
-#define TMM3 3
-#define TMM4 4
-#define TMM5 5
-#define TMM6 6
-#define TMM7 7
-
-// parallel routines
-template <typename T, typename std::enable_if<std::is_integral<T>::value, int>::type = 0>
-inline T div_up(T x, T y) { return (x + y - 1) / y; }
-
-template <typename T>
-inline void balance211(T n, T nth, T ith, T& n_start, T& n_end) {
-#if 0
-    // onednn partition pattern
-    T& n_my = n_end;
-    if (nth <= 1 || n == 0) {
-        n_start = 0;
-        n_my = n;
-    } else {
-        T n1 = div_up(n, nth);
-        T n2 = n1 - 1;
-        T T1 = n - n2 * nth;
-        n_my = ith < T1 ? n1 : n2;
-        n_start = ith <= T1 ? ith*n1 : T1 * n1 + (ith - T1) * n2;
-    }
-    n_end += n_start;
-#else
-    // pytorch aten partition pattern
-    T n_my = div_up(n, nth);
-    n_start = ith * n_my;
-    n_end = std::min(n_start + n_my, n);
-#endif
-}
-
-template <typename func_t>
-inline void parallel_for(int n, const func_t& f) {
-#if defined(GGML_USE_OPENMP)
-#pragma omp parallel
-{
-    int nth = omp_get_num_threads();
-    int ith = omp_get_thread_num();
-    int tbegin, tend;
-    balance211(n, nth, ith, tbegin, tend);
-    f(tbegin, tend);
-}
-#else
-    f(0, n);
-#endif
-}
-
-template <typename func_t>
-inline void parallel_for_ggml(const ggml_compute_params * params, int n, const func_t & f) {
-    int tbegin, tend;
-    balance211(n, params->nth, params->ith, tbegin, tend);
-    f(tbegin, tend);
-}
-
-// quantized types that have AMX support
-inline bool qtype_has_amx_kernels(const enum ggml_type type) {
-    // TODO: fix padding for vnni format
-    return (type == GGML_TYPE_Q4_0) ||
-        (type == GGML_TYPE_Q4_1) ||
-        (type == GGML_TYPE_Q8_0) ||
-        (type == GGML_TYPE_Q4_K) ||
-        (type == GGML_TYPE_Q5_K) ||
-        (type == GGML_TYPE_Q6_K) ||
-        (type == GGML_TYPE_IQ4_XS);
-}

ggml/src/ggml-cpu/amx/mmq.h

@@ -1,10 +0,0 @@
-#pragma once
-#include "common.h"
-
-size_t ggml_backend_amx_desired_wsize(const struct ggml_tensor * dst);
-
-size_t ggml_backend_amx_get_alloc_size(const struct ggml_tensor * tensor);
-
-void ggml_backend_amx_convert_weight(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
-
-void ggml_backend_amx_mul_mat(const struct ggml_compute_params * params, struct ggml_tensor * dst);

ggml/src/ggml-cpu/cpu-feats-x86.cpp

@@ -1,323 +0,0 @@
-#include "ggml-backend-impl.h"
-
-#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))
-
-#ifdef _MSC_VER
-#include <intrin.h>
-#endif
-
-#include <cstring>
-#include <vector>
-#include <bitset>
-#include <array>
-#include <string>
-
-// ref: https://cdrdv2-public.intel.com/782156/325383-sdm-vol-2abcd.pdf
-struct cpuid_x86 {
-    bool SSE3(void) { return f_1_ecx[0]; }
-    bool PCLMULQDQ(void) { return f_1_ecx[1]; }
-    bool MONITOR(void) { return f_1_ecx[3]; }
-    bool SSSE3(void) { return f_1_ecx[9]; }
-    bool FMA(void) { return f_1_ecx[12]; }
-    bool CMPXCHG16B(void) { return f_1_ecx[13]; }
-    bool SSE41(void) { return f_1_ecx[19]; }
-    bool SSE42(void) { return f_1_ecx[20]; }
-    bool MOVBE(void) { return f_1_ecx[22]; }
-    bool POPCNT(void) { return f_1_ecx[23]; }
-    bool AES(void) { return f_1_ecx[25]; }
-    bool XSAVE(void) { return f_1_ecx[26]; }
-    bool OSXSAVE(void) { return f_1_ecx[27]; }
-    bool AVX(void) { return f_1_ecx[28]; }
-    bool F16C(void) { return f_1_ecx[29]; }
-    bool RDRAND(void) { return f_1_ecx[30]; }
-
-    bool MSR(void) { return f_1_edx[5]; }
-    bool CX8(void) { return f_1_edx[8]; }
-    bool SEP(void) { return f_1_edx[11]; }
-    bool CMOV(void) { return f_1_edx[15]; }
-    bool CLFSH(void) { return f_1_edx[19]; }
-    bool MMX(void) { return f_1_edx[23]; }
-    bool FXSR(void) { return f_1_edx[24]; }
-    bool SSE(void) { return f_1_edx[25]; }
-    bool SSE2(void) { return f_1_edx[26]; }
-
-    bool FSGSBASE(void) { return f_7_ebx[0]; }
-    bool BMI1(void) { return f_7_ebx[3]; }
-    bool HLE(void) { return is_intel && f_7_ebx[4]; }
-    bool AVX2(void) { return f_7_ebx[5]; }
-    bool BMI2(void) { return f_7_ebx[8]; }
-    bool ERMS(void) { return f_7_ebx[9]; }
-    bool INVPCID(void) { return f_7_ebx[10]; }
-    bool RTM(void) { return is_intel && f_7_ebx[11]; }
-    bool AVX512F(void) { return f_7_ebx[16]; }
-    bool AVX512DQ(void) { return f_7_ebx[17]; }
-    bool RDSEED(void) { return f_7_ebx[18]; }
-    bool ADX(void) { return f_7_ebx[19]; }
-    bool AVX512PF(void) { return f_7_ebx[26]; }
-    bool AVX512ER(void) { return f_7_ebx[27]; }
-    bool AVX512CD(void) { return f_7_ebx[28]; }
-    bool AVX512BW(void) { return f_7_ebx[30]; }
-    bool AVX512VL(void) { return f_7_ebx[31]; }
-
-    bool SHA(void) { return f_7_ebx[29]; }
-
-    bool PREFETCHWT1(void) { return f_7_ecx[0]; }
-
-    bool LAHF(void) { return f_81_ecx[0]; }
-    bool LZCNT(void) { return is_intel && f_81_ecx[5]; }
-    bool ABM(void) { return is_amd && f_81_ecx[5]; }
-    bool SSE4a(void) { return is_amd && f_81_ecx[6]; }
-    bool XOP(void) { return is_amd && f_81_ecx[11]; }
-    bool TBM(void) { return is_amd && f_81_ecx[21]; }
-
-    bool SYSCALL(void) { return is_intel && f_81_edx[11]; }
-    bool MMXEXT(void) { return is_amd && f_81_edx[22]; }
-    bool RDTSCP(void) { return is_intel && f_81_edx[27]; }
-    bool _3DNOWEXT(void) { return is_amd && f_81_edx[30]; }
-    bool _3DNOW(void) { return is_amd && f_81_edx[31]; }
-
-    bool AVX512_VBMI(void) { return f_7_ecx[1]; }
-    bool AVX512_VNNI(void) { return f_7_ecx[11]; }
-    bool AVX512_FP16(void) { return f_7_edx[23]; }
-    bool AVX512_BF16(void) { return f_7_1_eax[5]; }
-    bool AVX_VNNI(void) { return f_7_1_eax[4]; }
-
-    bool AMX_TILE(void) { return f_7_edx[24]; }
-    bool AMX_INT8(void) { return f_7_edx[25]; }
-    bool AMX_FP16(void) { return f_7_1_eax[21]; }
-    bool AMX_BF16(void) { return f_7_edx[22]; }
-
-#ifdef _MSC_VER
-    static void cpuid(int cpu_info[4], int eax) {
-        __cpuid(cpu_info, eax);
-    }
-    static void cpuidex(int cpu_info[4], int eax, int ecx) {
-        __cpuidex(cpu_info, eax, ecx);
-    }
-#else
-    static void cpuid(int cpu_info[4], int eax) {
-        __asm__ __volatile__(
-            "cpuid"
-            : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3])
-            : "a"(eax), "c"(0));
-    }
-    static void cpuidex(int cpu_info[4], int eax, int ecx) {
-        __asm__ __volatile__(
-            "cpuid"
-            : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3])
-            : "a"(eax), "c"(ecx));
-    }
-#endif
-
-    cpuid_x86() {
-        std::array<int, 4> cpui;
-        std::vector<std::array<int, 4>> data;
-
-        // calling __cpuid with 0x0 as the function_id argument
-        // gets the number of the highest valid function ID.
-        cpuid(cpui.data(), 0);
-        int n_ids = cpui[0];
-
-        for (int i = 0; i <= n_ids; ++i) {
-            cpuidex(cpui.data(), i, 0);
-            data.push_back(cpui);
-        }
-
-        // capture vendor string
-        char vendor[0x20] = {};
-        *reinterpret_cast<int *>(vendor)     = data[0][1];
-        *reinterpret_cast<int *>(vendor + 4) = data[0][3];
-        *reinterpret_cast<int *>(vendor + 8) = data[0][2];
-        this->vendor = vendor;
-        if (this->vendor == "GenuineIntel") {
-            is_intel = true;
-        } else if (this->vendor == "AuthenticAMD") {
-            is_amd = true;
-        }
-
-        // load bitset with flags for function 0x00000001
-        if (n_ids >= 1) {
-            f_1_ecx = data[1][2];
-            f_1_edx = data[1][3];
-        }
-
-        // load bitset with flags for function 0x00000007
-        if (n_ids >= 7) {
-            f_7_ebx = data[7][1];
-            f_7_ecx = data[7][2];
-            f_7_edx = data[7][3];
-            cpuidex(cpui.data(), 7, 1);
-            f_7_1_eax = cpui[0];
-        }
-
-        // calling __cpuid with 0x80000000 as the function_id argument
-        // gets the number of the highest valid extended ID.
-        cpuid(cpui.data(), 0x80000000);
-        unsigned int n_ex_ids = cpui[0];
-
-        std::vector<std::array<int, 4>> ext_data;
-        for (unsigned int i = 0x80000000; i <= n_ex_ids; ++i) {
-            cpuidex(cpui.data(), i, 0);
-            ext_data.push_back(cpui);
-        }
-
-        // load bitset with flags for function 0x80000001
-        if (n_ex_ids >= 0x80000001) {
-            f_81_ecx = ext_data[1][2];
-            f_81_edx = ext_data[1][3];
-        }
-
-        // interpret CPU brand string if reported
-        char brand[0x40] = {};
-        if (n_ex_ids >= 0x80000004) {
-            std::memcpy(brand, ext_data[2].data(), sizeof(cpui));
-            std::memcpy(brand + 16, ext_data[3].data(), sizeof(cpui));
-            std::memcpy(brand + 32, ext_data[4].data(), sizeof(cpui));
-            this->brand = brand;
-        }
-    }
-
-    bool is_intel = false;
-    bool is_amd = false;
-    std::string vendor;
-    std::string brand;
-    std::bitset<32> f_1_ecx;
-    std::bitset<32> f_1_edx;
-    std::bitset<32> f_7_ebx;
-    std::bitset<32> f_7_ecx;
-    std::bitset<32> f_7_edx;
-    std::bitset<32> f_7_1_eax;
-    std::bitset<32> f_81_ecx;
-    std::bitset<32> f_81_edx;
-};
-
-#if 0
-void test_x86_is() {
-    cpuid_x86 is;
-    printf("CPU Vendor: %s\n", is.vendor.c_str());
-    printf("Brand: %s\n", is.brand.c_str());
-    printf("is_intel: %d\n", is.is_intel);
-    printf("is_amd: %d\n", is.is_amd);
-    printf("sse3: %d\n", is.SSE3());
-    printf("pclmulqdq: %d\n", is.PCLMULQDQ());
-    printf("ssse3: %d\n", is.SSSE3());
-    printf("fma: %d\n", is.FMA());
-    printf("cmpxchg16b: %d\n", is.CMPXCHG16B());
-    printf("sse41: %d\n", is.SSE41());
-    printf("sse42: %d\n", is.SSE42());
-    printf("movbe: %d\n", is.MOVBE());
-    printf("popcnt: %d\n", is.POPCNT());
-    printf("aes: %d\n", is.AES());
-    printf("xsave: %d\n", is.XSAVE());
-    printf("osxsave: %d\n", is.OSXSAVE());
-    printf("avx: %d\n", is.AVX());
-    printf("f16c: %d\n", is.F16C());
-    printf("rdrand: %d\n", is.RDRAND());
-    printf("msr: %d\n", is.MSR());
-    printf("cx8: %d\n", is.CX8());
-    printf("sep: %d\n", is.SEP());
-    printf("cmov: %d\n", is.CMOV());
-    printf("clflush: %d\n", is.CLFSH());
-    printf("mmx: %d\n", is.MMX());
-    printf("fxsr: %d\n", is.FXSR());
-    printf("sse: %d\n", is.SSE());
-    printf("sse2: %d\n", is.SSE2());
-    printf("fsgsbase: %d\n", is.FSGSBASE());
-    printf("bmi1: %d\n", is.BMI1());
-    printf("hle: %d\n", is.HLE());
-    printf("avx2: %d\n", is.AVX2());
-    printf("bmi2: %d\n", is.BMI2());
-    printf("erms: %d\n", is.ERMS());
-    printf("invpcid: %d\n", is.INVPCID());
-    printf("rtm: %d\n", is.RTM());
-    printf("avx512f: %d\n", is.AVX512F());
-    printf("rdseed: %d\n", is.RDSEED());
-    printf("adx: %d\n", is.ADX());
-    printf("avx512pf: %d\n", is.AVX512PF());
-    printf("avx512er: %d\n", is.AVX512ER());
-    printf("avx512cd: %d\n", is.AVX512CD());
-    printf("sha: %d\n", is.SHA());
-    printf("prefetchwt1: %d\n", is.PREFETCHWT1());
-    printf("lahf: %d\n", is.LAHF());
-    printf("lzcnt: %d\n", is.LZCNT());
-    printf("abm: %d\n", is.ABM());
-    printf("sse4a: %d\n", is.SSE4a());
-    printf("xop: %d\n", is.XOP());
-    printf("tbm: %d\n", is.TBM());
-    printf("syscall: %d\n", is.SYSCALL());
-    printf("mmxext: %d\n", is.MMXEXT());
-    printf("rdtscp: %d\n", is.RDTSCP());
-    printf("3dnowext: %d\n", is._3DNOWEXT());
-    printf("3dnow: %d\n", is._3DNOW());
-    printf("avx512_vbmi: %d\n", is.AVX512_VBMI());
-    printf("avx512_vnni: %d\n", is.AVX512_VNNI());
-    printf("avx512_fp16: %d\n", is.AVX512_FP16());
-    printf("avx512_bf16: %d\n", is.AVX512_BF16());
-    printf("amx_tile: %d\n", is.AMX_TILE());
-    printf("amx_int8: %d\n", is.AMX_INT8());
-    printf("amx_fp16: %d\n", is.AMX_FP16());
-    printf("amx_bf16: %d\n", is.AMX_BF16());
-}
-#endif
-
-static int ggml_backend_cpu_x86_score() {
-    // FIXME: this does not check for OS support
-
-    int score = 0;
-    cpuid_x86 is;
-
-#ifdef GGML_FMA
-    if (!is.FMA()) { return 0; }
-    score += 1;
-#endif
-#ifdef GGML_F16C
-    if (!is.F16C()) { return 0; }
-    score += 1<<1;
-#endif
-#ifdef GGML_SSE42
-    if (!is.SSE42()) { return 0; }
-    score += 1<<2;
-#endif
-#ifdef GGML_AVX
-    if (!is.AVX()) { return 0; }
-    score += 1<<4;
-#endif
-#ifdef GGML_AVX2
-    if (!is.AVX2()) { return 0; }
-    score += 1<<5;
-#endif
-#ifdef GGML_AVX_VNNI
-    if (!is.AVX_VNNI()) { return 0; }
-    score += 1<<6;
-#endif
-#ifdef GGML_AVX512
-    if (!is.AVX512F()) { return 0; }
-    if (!is.AVX512CD()) { return 0; }
-    if (!is.AVX512VL()) { return 0; }
-    if (!is.AVX512DQ()) { return 0; }
-    if (!is.AVX512BW()) { return 0; }
-    score += 1<<7;
-#endif
-#ifdef GGML_AVX512_VBMI
-    if (!is.AVX512_VBMI()) { return 0; }
-    score += 1<<8;
-#endif
-#ifdef GGML_AVX512_BF16
-    if (!is.AVX512_BF16()) { return 0; }
-    score += 1<<9;
-#endif
-#ifdef GGML_AVX512_VNNI
-    if (!is.AVX512_VNNI()) { return 0; }
-    score += 1<<10;
-#endif
-#ifdef GGML_AMX_INT8
-    if (!is.AMX_INT8()) { return 0; }
-    score += 1<<11;
-#endif
-
-    return score;
-}
-
-GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_x86_score)
-
-#endif // defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))

ggml/src/ggml-cpu/ggml-cpu-aarch64.h

@@ -1,8 +1,30 @@
 #pragma once
 
-#include "ggml-cpu-traits.h"
 #include "ggml.h"
 
 // GGML internal header
 
-ggml_backend_buffer_type_t ggml_backend_cpu_aarch64_buffer_type(void);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Quantization
+void quantize_mat_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t nrows, int64_t n_per_row, int64_t blck_size_interleave);
+
+// GEMV
+void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+
+// GEMM
+void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);
+
+void           ggml_aarch64_repack_tensor(struct ggml_tensor * cur, enum ggml_type repack_type, const void * data, size_t data_size);
+enum ggml_type ggml_aarch64_get_optimal_repack_type(const struct ggml_tensor * cur);
+
+#ifdef __cplusplus
+}
+#endif
+

ggml/src/ggml-cpu/ggml-cpu-hbm.cpp

@@ -1,55 +0,0 @@
-#ifdef GGML_USE_CPU_HBM
-
-#include "ggml-backend.h"
-#include "ggml-backend-impl.h"
-#include "ggml-cpu.h"
-#include "ggml-impl.h"
-
-#include "ggml-cpu-hbm.h"
-
-// buffer type HBM
-
-#include <hbwmalloc.h>
-
-static const char * ggml_backend_cpu_hbm_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
-    return "CPU_HBM";
-
-    GGML_UNUSED(buft);
-}
-
-static void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) {
-    hbw_free(buffer->context);
-}
-
-static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,
-                                                                           size_t                     size) {
-    void * ptr;
-    int    result = hbw_posix_memalign(&ptr, ggml_backend_cpu_buffer_type_get_alignment(buft), size);
-    if (result != 0) {
-        GGML_LOG_ERROR("failed to allocate HBM buffer of size %zu\n", size);
-        return NULL;
-    }
-
-    ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
-    buffer->buft                 = buft;
-    buffer->iface.free_buffer    = ggml_backend_cpu_hbm_buffer_free_buffer;
-
-    return buffer;
-}
-
-ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) {
-    static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_hbm = {
-        /* .iface    = */ {
-                           /* .get_name         = */ ggml_backend_cpu_hbm_buffer_type_get_name,
-                           /* .alloc_buffer     = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer,
-                           /* .get_alignment    = */ ggml_backend_cpu_buffer_type_get_alignment,
-                           /* .get_max_size     = */ nullptr,  // defaults to SIZE_MAX
-                           /* .get_alloc_size   = */ nullptr,  // defaults to ggml_nbytes
-                           /* .is_host          = */ ggml_backend_cpu_buffer_type_is_host,
-                           },
-        /* .context  = */ nullptr,
-    };
-
-    return &ggml_backend_cpu_buffer_type_hbm;
-}
-#endif

ggml/src/ggml-cpu/ggml-cpu-hbm.h

@@ -1,8 +0,0 @@
-#pragma once
-
-#include "ggml-backend.h"
-#include "ggml.h"
-
-// GGML CPU internal header
-
-ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);

ggml/src/ggml-cpu/ggml-cpu-impl.h

@@ -15,18 +15,6 @@
 extern "C" {
 #endif
 
-struct ggml_compute_params {
-    // ith = thread index, nth = number of threads
-    int ith, nth;
-
-    // work buffer for all threads
-    size_t wsize;
-    void * wdata;
-
-    struct ggml_threadpool * threadpool;
-};
-
-
 #if defined(_MSC_VER)
 
 #define m512bh(p) p
@@ -378,9 +366,6 @@ static __m256 __lasx_xvreplfr2vr_s(float val) {
 }
 #endif
 
-// TODO: move to ggml-threading
-void ggml_barrier(struct ggml_threadpool * tp);
-
 #ifdef __cplusplus
 }
 #endif

ggml/src/ggml-cpu/ggml-cpu-quants.c

@@ -1791,12 +1791,11 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r
             const int8x16_t y1_l = vld1q_s8(b_y1->qs);
             const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
 
-            float32_t _scale[4] = {
-                GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
-                GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
-                GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
-                GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)
-            };
+            float32_t _scale[4] = { GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
+                                    GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
+                                    GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
+                                    GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};
+
             float32x4_t scale = vld1q_f32(_scale);
 
             int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
@@ -1812,15 +1811,13 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r
             int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
 
             sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
-                                                l1, r1)), l2, r2)), l3, r3))), scale);
+                                                                                l1, r1)), l2, r2)), l3, r3))), scale);
         }
-
-        float32x4_t sumv1 = vextq_f32 (sumv0, sumv0, 2);
+        float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
         float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
 
-        vst1_f32(s,      vget_low_f32 (sumv2));
+        vst1_f32(s,      vget_low_f32(sumv2));
         vst1_f32(s + bs, vget_high_f32(sumv2));
-
         return;
     }
 #endif
@@ -2348,12 +2345,10 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
             const block_q8_1 * restrict b_y0 = &vy0[i];
             const block_q8_1 * restrict b_y1 = &vy1[i];
 
-            float32_t summs_t[4] = {
-                GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),
-                GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),
-                GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),
-                GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)
-            };
+            float32_t summs_t[4] = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),
+                                    GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),
+                                    GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),
+                                    GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};
             summs0 = vaddq_f32(summs0, vld1q_f32(summs_t));
 
             const uint8x16_t m4b = vdupq_n_u8(0x0F);
@@ -2374,12 +2369,10 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
             const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
 
             // mmla into int32x4_t
-            float32_t _scale[4] = {
-                GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
-                GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
-                GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
-                GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)
-            };
+            float32_t _scale[4] = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,
+                                   GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,
+                                   GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,
+                                   GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};
             float32x4_t scale = vld1q_f32(_scale);
 
             int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
@@ -2394,17 +2387,15 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r
             int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
             int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
             sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
-                                                l1, r1)), l2, r2)), l3, r3))), scale);
+                                                                                l1, r1)), l2, r2)), l3, r3))), scale);
         }
 
-        float32x4_t sumv1 = vextq_f32 (sumv0, sumv0, 2);
+        float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
         float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
-
         sumv2 = vaddq_f32(sumv2, summs0);
 
         vst1_f32(s,      vget_low_f32 (sumv2));
         vst1_f32(s + bs, vget_high_f32(sumv2));
-
         return;
     }
 #endif
@@ -3381,12 +3372,10 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r
             const int8x16_t y1_l = vld1q_s8(b_y1->qs);
             const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);
 
-            float32_t _scale[4] = {
-                GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
-                GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
-                GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
-                GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)
-            };
+            float32_t _scale[4] = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),
+                                   GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),
+                                   GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),
+                                   GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};
             float32x4_t scale = vld1q_f32(_scale);
 
             int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));
@@ -3402,15 +3391,13 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r
             int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));
 
             sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),
-                                                l1, r1)), l2, r2)), l3, r3))), scale);
+                                                                                       l1, r1)), l2, r2)), l3, r3))), scale);
         }
-
-        float32x4_t sumv1 = vextq_f32 (sumv0, sumv0, 2);
+        float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);
         float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);
 
-        vst1_f32(s,      vget_low_f32 (sumv2));
+        vst1_f32(s, vget_low_f32(sumv2));
         vst1_f32(s + bs, vget_high_f32(sumv2));
-
         return;
     }
 #endif

ggml/src/ggml-cpu/ggml-cpu-traits.cpp

@@ -1,36 +0,0 @@
-#include "ggml-cpu-traits.h"
-
-#include "ggml-backend-impl.h"
-#include "ggml-backend.h"
-
-namespace ggml::cpu {
-tensor_traits::~tensor_traits() {}
-
-extra_buffer_type::~extra_buffer_type() {}
-}  // namespace ggml::cpu
-
-bool ggml_cpu_extra_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) {
-    for (auto extra : ggml_backend_cpu_get_extra_buffers_type()) {
-        if (extra && extra->context) {
-            auto buf_extra     = (ggml::cpu::extra_buffer_type *) extra->context;
-            auto tensor_traits = buf_extra->get_tensor_traits(op);
-            if (tensor_traits && tensor_traits->compute_forward(params, op)) {
-                return true;
-            }
-        }
-    }
-    return false;
-}
-
-bool ggml_cpu_extra_work_size(int n_threads, const struct ggml_tensor * op, size_t * size) {
-    for (auto extra : ggml_backend_cpu_get_extra_buffers_type()) {
-        if (extra && extra->context) {
-            auto buf_extra     = (ggml::cpu::extra_buffer_type *) extra->context;
-            auto tensor_traits = buf_extra->get_tensor_traits(op);
-            if (tensor_traits && tensor_traits->work_size(n_threads, op, *size)) {
-                return true;
-            }
-        }
-    }
-    return false;
-}

ggml/src/ggml-cpu/ggml-cpu-traits.h

@@ -1,38 +0,0 @@
-#pragma once
-#include "ggml-backend-impl.h"
-#include "ggml-cpu-impl.h"
-#include "ggml.h"
-
-#ifdef __cplusplus
-#    include <vector>
-extern "C" {
-#endif
-
-// return true if op part of extra "accelerator"
-bool ggml_cpu_extra_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op);
-bool ggml_cpu_extra_work_size(int n_threads, const struct ggml_tensor * op, size_t * size);
-
-#ifdef __cplusplus
-}
-
-namespace ggml::cpu {
-// register in tensor->extra
-class tensor_traits {
-  public:
-    virtual ~tensor_traits();
-    virtual bool work_size(int n_threads, const struct ggml_tensor * op, size_t & size)        = 0;
-    virtual bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) = 0;
-};
-
-class extra_buffer_type {
-  public:
-    virtual ~extra_buffer_type();
-    virtual bool            supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) = 0;
-    virtual tensor_traits * get_tensor_traits(const struct ggml_tensor * op)                   = 0;
-};
-}  // namespace ggml::cpu
-
-// implemented in ggml-cpu.cpp.
-std::vector<ggml_backend_buffer_type_t> & ggml_backend_cpu_get_extra_buffers_type();
-
-#endif

ggml/src/ggml-cpu/ggml-cpu.cpp

@@ -2,18 +2,11 @@
 #include "ggml-backend-impl.h"
 #include "ggml-cpu.h"
 #include "ggml-cpu-aarch64.h"
-#include "ggml-cpu-traits.h"
 #include "ggml-impl.h"
-#include "amx/amx.h"
-
 #include <cctype>
 #include <string>
 #include <vector>
 
-#ifdef GGML_USE_CPU_HBM
-#include "ggml-cpu-hbm.h"
-#endif
-
 #if defined(__APPLE__)
 #include <sys/types.h>
 #include <sys/sysctl.h>
@@ -29,20 +22,124 @@
 
 // ggml-backend interface
 
-std::vector<ggml_backend_buffer_type_t>& ggml_backend_cpu_get_extra_buffers_type() {
+#ifdef GGML_USE_CPU_HBM
+
+// buffer type HBM
+
+#include <hbwmalloc.h>
+
+static const char * ggml_backend_cpu_hbm_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
+    return "CPU_HBM";
+
+    GGML_UNUSED(buft);
+}
+
+static void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+    hbw_free(buffer->context);
+}
+
+static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+    void * ptr;
+    int result = hbw_posix_memalign(&ptr, ggml_backend_cpu_buffer_type_get_alignment(buft), size);
+    if (result != 0) {
+        GGML_LOG_ERROR("failed to allocate HBM buffer of size %zu\n", size);
+        return NULL;
+    }
+
+    ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
+    buffer->buft = buft;
+    buffer->iface.free_buffer = ggml_backend_cpu_hbm_buffer_free_buffer;
+
+    return buffer;
+}
+
+ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) {
+    static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_hbm = {
+        /* .iface    = */ {
+            /* .get_name         = */ ggml_backend_cpu_hbm_buffer_type_get_name,
+            /* .alloc_buffer     = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer,
+            /* .get_alignment    = */ ggml_backend_cpu_buffer_type_get_alignment,
+            /* .get_max_size     = */ NULL, // defaults to SIZE_MAX
+            /* .get_alloc_size   = */ NULL, // defaults to ggml_nbytes
+            /* .is_host          = */ ggml_backend_cpu_buffer_type_is_host,
+        },
+        /* .context  = */ NULL,
+    };
+
+    return &ggml_backend_cpu_buffer_type_hbm;
+}
+#endif
+
+// buffer type AARCH64
+
+static void ggml_backend_cpu_aarch64_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {
+    tensor->extra = (void *)ggml_aarch64_get_optimal_repack_type(tensor); // NOLINT
+
+    GGML_UNUSED(buffer);
+}
+
+static void ggml_backend_cpu_aarch64_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+    GGML_ASSERT(offset == 0);
+    GGML_ASSERT(size == ggml_nbytes(tensor));
+
+    enum ggml_type repack_type = (enum ggml_type)(intptr_t)tensor->extra;
+
+    ggml_aarch64_repack_tensor(tensor, repack_type, data, size);
+
+    GGML_UNUSED(buffer);
+}
+
+static const char * ggml_backend_cpu_aarch64_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
+    return "CPU_AARCH64";
+
+    GGML_UNUSED(buft);
+}
+
+static ggml_backend_buffer_t ggml_backend_cpu_aarch64_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+    auto * buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);
+
+    if (buffer == NULL) {
+        return NULL;
+    }
+
+    buffer->buft = buft;
+    buffer->iface.init_tensor = ggml_backend_cpu_aarch64_buffer_init_tensor;
+    buffer->iface.set_tensor = ggml_backend_cpu_aarch64_buffer_set_tensor;
+
+    return buffer;
+}
+
+ggml_backend_buffer_type_t ggml_backend_cpu_aarch64_buffer_type(void) {
+    static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_aarch64 = {
+        /* .iface    = */ {
+            /* .get_name         = */ ggml_backend_cpu_aarch64_buffer_type_get_name,
+            /* .alloc_buffer     = */ ggml_backend_cpu_aarch64_buffer_type_alloc_buffer,
+            /* .get_alignment    = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,
+            /* .get_max_size     = */ NULL, // defaults to SIZE_MAX
+            /* .get_alloc_size   = */ NULL, // defaults to ggml_nbytes
+            /* .is_host          = */ NULL,
+        },
+        /* .device  = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),
+        /* .context = */ NULL,
+    };
+
+    return &ggml_backend_cpu_buffer_type_aarch64;
+}
+
+bool ggml_backend_cpu_buft_is_aarch64(ggml_backend_buffer_type_t buft) {
+    return buft == ggml_backend_cpu_aarch64_buffer_type();
+}
+
+static ggml_backend_buffer_type_t * ggml_backend_cpu_get_extra_bufts(ggml_backend_dev_t device) {
     static std::vector<ggml_backend_buffer_type_t> bufts = []() {
         std::vector<ggml_backend_buffer_type_t> bufts;
 
-#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)
-        if (ggml_backend_amx_buffer_type()) {
-            bufts.push_back(ggml_backend_amx_buffer_type());
-        }
+#ifdef GGML_USE_CPU_HBM
+        bufts.push_back(ggml_backend_cpu_hbm_buffer_type());
 #endif
 
 #ifdef GGML_USE_CPU_AARCH64
-        if (ggml_backend_cpu_aarch64_buffer_type()) {
-            bufts.push_back(ggml_backend_cpu_aarch64_buffer_type());
-        }
+        bufts.push_back(ggml_backend_cpu_aarch64_buffer_type());
 #endif
 
         bufts.push_back(NULL);
@@ -50,22 +147,11 @@ std::vector<ggml_backend_buffer_type_t>& ggml_backend_cpu_get_extra_buffers_type
         return bufts;
     }();
 
-    return bufts;
-}
-
-static ggml_backend_buffer_type_t * ggml_backend_cpu_device_get_extra_buffers_type(ggml_backend_dev_t device) {
-    return ggml_backend_cpu_get_extra_buffers_type().data();
+    return bufts.data();
 
     GGML_UNUSED(device);
 }
 
-static bool ggml_backend_cpu_is_extra_buffer_type(ggml_backend_buffer_type_t buft) {
-    for (auto extra : ggml_backend_cpu_get_extra_buffers_type()) {
-        if (extra && extra == buft) return true;
-    }
-    return false;
-}
-
 // CPU backend - backend (stream)
 
 struct ggml_backend_cpu_context {
@@ -370,23 +456,14 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
     const struct ggml_tensor * src0 = op->src[0];
     const struct ggml_tensor * src1 = op->src[1];
 
-    if (op->op == GGML_OP_NONE || op->op == GGML_OP_RESHAPE || op->op == GGML_OP_VIEW || op->op == GGML_OP_PERMUTE || op->op == GGML_OP_TRANSPOSE) {
-        return true;
-    }
-
-    // extra_buffer_op?
-    for (auto extra : ggml_backend_cpu_get_extra_buffers_type()) {
-        if (extra) {
-            auto buf_extra = (ggml::cpu::extra_buffer_type*) extra->context;
-            if (buf_extra && buf_extra->supports_op(dev, op)) {
-                return true;
-            }
+    if (src0 && src0->buffer && ggml_backend_cpu_buft_is_aarch64(src0->buffer->buft)) {
+        if (op->op != GGML_OP_MUL_MAT || src0->type != GGML_TYPE_Q4_0 || ggml_aarch64_get_optimal_repack_type(src0) == GGML_TYPE_Q4_0) {
+            return false;
         }
     }
 
-    // the other case need host buffer.
-    for (int i = 0; i < GGML_MAX_SRC; i++) {
-        if (op->src[i] && op->src[i]->buffer && !ggml_backend_buft_is_host(op->src[i]->buffer->buft)) {
+    for (int i = 1; i < GGML_MAX_SRC; i++) {
+        if (op->src[i] && op->src[i]->buffer && ggml_backend_cpu_buft_is_aarch64(op->src[i]->buffer->buft)) {
             return false;
         }
     }
@@ -394,11 +471,8 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
     switch (op->op) {
         case GGML_OP_CPY:
             return
-                op->type != GGML_TYPE_IQ3_XXS &&
-                op->type != GGML_TYPE_IQ3_S   &&
                 op->type != GGML_TYPE_IQ2_XXS &&
                 op->type != GGML_TYPE_IQ2_XS  &&
-                op->type != GGML_TYPE_IQ2_S   &&
                 op->type != GGML_TYPE_IQ1_S   &&
                 op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float
         case GGML_OP_MUL_MAT:
@@ -412,10 +486,13 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
         default:
             return true;
     }
+
+    GGML_UNUSED(dev);
 }
 
 static bool ggml_backend_cpu_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
-    return ggml_backend_buft_is_host(buft) || ggml_backend_cpu_is_extra_buffer_type(buft);
+    return ggml_backend_buft_is_host(buft) || ggml_backend_cpu_buft_is_aarch64(buft);
+
     GGML_UNUSED(dev);
 }
 
@@ -464,12 +541,16 @@ static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg
     return &ggml_backend_cpu_device;
 }
 
+struct ggml_backend_feature {
+    const char * name;
+    const char * value;
+};
+
+// Not used yet
 // This is intended to replace the the ggml_cpu_has_* functions when loading the CPU backend dynamically,
-// and additionally to allow other backends to expose their own list of features that applications can query using the same API
+// and additionally to allow other backends to expose their own list of features that applications can query using the same API.
 static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t reg) {
     static std::vector<ggml_backend_feature> features = []() {
-        ggml_cpu_init();
-
         std::vector<ggml_backend_feature> features;
         if (ggml_cpu_has_sse3()) {
             features.push_back({ "SSE3", "1" });
@@ -480,9 +561,6 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r
         if (ggml_cpu_has_avx()) {
             features.push_back({ "AVX", "1" });
         }
-        if (ggml_cpu_has_avx_vnni()) {
-            features.push_back({ "AVX_VNNI", "1" });
-        }
         if (ggml_cpu_has_avx2()) {
             features.push_back({ "AVX2", "1" });
         }
@@ -492,6 +570,9 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r
         if (ggml_cpu_has_fma()) {
             features.push_back({ "FMA", "1" });
         }
+        if (ggml_cpu_has_avx_vnni()) {
+            features.push_back({ "AVX_VNNI", "1" });
+        }
         if (ggml_cpu_has_avx512()) {
             features.push_back({ "AVX512", "1" });
         }
@@ -538,18 +619,6 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r
         if (ggml_cpu_has_llamafile()) {
             features.push_back({ "LLAMAFILE", "1" });
         }
-    #ifdef GGML_USE_ACCELERATE
-        features.push_back({ "ACCELERATE", "1" });
-    #endif
-    #ifdef GGML_USE_CPU_HBM
-        features.push_back({ "CPU_HBM", "1" });
-    #endif
-    #ifdef GGML_USE_OPENMP
-        features.push_back({ "OPENMP", "1" });
-    #endif
-    #ifdef GGML_USE_CPU_AARCH64
-        features.push_back({ "AARCH64_REPACK", "1" });
-    #endif
 
         features.push_back({ nullptr, nullptr });
 
@@ -563,35 +632,10 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r
 
 static void * ggml_backend_cpu_get_proc_address(ggml_backend_reg_t reg, const char * name) {
     if (strcmp(name, "ggml_backend_set_n_threads") == 0) {
-        ggml_backend_set_n_threads_t fct = ggml_backend_cpu_set_n_threads;
-        return (void *)fct;
+        return (void *)ggml_backend_cpu_set_n_threads;
     }
     if (strcmp(name, "ggml_backend_dev_get_extra_bufts") == 0) {
-        ggml_backend_dev_get_extra_bufts_t fct = ggml_backend_cpu_device_get_extra_buffers_type;
-        return (void *)fct;
-    }
-    if (strcmp(name, "ggml_backend_get_features") == 0) {
-        return (void *)ggml_backend_cpu_get_features;
-    }
-    if (strcmp(name, "ggml_backend_set_abort_callback") == 0) {
-        return (void *)ggml_backend_cpu_set_abort_callback;
-    }
-    if (strcmp(name, "ggml_backend_cpu_numa_init") == 0) {
-        return (void *)ggml_numa_init;
-    }
-    if (strcmp(name, "ggml_backend_cpu_is_numa") == 0) {
-        return (void *)ggml_is_numa;
-    }
-
-    // threadpool - TODO:  move to ggml-base
-    if (strcmp(name, "ggml_threadpool_new") == 0) {
-        return (void *)ggml_threadpool_new;
-    }
-    if (strcmp(name, "ggml_threadpool_free") == 0) {
-        return (void *)ggml_threadpool_free;
-    }
-    if (strcmp(name, "ggml_backend_cpu_set_threadpool") == 0) {
-        return (void *)ggml_backend_cpu_set_threadpool;
+        return (void *)ggml_backend_cpu_get_extra_bufts;
     }
 
     return NULL;
@@ -611,12 +655,9 @@ ggml_backend_reg_t ggml_backend_cpu_reg(void) {
     ggml_cpu_init();
 
     static struct ggml_backend_reg ggml_backend_cpu_reg = {
-        /* .api_version = */ GGML_BACKEND_API_VERSION,
-        /* .iface       = */ ggml_backend_cpu_reg_i,
-        /* .context     = */ NULL,
+        /* .iface   = */ ggml_backend_cpu_reg_i,
+        /* .context = */ NULL,
     };
 
     return &ggml_backend_cpu_reg;
 }
-
-GGML_BACKEND_DL_IMPL(ggml_backend_cpu_reg)

ggml/src/ggml-cpu/llamafile/sgemm.cpp

@@ -50,7 +50,8 @@
 
 #include "sgemm.h"
 #include "ggml-impl.h"
-#include "ggml-cpu-impl.h"
+// hack until moved into the CPU backend
+#include "../ggml-cpu-impl.h"
 #include "ggml-quants.h"
 
 #ifdef _MSC_VER

ggml/src/ggml-cuda/CMakeLists.txt

@@ -46,10 +46,13 @@ if (CUDAToolkit_FOUND)
         list(APPEND GGML_SOURCES_CUDA ${SRCS})
     endif()
 
-    ggml_add_backend_library(ggml-cuda
-                             ${GGML_HEADERS_CUDA}
-                             ${GGML_SOURCES_CUDA}
-                            )
+    add_library(ggml-cuda
+                ${GGML_HEADERS_CUDA}
+                ${GGML_SOURCES_CUDA}
+                )
+
+    target_link_libraries(ggml-cuda PRIVATE ggml-base)
+    target_include_directories(ggml-cuda PRIVATE . ..)
 
     add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})
 
@@ -132,7 +135,7 @@ if (CUDAToolkit_FOUND)
 
         message("-- CUDA host compiler is ${CUDA_CCID} ${CUDA_CCVER}")
 
-        ggml_get_flags(${CUDA_CCID} ${CUDA_CCVER})
+        get_flags(${CUDA_CCID} ${CUDA_CCVER})
         list(APPEND CUDA_CXX_FLAGS ${CXX_FLAGS} ${GF_CXX_FLAGS})  # This is passed to -Xcompiler later
     endif()
 

ggml/src/ggml-cuda/argmax.cu

@@ -1,69 +1,57 @@
-#include <algorithm>
-#include <cstdint>
-
-#include "argmax.cuh"
 #include "common.cuh"
+#include "argmax.cuh"
 #include "sum.cuh"
 
-static __global__ void argmax_f32(const float * __restrict__ x, int32_t * __restrict__ dst, const int64_t ncols) {
-    const int64_t row = blockIdx.x;
+#include <cstdint>
 
-    float maxval = -FLT_MAX;
-    int   argmax = -1;
-    const float * rowx = x + row * ncols;
+static __global__ void argmax_f32(
+    const float * x, int32_t * dst, const int64_t ncols, const int64_t nrows) {
 
-    for (int32_t col = threadIdx.x; col < ncols; col += blockDim.x) {
-        const float val = rowx[col];
-        if (val > maxval) {
-            maxval = val;
-            argmax = col;
-        }
-    }
+    int argmax_thread = 0;
+    const int64_t row0 = (int64_t)blockIdx.x*WARP_SIZE;
 
 #pragma unroll
-    for (int offset = 16; offset > 0; offset >>= 1) {
-        const float val = __shfl_xor_sync(0xFFFFFFFF, maxval, offset, WARP_SIZE);
-        const int   col = __shfl_xor_sync(0xFFFFFFFF, argmax, offset, WARP_SIZE);
-        if (val > maxval) {
-            maxval = val;
-            argmax = col;
-        }
-    }
+    for (int64_t row1 = 0; row1 < WARP_SIZE; ++row1) {
+        const int64_t row = row0 + row1;
 
-    const int n_warps = blockDim.x / WARP_SIZE;
-    const int lane_id = threadIdx.x % WARP_SIZE;
-    const int warp_id = threadIdx.x / WARP_SIZE;
-    if (n_warps > 1) {
-        constexpr int    max_warps = 1024 / WARP_SIZE;
-        __shared__ float shared_maxval[max_warps];
-        __shared__ int   shared_argmax[max_warps];
-        if (lane_id == 0) {
-            shared_maxval[warp_id] = maxval;
-            shared_argmax[warp_id] = argmax;
+        if (row >= nrows) {
+            break;
         }
 
-        __syncthreads();
+        float maxval = -FLT_MAX;
+        int   argmax = -1;
+
+        for (int32_t col = threadIdx.x; col < ncols; col += WARP_SIZE) {
+            const float val        = x[row*ncols + col];
+            const int   bigger     = val > maxval;
+            const int   not_bigger = bigger ^ 0x00000001;
+
+            maxval = maxval*not_bigger + val*bigger;
+            argmax = argmax*not_bigger + col*bigger;
+        }
 
-        if (warp_id == 0) {
-            if (lane_id < n_warps) {
-                maxval = shared_maxval[lane_id];
-                argmax = shared_argmax[lane_id];
-            }
 #pragma unroll
-            for (int offset = 16; offset > 0; offset >>= 1) {
-                const float val = __shfl_xor_sync(0xFFFFFFFF, maxval, offset, WARP_SIZE);
-                const int   col = __shfl_xor_sync(0xFFFFFFFF, argmax, offset, WARP_SIZE);
-                if (val > maxval) {
-                    maxval = val;
-                    argmax = col;
-                }
-            }
+        for (int mask = 16; mask > 0; mask >>= 1) {
+            const float val        = __shfl_xor_sync(0xFFFFFFFF, maxval, mask, WARP_SIZE);
+            const int   col        = __shfl_xor_sync(0xFFFFFFFF, argmax, mask, WARP_SIZE);
+            const int   bigger     = val > maxval;
+            const int   not_bigger = bigger ^ 0x00000001;
+
+            maxval = maxval*not_bigger + val*bigger;
+            argmax = argmax*not_bigger + col*bigger;
         }
+
+        const int store = row1 == threadIdx.x;
+        argmax_thread += store*argmax;
     }
 
-    if (warp_id == 0 && lane_id == 0) {
-        dst[row] = argmax;
+    const int row = row0 + threadIdx.x;
+
+    if (row >= nrows) {
+        return;
     }
+
+    dst[row] = argmax_thread;
 }
 
 void ggml_cuda_argmax(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
@@ -82,10 +70,10 @@ void ggml_cuda_argmax(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
 
     cudaStream_t stream = ctx.stream();
 
-    const int64_t num_blocks = nrows;
-    const int64_t num_threads = std::min<int64_t>(1024, (ne00 + WARP_SIZE - 1) / WARP_SIZE * WARP_SIZE);
-    const dim3 blocks_dim(num_threads, 1, 1);
+    const int64_t num_blocks = (nrows + WARP_SIZE - 1) / WARP_SIZE;
+
+    const dim3 blocks_dim(WARP_SIZE, 1, 1);
     const dim3 blocks_num(num_blocks, 1, 1);
 
-    argmax_f32<<<blocks_num, blocks_dim, 0, stream>>>(src0_d, dst_d, ne00);
+    argmax_f32<<<blocks_num, blocks_dim, 0, stream>>>(src0_d, dst_d, ne00, nrows);
 }

ggml/src/ggml-cuda/common.cuh

@@ -41,28 +41,17 @@
 #define CUDART_HMAX   11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed)
 #define CUDART_HMASK  12000 // CUDA 12.0, min. ver. for half2 -> uint mask comparisons
 
-#define GGML_CUDA_CC_PASCAL     600
-#define GGML_CUDA_CC_DP4A       610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products
-#define GGML_CUDA_CC_VOLTA      700
-#define GGML_CUDA_CC_TURING     750
-#define GGML_CUDA_CC_AMPERE     800
-#define GGML_CUDA_CC_OFFSET_AMD 1000000
-
-// GCN/CNDA, wave size is 64
-#define GGML_CUDA_CC_GCN4       (GGML_CUDA_CC_OFFSET_AMD + 803)  // Tonga, Fiji, Polaris, minimum for fast fp16
-#define GGML_CUDA_CC_VEGA       (GGML_CUDA_CC_OFFSET_AMD + 900)  // Vega56/64, minimum for fp16 dual issue
-#define GGML_CUDA_CC_VEGA20     (GGML_CUDA_CC_OFFSET_AMD + 906)  // MI50/Radeon VII, minimum for dp4a
-#define GGML_CUDA_CC_CDNA       (GGML_CUDA_CC_OFFSET_AMD + 908)  // MI100, minimum for MFMA, acc registers
-#define GGML_CUDA_CC_CDNA2      (GGML_CUDA_CC_OFFSET_AMD + 910)  // MI210, minimum acc register renameing
-#define GGML_CUDA_CC_CDNA3      (GGML_CUDA_CC_OFFSET_AMD + 942)  // MI300
-
-// RNDA removes MFMA, dp4a, xnack, acc registers, wave size is 32
-#define GGML_CUDA_CC_RDNA1      (GGML_CUDA_CC_OFFSET_AMD + 1010) // RX 5000
-#define GGML_CUDA_CC_RDNA2      (GGML_CUDA_CC_OFFSET_AMD + 1030) // RX 6000, minimum for dp4a
-#define GGML_CUDA_CC_RDNA3      (GGML_CUDA_CC_OFFSET_AMD + 1100) // RX 7000, minimum for WMMA
-
-#define GGML_CUDA_CC_QY1        210
-#define GGML_CUDA_CC_QY2        220
+#define CC_PASCAL     600
+#define MIN_CC_DP4A   610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products
+#define CC_VOLTA      700
+#define CC_TURING     750
+#define CC_AMPERE     800
+#define CC_OFFSET_AMD 1000000
+#define CC_RDNA1      (CC_OFFSET_AMD + 1010)
+#define CC_RDNA2      (CC_OFFSET_AMD + 1030)
+#define CC_RDNA3      (CC_OFFSET_AMD + 1100)
+#define CC_QY1        210
+#define CC_QY2        220
 
 #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses
 
@@ -131,36 +120,36 @@ typedef float dfloat; // dequantize float
 typedef float2 dfloat2;
 #endif // GGML_CUDA_F16
 
-#if (defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL
+#if (defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ >= CC_PASCAL
 #define FP16_AVAILABLE
-#endif // (defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL
+#endif // (defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ >= CC_PASCAL
 
 #if defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610
 #define FAST_FP16_AVAILABLE
 #endif // defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610
 
-#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
+#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_VOLTA
 #define FP16_MMA_AVAILABLE
-#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
+#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_VOLTA
 
-#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING
+#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_TURING
 #define INT8_MMA_AVAILABLE
-#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING
+#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_TURING
 
-#if !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ <= GGML_CUDA_CC_QY1)
+#if !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ <= CC_QY1)
 #define FLASH_ATTN_AVAILABLE
-#endif // !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ <= GGML_CUDA_CC_QY1)
+#endif // !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ <= CC_QY1)
 
 static constexpr bool fast_fp16_available(const int cc) {
-    return cc >= GGML_CUDA_CC_PASCAL && cc != 610;
+    return cc >= CC_PASCAL && cc != 610;
 }
 
 static constexpr bool fp16_mma_available(const int cc) {
-    return cc < GGML_CUDA_CC_OFFSET_AMD && cc >= GGML_CUDA_CC_VOLTA;
+    return cc < CC_OFFSET_AMD && cc >= CC_VOLTA;
 }
 
 static constexpr bool int8_mma_available(const int cc) {
-    return cc < GGML_CUDA_CC_OFFSET_AMD && cc >= GGML_CUDA_CC_TURING;
+    return cc < CC_OFFSET_AMD && cc >= CC_TURING;
 }
 
 [[noreturn]]
@@ -187,30 +176,30 @@ static __device__ void no_device_code(
 #endif // __CUDA_ARCH__
 
 static __device__ __forceinline__ int warp_reduce_sum(int x) {
-#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_AMPERE
     return __reduce_add_sync(0xffffffff, x);
 #else
 #pragma unroll
-    for (int offset = 16; offset > 0; offset >>= 1) {
-        x += __shfl_xor_sync(0xffffffff, x, offset, 32);
+    for (int mask = 16; mask > 0; mask >>= 1) {
+        x += __shfl_xor_sync(0xffffffff, x, mask, 32);
     }
     return x;
-#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
+#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_AMPERE
 }
 
 static __device__ __forceinline__ float warp_reduce_sum(float x) {
 #pragma unroll
-    for (int offset = 16; offset > 0; offset >>= 1) {
-        x += __shfl_xor_sync(0xffffffff, x, offset, 32);
+    for (int mask = 16; mask > 0; mask >>= 1) {
+        x += __shfl_xor_sync(0xffffffff, x, mask, 32);
     }
     return x;
 }
 
 static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {
 #pragma unroll
-    for (int offset = 16; offset > 0; offset >>= 1) {
-        a.x += __shfl_xor_sync(0xffffffff, a.x, offset, 32);
-        a.y += __shfl_xor_sync(0xffffffff, a.y, offset, 32);
+    for (int mask = 16; mask > 0; mask >>= 1) {
+        a.x += __shfl_xor_sync(0xffffffff, a.x, mask, 32);
+        a.y += __shfl_xor_sync(0xffffffff, a.y, mask, 32);
     }
     return a;
 }
@@ -220,16 +209,16 @@ static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {
 
 #if defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)
 #pragma unroll
-    for (int offset = 16; offset > 0; offset >>= 1) {
-        const half2 a_other = __shfl_xor_sync(0xffffffff, a, offset, 32);
+    for (int mask = 16; mask > 0; mask >>= 1) {
+        const half2 a_other = __shfl_xor_sync(0xffffffff, a, mask, 32);
         reinterpret_cast<half&>(a.x) +=  __low2half(a_other);
         reinterpret_cast<half&>(a.y) += __high2half(a_other);
     }
     return a;
 #else
 #pragma unroll
-    for (int offset = 16; offset > 0; offset >>= 1) {
-        a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, offset, 32));
+    for (int mask = 16; mask > 0; mask >>= 1) {
+        a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32));
     }
     return a;
 #endif // defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)
@@ -242,8 +231,8 @@ static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {
 
 static __device__ __forceinline__ float warp_reduce_max(float x) {
 #pragma unroll
-    for (int offset = 16; offset > 0; offset >>= 1) {
-        x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, offset, 32));
+    for (int mask = 16; mask > 0; mask >>= 1) {
+        x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, mask, 32));
     }
     return x;
 }
@@ -284,16 +273,16 @@ static __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const hal
 }
 
 static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {
-#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL
+#if !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL
 #pragma unroll
-   for (int offset = 16; offset > 0; offset >>= 1) {
-       x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, offset, 32));
+   for (int mask = 16; mask > 0; mask >>= 1) {
+       x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));
    }
    return x;
 #else
    GGML_UNUSED(x);
    NO_DEVICE_CODE;
-#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL
+#endif // !(defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL
 }
 
 #if CUDART_VERSION < CUDART_HMASK
@@ -333,13 +322,13 @@ static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, i
 
 #else // defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)
 
-#if __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A
+#if __CUDA_ARCH__ >= MIN_CC_DP4A
     return __dp4a(a, b, c);
-#else // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A
+#else // __CUDA_ARCH__ >= MIN_CC_DP4A
     const int8_t * a8 = (const int8_t *) &a;
     const int8_t * b8 = (const int8_t *) &b;
     return c + a8[0]*b8[0] + a8[1]*b8[1] + a8[2]*b8[2] + a8[3]*b8[3];
-#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A
+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A
 
 #endif // defined(GGML_USE_HIP) && defined(__HIP_PLATFORM_AMD__)
 }

ggml/src/ggml-cuda/concat.cu

@@ -94,9 +94,7 @@ static void concat_f32_cuda(const float * x, const float * y, float * dst, int n
 }
 
 // non-contiguous kernel (slow)
-template <int dim>
-static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE)
-    concat_f32_non_cont(
+static __global__ void concat_f32_non_cont(
         const char * src0,
         const char * src1,
               char * dst,
@@ -123,28 +121,22 @@ static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE)
           uint64_t   nb0,
           uint64_t   nb1,
           uint64_t   nb2,
-          uint64_t   nb3){
-    static_assert(dim >= 0 && dim <= 3);
-
+          uint64_t   nb3,
+          int32_t   dim) {
     const int64_t i3 = blockIdx.z;
     const int64_t i2 = blockIdx.y;
     const int64_t i1 = blockIdx.x;
 
+    int64_t o[4] = {0, 0, 0, 0};
+    o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));
+
     const float * x;
 
-    for (int64_t i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) {
+    for (int i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) {
         if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
             x = (const float *)(src0 + (i3       )*nb03 + (i2       )*nb02 + (i1       )*nb01 + (i0       )*nb00);
         } else {
-            if constexpr (dim == 0) {
-                x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + i1 * nb11 + (i0 - ne00) * nb10);
-            } else if constexpr (dim == 1) {
-                x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + (i1 - ne01) * nb11 + i0 * nb10);
-            } else if constexpr (dim == 2) {
-                x = (const float *) (src1 + i3 * nb13 + (i2 - ne02) * nb12 + i1 * nb11 + i0 * nb10);
-            } else if constexpr (dim == 3) {
-                x = (const float *) (src1 + (i3 - ne03) * nb13 + i2 * nb12 + i1 * nb11 + i0 * nb10);
-            }
+            x = (const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
         }
 
         float * y = (float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
@@ -190,32 +182,15 @@ void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
         }
     } else {
         dim3 grid_dim(dst->ne[1], dst->ne[2], dst->ne[3]);
-        auto launch_kernel = [&](auto dim) {
-            concat_f32_non_cont<dim><<<grid_dim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(
-                (const char *) src0->data, (const char *) src1->data, (char *) dst->data,
+        concat_f32_non_cont<<<grid_dim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(
+                (const char *)src0->data,
+                (const char *)src1->data,
+                (      char *)dst->data,
                 src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
                 src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
                 src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],
                 src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
-                dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
-                dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3]);
-        };
-        switch (dim) {
-            case 0:
-                launch_kernel(std::integral_constant<int, 0>{});
-                break;
-            case 1:
-                launch_kernel(std::integral_constant<int, 1>{});
-                break;
-            case 2:
-                launch_kernel(std::integral_constant<int, 2>{});
-                break;
-            case 3:
-                launch_kernel(std::integral_constant<int, 3>{});
-                break;
-            default:
-                GGML_ABORT("Invalid dim: %d", dim);
-                break;
-        }
+                dst->ne[0],  dst->ne[1],  dst->ne[2],  dst->ne[3],
+                dst->nb[0],  dst->nb[1],  dst->nb[2],  dst->nb[3], dim);
     }
 }