ggml : disable CUDA graphs for non-llama.cpp projects

.devops/cublas.Dockerfile

@@ -12,7 +12,7 @@ FROM ${BASE_CUDA_DEV_CONTAINER} as build
 ARG CUDA_DOCKER_ARCH=all
 
 RUN apt-get update && \
-    apt-get install -y build-essential git cmake libsdl2-dev wget
+    apt-get install -y build-essential git cmake
 
 WORKDIR /app
 
@@ -23,6 +23,6 @@ ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
 # Enable cuBLAS
 ENV GGML_CUDA=1
 
-RUN make base.en
+RUN make
 
 ENTRYPOINT ["/app/main"]

.devops/main-cuda.Dockerfile

@@ -17,7 +17,7 @@ ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
 ENV GGML_CUDA=1
 
 RUN apt-get update && \
-    apt-get install -y build-essential libsdl2-dev wget cmake \
+    apt-get install -y build-essential \
     && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 # Ref: https://stackoverflow.com/a/53464012
@@ -25,7 +25,7 @@ ENV CUDA_MAIN_VERSION=12.3
 ENV LD_LIBRARY_PATH /usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH
 
 COPY .. .
-RUN make base.en
+RUN make
 
 FROM ${BASE_CUDA_RUN_CONTAINER} AS runtime
 ENV CUDA_MAIN_VERSION=12.3
@@ -33,7 +33,7 @@ ENV LD_LIBRARY_PATH /usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y curl ffmpeg wget cmake \
+  apt-get install -y curl ffmpeg \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY --from=build /app /app

.devops/main.Dockerfile

@@ -2,17 +2,17 @@ FROM ubuntu:22.04 AS build
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y build-essential wget cmake \
+  apt-get install -y build-essential \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY .. .
-RUN make base.en
+RUN make
 
 FROM ubuntu:22.04 AS runtime
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y curl ffmpeg libsdl2-dev wget cmake \
+  apt-get install -y curl ffmpeg \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY --from=build /app /app

.github/workflows/bindings-go.yml

@@ -13,10 +13,10 @@ jobs:
   ubuntu-latest:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v3
         with:
-          go-version: '^1.23'
+          go-version: '^1.19'
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v1
       - run: |
           cd bindings/go
           make test

.github/workflows/bindings-ruby.yml

@@ -3,53 +3,20 @@ on:
   push:
     paths:
       - bindings/ruby/**
-      - src/**/*.c
+      - whisper.h
-      - src/**/*.cpp
-      - src/**/*.h
-      - src/**/*.m
-      - src/**/*.metal
-      - include/**/*.c
-      - include/**/*.cpp
-      - include/**/*.h
-      - include/**/*.m
-      - include/**/*.metal
-      - ggml/**/*.c
-      - ggml/**/*.cpp
-      - ggml/**/*.h
-      - ggml/**/*.m
-      - ggml/**/*.metal
-      - scripts/get-flags.mk
-      - examples/dr_wav.h
   pull_request:
     paths:
       - bindings/ruby/**
-      - src/**/*.c
+      - whisper.h
-      - src/**/*.cpp
-      - src/**/*.h
-      - src/**/*.m
-      - src/**/*.metal
-      - include/**/*.c
-      - include/**/*.cpp
-      - include/**/*.h
-      - include/**/*.m
-      - include/**/*.metal
-      - ggml/**/*.c
-      - ggml/**/*.cpp
-      - ggml/**/*.h
-      - ggml/**/*.m
-      - ggml/**/*.metal
-      - scripts/get-flags.mk
-      - examples/dr_wav.h
 
 jobs:
   ubuntu-latest:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: bindings/ruby
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.1'
+          ruby-version: '3.0'
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v1
-      - run: rake test
+      - run: |
+          cd bindings/ruby/ext
+          ruby extconf.rb && make

.github/workflows/build.yml

@@ -1,19 +1,8 @@
 name: CI
-
+on: [push, pull_request]
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    types: [opened, synchronize, reopened]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
 
 env:
   ubuntu_image: "ubuntu:22.04"
-  VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
 
 jobs:
   ubuntu-latest:
@@ -22,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        arch: [linux/amd64, linux/ppc64le]
+        arch: [linux/amd64, linux/arm64, linux/arm/v7, linux/ppc64le]
 
     steps:
       - name: Clone
@@ -38,61 +27,9 @@ jobs:
             -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
             set -e
             apt update
-            apt install -y build-essential libsdl2-dev cmake
+            apt install -y build-essential libsdl2-dev
-            cmake -B build
+            make
-            cmake --build build --config Release -j $(nproc)'
+            make stream'
-
-  ubuntu-latest-arm64:
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        arch: [linux/arm64]
-
-    steps:
-      - name: Clone
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Build ${{ matrix.arch }}
-        run: |
-          docker run --platform ${{ matrix.arch }} --rm \
-            -v ${{ github.workspace }}:/workspace \
-            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
-            set -e
-            apt update
-            apt install -y build-essential libsdl2-dev cmake
-            cmake -B build -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8-a
-            cmake --build build --config Release -j $(nproc)'
-
-  ubuntu-latest-arm-v7:
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        arch: [linux/arm/v7]
-
-    steps:
-      - name: Clone
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Build ${{ matrix.arch }}
-        run: |
-          docker run --platform ${{ matrix.arch }} --rm \
-            -v ${{ github.workspace }}:/workspace \
-            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
-            set -e
-            apt update
-            apt install -y build-essential libsdl2-dev cmake
-            cmake -B build -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv7-a+fp
-            cmake --build build --config Release -j $(nproc)'
 
   macOS-latest:
     runs-on: macOS-latest
@@ -104,30 +41,30 @@ jobs:
       - name: Dependencies
         run: |
           brew update
-          brew install sdl2 cmake
+          brew install sdl2
 
       - name: Build
         run: |
-          cmake -B build
+          make
-          cmake --build build --config Release
+          make stream
 
-#  freeBSD-latest:
+  freeBSD-latest:
-#    runs-on: macos-12
+    runs-on: macos-12
-#
+
-#    steps:
+    steps:
-#      - name: Clone
+      - name: Clone
-#        uses: actions/checkout@v4
+        uses: actions/checkout@v4
-#
+
-#      - name: Build
+      - name: Build
-#        uses: cross-platform-actions/action@v0.24.0
+        uses: cross-platform-actions/action@v0.24.0
-#        with:
+        with:
-#          operating_system: freebsd
+          operating_system: freebsd
-#          version: '13.3'
+          version: '13.2'
-#          run: |
+          run: |
-#            sudo pkg update
+            sudo pkg update
-#            sudo pkg install -y gmake sdl2 cmake
+            sudo pkg install -y gmake sdl2
-#            cmake -B build
+            gmake
-#            cmake --build build --config Release
+            gmake stream
 
   ubuntu-latest-gcc:
     runs-on: ubuntu-latest
@@ -136,7 +73,7 @@ jobs:
       fail-fast: false
       matrix:
         build: [Debug, Release]
-        arch: [linux/amd64, linux/ppc64le]
+        arch: [linux/amd64, linux/arm64, linux/arm/v7, linux/ppc64le]
 
     steps:
       - name: Clone
@@ -157,62 +94,6 @@ jobs:
             make
             ctest -L gh --output-on-failure'
 
-  ubuntu-latest-gcc-arm64:
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        build: [Debug, Release]
-        arch: [linux/arm64]
-
-    steps:
-      - name: Clone
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Build ${{ matrix.arch }}
-        run: |
-          docker run --platform ${{ matrix.arch }} --rm \
-            -v ${{ github.workspace }}:/workspace \
-            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
-            set -e
-            apt update
-            apt install -y build-essential cmake libsdl2-dev
-            cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8-a
-            make
-            ctest -L gh --output-on-failure'
-
-  ubuntu-latest-gcc-arm-v7:
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        build: [Debug, Release]
-        arch: [linux/arm/v7]
-
-    steps:
-      - name: Clone
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Build ${{ matrix.arch }}
-        run: |
-          docker run --platform ${{ matrix.arch }} --rm \
-            -v ${{ github.workspace }}:/workspace \
-            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
-            set -e
-            apt update
-            apt install -y build-essential cmake libsdl2-dev
-            cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv7-a+fp
-            make
-            ctest -L gh --output-on-failure'
-
   ubuntu-latest-clang:
     runs-on: ubuntu-latest
 
@@ -398,10 +279,25 @@ jobs:
             mingw-w64-${{matrix.env}}-SDL2
             mingw-w64-${{matrix.env}}-openblas
 
+      - name: Build using make
+        shell: msys2 {0}
+        run: |
+            make -j $(nproc)
+
+      - name: Clean after building using make
+        shell: msys2 {0}
+        run: |
+            make clean
+
+      - name: Build using make w/ OpenBLAS
+        shell: msys2 {0}
+        run: |
+            make GGML_OPENBLAS=1 -j $(nproc)
+
       - name: Build using CMake
         shell: msys2 {0}
         run: |
-            cmake -B build -DWHISPER_SDL2=ON
+            cmake -B build
             cmake --build build --config ${{ matrix.build }} -j $(nproc)
 
       - name: Clean after building using CMake
@@ -412,7 +308,7 @@ jobs:
       - name: Build using CMake w/ OpenBLAS
         shell: msys2 {0}
         run: |
-            cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
+            cmake -B build -DGGML_OPENBLAS=ON
             cmake --build build --config ${{ matrix.build }} -j $(nproc)
 
   windows:
@@ -486,8 +382,10 @@ jobs:
         sdl2: [ON]
         include:
           - arch: Win32
+            obzip: https://github.com/OpenMathLib/OpenBLAS/releases/download/v0.3.25/OpenBLAS-0.3.25-x86.zip
             s2arc: x86
           - arch: x64
+            obzip: https://github.com/OpenMathLib/OpenBLAS/releases/download/v0.3.25/OpenBLAS-0.3.25-x64.zip
             s2arc: x64
           - sdl2: ON
             s2ver: 2.28.5
@@ -496,21 +394,17 @@ jobs:
       - name: Clone
         uses: actions/checkout@v4
 
-      - name: Export GitHub Actions cache environment variables
-        uses: actions/github-script@v7
-        with:
-          script: |
-            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
-            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
-
       - name: Add msbuild to PATH
         uses: microsoft/setup-msbuild@v2
 
-      - name: Install OpenBLAS and pkgconfiglite
+      - name: Fetch OpenBLAS
         if: matrix.blas == 'ON'
         run: |
-          vcpkg install --triplet=${{ matrix.s2arc }}-windows openblas
+          C:/msys64/usr/bin/wget.exe -qO blas.zip ${{ matrix.obzip }}
-          choco install pkgconfiglite
+          7z x blas.zip -oblas -y
+          copy blas/include/cblas.h .
+          copy blas/include/openblas_config.h .
+          echo "OPENBLAS_PATH=$env:GITHUB_WORKSPACE/blas" >> $env:GITHUB_ENV
 
       - name: Fetch SDL2 and set SDL2_DIR
         if: matrix.sdl2 == 'ON'
@@ -522,10 +416,9 @@ jobs:
       - name: Configure
         run: >
           cmake -S . -B ./build -A ${{ matrix.arch }}
-          -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake"
           -DCMAKE_BUILD_TYPE=${{ matrix.build }}
-          -DGGML_BLAS=${{ matrix.blas }}
+          -DGGML_OPENBLAS=${{ matrix.blas }}
-          -DGGML_BLAS_VENDOR=OpenBLAS
+          -DCMAKE_LIBRARY_PATH="$env:OPENBLAS_PATH/lib"
           -DWHISPER_SDL2=${{ matrix.sdl2 }}
 
       - name: Build
@@ -533,9 +426,9 @@ jobs:
           cd ./build
           msbuild ALL_BUILD.vcxproj -t:build -p:configuration=${{ matrix.build }} -p:platform=${{ matrix.arch }}
 
-      - name: Copy openblas.dll
+      - name: Copy libopenblas.dll
         if: matrix.blas == 'ON'
-        run: copy "C:/vcpkg/packages/openblas_${{ matrix.s2arc }}-windows/bin/openblas.dll" build/bin/${{ matrix.build }}
+        run: copy "$env:OPENBLAS_PATH/bin/libopenblas.dll" build/bin/${{ matrix.build }}
 
       - name: Copy SDL2.dll
         if: matrix.sdl2 == 'ON'
@@ -550,6 +443,7 @@ jobs:
 
   windows-cublas:
     runs-on: windows-2019
+
     strategy:
       matrix:
         build: [Release]
@@ -559,10 +453,12 @@ jobs:
         cuda-toolkit: [12.2.0, 11.8.0]
         include:
           - arch: x64
-            sdl2: ON
+            s2arc: x64
-            sdl2_ver: 2.28.5
+          - sdl2: ON
+            s2ver: 2.28.5
+
     steps:
-      - name: Clone repository
+      - name: Clone
         uses: actions/checkout@v4
 
       - name: Add msbuild to PATH
@@ -574,43 +470,38 @@ jobs:
         with:
           cuda: '${{ matrix.cuda-toolkit }}'
 
-      - name: Install 7-Zip
-        run: choco install 7zip -y
-
       - name: Fetch SDL2 and set SDL2_DIR
         if: matrix.sdl2 == 'ON'
         run: |
-          Invoke-WebRequest -Uri https://github.com/libsdl-org/SDL/releases/download/release-${{ matrix.sdl2_ver }}/SDL2-devel-${{ matrix.sdl2_ver }}-VC.zip -OutFile sdl2.zip
+          C:/msys64/usr/bin/wget.exe -qO sdl2.zip https://github.com/libsdl-org/SDL/releases/download/release-${{ matrix.s2ver }}/SDL2-devel-${{ matrix.s2ver }}-VC.zip
           7z x sdl2.zip
-          echo "SDL2_DIR=${{ github.workspace }}\SDL2-${{ matrix.sdl2_ver }}\cmake" | Out-File -FilePath $env:GITHUB_ENV -Append
+          echo "SDL2_DIR=$env:GITHUB_WORKSPACE/SDL2-${{ matrix.s2ver }}/cmake" >> $env:GITHUB_ENV
-          echo "${{ github.workspace }}\SDL2-${{ matrix.sdl2_ver }}\cmake" > SDL2_PATH.txt
+
- 
+      - name: Configure
-      - name: Configure CMake
+        run: >
-        shell: cmd
+          cmake -S . -B ./build -A ${{ matrix.arch }}
-        run: |
+          -DCMAKE_BUILD_TYPE=${{ matrix.build }}
-          cmake -S . -B ./build -A ${{ matrix.arch }} ^
+          -DGGML_CUDA=${{ matrix.cublas }}
-            -DCMAKE_BUILD_TYPE=${{ matrix.build }} ^
+          -DWHISPER_SDL2=${{ matrix.sdl2 }}
-            -DGGML_CUDA=${{ matrix.cublas }} ^
+
-            -DCMAKE_CUDA_ARCHITECTURES=all  ^
+      - name: Build ${{ matrix.cuda-toolkit }}
-            -DWHISPER_SDL2=${{ matrix.sdl2 }} ^
-            -DSDL2_DIR="%SDL2_DIR%"
-  
-      - name: Build Project
-        shell: cmd
         run: |
           cd ./build
-          cmake --build . --config ${{ matrix.build }}          
+          cmake --build . --config ${{ matrix.build }}
 
       - name: Copy CUDA DLLs
-        run: |
+        run: >
-          Get-ChildItem "${{ steps.cuda-toolkit.outputs.CUDA_PATH }}/bin/" -Filter "*.dll" | 
+          Copy-Item -PassThru
-          Copy-Item -Destination "build/bin/${{ matrix.build }}"
+          -Path "${{ steps.cuda-toolkit.outputs.CUDA_PATH }}/bin/*.dll"
+          -Include cudart64_*,cublas64_*,cublasLt64_*
+          -Destination build/bin/${{ matrix.build }}
 
       - name: Copy SDL2.dll
         if: matrix.sdl2 == 'ON'
-        run: copy "$env:SDL2_DIR/../lib/${{ matrix.arch }}/SDL2.dll" build/bin/${{ matrix.build }}
+        run: copy "$env:SDL2_DIR/../lib/${{ matrix.s2arc }}/SDL2.dll" build/bin/${{ matrix.build }}
 
       - name: Upload binaries
+        if: matrix.sdl2 == 'ON'
         uses: actions/upload-artifact@v4
         with:
           name: whisper-cublas-${{ matrix.cuda-toolkit }}-bin-${{ matrix.arch }}
@@ -638,7 +529,7 @@ jobs:
           emcmake cmake . -DCMAKE_BUILD_TYPE=${{ matrix.build }}
           make
 
-  ios-xcode-build:
+  ios:
     runs-on: macos-latest
 
     strategy:
@@ -646,7 +537,7 @@ jobs:
         build: [Release]
 
     steps:
-      - name: Checkout code
+      - name: Clone
         uses: actions/checkout@v4
 
       - name: Configure
@@ -654,34 +545,11 @@ jobs:
           cp models/for-tests-ggml-base.en.bin models/ggml-base.en.bin
           mkdir models/ggml-base.en-encoder.mlmodelc
 
-      - name: Build
+      - name: Build objc example
-        id: cmake_build
+        run: xcodebuild -project examples/whisper.objc/whisper.objc.xcodeproj -scheme whisper.objc -configuration ${{ matrix.build }} -sdk iphonesimulator build
-        run: |
-          sysctl -a
-          mkdir build
-          cd build
-          cmake -G Xcode .. \
-            -DGGML_METAL_USE_BF16=ON \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DWHISPER_BUILD_EXAMPLES=OFF \
-            -DWHISPER_BUILD_TESTS=OFF \
-            -DWHISPER_BUILD_SERVER=OFF \
-            -DCMAKE_SYSTEM_NAME=iOS \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
-            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
-          cmake --build . --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
-          sudo cmake --install . --config Release
-
-      - name: xcodebuild for swift package
-        id: xcodebuild
-        run: |
-          xcodebuild -scheme whisper-Package -destination 'generic/platform=iOS'
-
-#- name: Build objc example
-#  run: xcodebuild -project examples/whisper.objc/whisper.objc.xcodeproj -scheme whisper.objc -configuration ${{ matrix.build }} -sdk iphoneos build
 
       - name: Build swiftui example
-        run: xcodebuild -project examples/whisper.swiftui/whisper.swiftui.xcodeproj -scheme WhisperCppDemo -configuration ${{ matrix.build }} -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' build
+        run: xcodebuild -project examples/whisper.swiftui/whisper.swiftui.xcodeproj -scheme WhisperCppDemo -configuration ${{ matrix.build }} -sdk iphonesimulator build
 
   android:
     runs-on: ubuntu-latest
@@ -692,6 +560,12 @@ jobs:
         with:
           path: whisper
 
+      - name: Clone
+        uses: actions/checkout@v4
+        with:
+          repository: ggerganov/ggml
+          path: ggml
+
       - name: Install Java
         uses: actions/setup-java@v4
         with:
@@ -710,77 +584,75 @@ jobs:
         run: |
           export PATH_TO_GGML=$PWD/ggml
           cd whisper/examples/whisper.android
-          ./gradlew assembleRelease --no-daemon
+          ./gradlew assembleRelease --no-daemon -PGGML_HOME=$PATH_TO_GGML
 
-# TODO: disable because of following fail: https://github.com/ggerganov/whisper.cpp/actions/runs/11019444420/job/30627193602
+  android_java:
-#  android_java:
+    runs-on: ubuntu-latest
-#    runs-on: ubuntu-latest
-#
-#    steps:
-#      - name: Clone
-#        uses: actions/checkout@v4
-#
-#      - name: set up JDK 11
-#        uses: actions/setup-java@v4
-#        with:
-#          java-version: '11'
-#          distribution: 'temurin'
-#          cache: gradle
-#
-#      - name: Setup Android SDK
-#        uses: android-actions/setup-android@v3
-#        with:
-#          cmdline-tools-version: 9.0
-#
-#      - name: Build
-#        run: |
-#          cd examples/whisper.android.java
-#          chmod +x ./gradlew
-#          ./gradlew assembleRelease
 
-# TODO: disabled because of following fail: https://github.com/ggerganov/whisper.cpp/actions/runs/9686220096/job/26735899598
+    steps:
-#  java:
+      - name: Clone
-#    needs: [ 'windows' ]
+        uses: actions/checkout@v4
-#    runs-on: windows-latest
+
-#    steps:
+      - name: set up JDK 11
-#      - uses: actions/checkout@v4
+        uses: actions/setup-java@v4
-#
+        with:
-#      - name: Install Java
+          java-version: '11'
-#        uses: actions/setup-java@v4
+          distribution: 'temurin'
-#        with:
+          cache: gradle
-#          distribution: zulu
+
-#          java-version: 20
+      - name: Setup Android SDK
-#
+        uses: android-actions/setup-android@v3
-#      - name: Download Windows lib
+        with:
-#        uses: actions/download-artifact@v4
+          cmdline-tools-version: 9.0
-#        with:
+
-#          name: win32-x86-64_whisper.dll
+      - name: Build
-#          path: bindings/java/build/generated/resources/main/win32-x86-64
+        run: |
-#
+          cd examples/whisper.android.java
-#      - name: Build
+          chmod +x ./gradlew
-#        run: |
+          ./gradlew assembleRelease
-#          models\download-ggml-model.cmd tiny.en
+
-#          cd bindings/java
+  java:
-#          chmod +x ./gradlew
+    needs: [ 'windows' ]
-#          ./gradlew build
+    runs-on: windows-latest
-#
+    steps:
-#      - name: Upload jar
+      - uses: actions/checkout@v4
-#        uses: actions/upload-artifact@v4
+
-#        with:
+      - name: Install Java
-#          name: whispercpp.jar
+        uses: actions/setup-java@v4
-#          path: bindings/java/build/libs/whispercpp-*.jar
+        with:
-#
+          distribution: zulu
-#      - name: Publish package
+          java-version: 20
-#        if: ${{ github.ref == 'refs/heads/master' }}
+
-#        uses: gradle/gradle-build-action@v2.4.2
+      - name: Download Windows lib
-#        with:
+        uses: actions/download-artifact@v4
-#          arguments: publish
+        with:
-#          build-root-directory: bindings/java
+          name: win32-x86-64_whisper.dll
-#        env:
+          path: bindings/java/build/generated/resources/main/win32-x86-64
-#          MAVEN_USERNAME: ${{ secrets.JIRA_USER }}
+
-#          MAVEN_PASSWORD: ${{ secrets.JIRA_PASS }}
+      - name: Build
-#          PGP_SECRET: ${{ secrets.GPG_PRIVATE_KEY }}
+        run: |
-#          PGP_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+          models\download-ggml-model.cmd tiny.en
+          cd bindings/java
+          chmod +x ./gradlew
+          ./gradlew build
+
+      - name: Upload jar
+        uses: actions/upload-artifact@v4
+        with:
+          name: whispercpp.jar
+          path: bindings/java/build/libs/whispercpp-*.jar
+
+      - name: Publish package
+        if: ${{ github.ref == 'refs/heads/master' }}
+        uses: gradle/gradle-build-action@v2.4.2
+        with:
+          arguments: publish
+          build-root-directory: bindings/java
+        env:
+          MAVEN_USERNAME: ${{ secrets.JIRA_USER }}
+          MAVEN_PASSWORD: ${{ secrets.JIRA_PASS }}
+          PGP_SECRET: ${{ secrets.GPG_PRIVATE_KEY }}
+          PGP_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
 
   quantize:
     runs-on: ubuntu-latest
@@ -792,6 +664,5 @@ jobs:
       - name: Test quantize
         run: |
           ./models/download-ggml-model.sh tiny.en
-          cmake -B build
+          make quantize
-          cmake --build build --config Release
+          ./quantize models/ggml-tiny.en.bin models/ggml-tiny.en-q4_0.bin q4_0
-          ./build/bin/quantize models/ggml-tiny.en.bin models/ggml-tiny.en-q4_0.bin q4_0

.github/workflows/docker.yml

@@ -17,10 +17,8 @@ jobs:
     strategy:
       matrix:
         config:
-          - { tag: "main", dockerfile: ".devops/main.Dockerfile", platform: "linux/amd64" }
+          - { tag: "main", dockerfile: ".devops/main.Dockerfile", platform: "linux/amd64,linux/arm64" }
-          #TODO: the cuda image keeps failing - disable for now
+          - { tag: "main-cuda", dockerfile: ".devops/main-cuda.Dockerfile", platform: "linux/amd64" }
-          #      https://github.com/ggerganov/whisper.cpp/actions/runs/11019444428/job/30602020339
-          #- { tag: "main-cuda", dockerfile: ".devops/main-cuda.Dockerfile", platform: "linux/amd64" }
 
     steps:
       - name: Check out the repo
@@ -45,7 +43,7 @@ jobs:
         with:
           context: .
           push: true
-          platforms: ${{ matrix.config.platform }}
+          platforms: ${{ matrix.config.platforms }}
           tags: "ghcr.io/${{ github.repository }}:${{ matrix.config.tag }}-${{ env.COMMIT_SHA }}"
           file: ${{ matrix.config.dockerfile }}
 
@@ -54,6 +52,6 @@ jobs:
         with:
           context: .
           push: ${{ github.event_name == 'push' }}
-          platforms: ${{ matrix.config.platform }}
+          platforms: ${{ matrix.config.platforms }}
           tags: "ghcr.io/${{ github.repository }}:${{ matrix.config.tag }}"
           file: ${{ matrix.config.dockerfile }}

.gitignore

@@ -1,16 +1,13 @@
 *.o
 *.a
-*.d
 .cache/
 .coreml/
 .test/
-.venv/
 .vs/
 .vscode/
 .DS_Store
 .vimspector.json
 /CMakeSettings.json
-/talk-llama.dSYM/
 
 build/
 build-*/
@@ -20,9 +17,6 @@ build-*/
 .swiftpm
 *.metallib
 
-ggml-metal-embed.metal
-ggml-metal-embed.metal.tmp
-
 /main
 /stream
 /command

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.5) # for add_link_options and implicit target directories.
 project("whisper.cpp" C CXX)
-project("whisper.cpp" VERSION 1.7.4)
+project("whisper.cpp" VERSION 1.6.2)
 include(CheckIncludeFileCXX)
 
 set(SOVERSION 1)
@@ -120,10 +120,7 @@ whisper_option_depr(WARNING     WHISPER_SYCL_F16            GGML_SYCL_F16)
 # build the library
 #
 
-if (NOT TARGET ggml)
+add_subdirectory(ggml)
-    add_subdirectory(ggml)
-    # ... otherwise assume ggml is added by a parent CMakeLists.txt
-endif()
 add_subdirectory(src)
 
 #
@@ -164,6 +161,18 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake
               ${CMAKE_CURRENT_BINARY_DIR}/whisper-version.cmake
         DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/whisper)
 
+install(
+    FILES convert-hf-to-gguf.py
+    PERMISSIONS
+        OWNER_READ
+        OWNER_WRITE
+        OWNER_EXECUTE
+        GROUP_READ
+        GROUP_EXECUTE
+        WORLD_READ
+        WORLD_EXECUTE
+    DESTINATION ${CMAKE_INSTALL_BINDIR})
+
 configure_file(cmake/whisper.pc.in
         "${CMAKE_CURRENT_BINARY_DIR}/whisper.pc"
         @ONLY)

Package.swift

@@ -14,6 +14,46 @@ let package = Package(
         .library(name: "whisper", targets: ["whisper"]),
     ],
     targets: [
-        .systemLibrary(name: "whisper", pkgConfig: "whisper"),
+        .target(
-    ]
+            name: "whisper",
+            path: ".",
+            exclude: [
+               "bindings",
+               "cmake",
+               "coreml",
+               "examples",
+               "extra",
+               "models",
+               "samples",
+               "tests",
+               "CMakeLists.txt",
+               "Makefile"
+            ],
+            sources: [
+                "ggml/src/ggml.c",
+                "src/whisper.cpp",
+                "ggml/src/ggml-alloc.c",
+                "ggml/src/ggml-backend.c",
+                "ggml/src/ggml-quants.c",
+                "ggml/src/ggml-metal.m"
+            ],
+            resources: [.process("ggml-metal.metal")],
+            publicHeadersPath: "spm-headers",
+            cSettings: [
+                .unsafeFlags(["-Wno-shorten-64-to-32", "-O3", "-DNDEBUG"]),
+                .define("GGML_USE_ACCELERATE"),
+                .unsafeFlags(["-fno-objc-arc"]),
+                .define("GGML_USE_METAL")
+                // NOTE: NEW_LAPACK will required iOS version 16.4+
+                // We should consider add this in the future when we drop support for iOS 14
+                // (ref: ref: https://developer.apple.com/documentation/accelerate/1513264-cblas_sgemm?language=objc)
+                // .define("ACCELERATE_NEW_LAPACK"),
+                // .define("ACCELERATE_LAPACK_ILP64")
+            ],
+            linkerSettings: [
+                .linkedFramework("Accelerate")
+            ]
+        )
+    ],
+    cxxLanguageStandard: .cxx11
 )

README.md

@@ -7,23 +7,21 @@
 [![Conan Center](https://shields.io/conan/v/whisper-cpp)](https://conan.io/center/whisper-cpp)
 [![npm](https://img.shields.io/npm/v/whisper.cpp.svg)](https://www.npmjs.com/package/whisper.cpp/)
 
-Stable: [v1.7.4](https://github.com/ggerganov/whisper.cpp/releases/tag/v1.7.4) / [Roadmap | F.A.Q.](https://github.com/ggerganov/whisper.cpp/discussions/126)
+Stable: [v1.6.2](https://github.com/ggerganov/whisper.cpp/releases/tag/v1.6.0) / [Roadmap | F.A.Q.](https://github.com/ggerganov/whisper.cpp/discussions/126)
 
 High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisper) automatic speech recognition (ASR) model:
 
 - Plain C/C++ implementation without dependencies
-- Apple Silicon first-class citizen - optimized via ARM NEON, Accelerate framework, Metal and [Core ML](#core-ml-support)
+- Apple Silicon first-class citizen - optimized via ARM NEON, Accelerate framework, Metal and [Core ML](https://github.com/ggerganov/whisper.cpp#core-ml-support)
 - AVX intrinsics support for x86 architectures
 - VSX intrinsics support for POWER architectures
 - Mixed F16 / F32 precision
-- [Integer quantization support](#quantization)
+- [4-bit and 5-bit integer quantization support](https://github.com/ggerganov/whisper.cpp#quantization)
 - Zero memory allocations at runtime
-- [Vulkan support](#vulkan-gpu-support)
 - Support for CPU-only inference
-- [Efficient GPU support for NVIDIA](#nvidia-gpu-support)
+- [Efficient GPU support for NVIDIA](https://github.com/ggerganov/whisper.cpp#nvidia-gpu-support-via-cublas)
-- [OpenVINO Support](#openvino-support)
+- [OpenVINO Support](https://github.com/ggerganov/whisper.cpp#openvino-support)
-- [Ascend NPU Support](#ascend-npu-support)
+- [C-style API](https://github.com/ggerganov/whisper.cpp/blob/master/whisper.h)
-- [C-style API](https://github.com/ggerganov/whisper.cpp/blob/master/include/whisper.h)
 
 Supported platforms:
 
@@ -35,9 +33,9 @@ Supported platforms:
 - [x] [WebAssembly](examples/whisper.wasm)
 - [x] Windows ([MSVC](https://github.com/ggerganov/whisper.cpp/blob/master/.github/workflows/build.yml#L117-L144) and [MinGW](https://github.com/ggerganov/whisper.cpp/issues/168)]
 - [x] [Raspberry Pi](https://github.com/ggerganov/whisper.cpp/discussions/166)
-- [x] [Docker](https://github.com/ggerganov/whisper.cpp/pkgs/container/whisper.cpp)
+- [x] [docker](https://github.com/ggerganov/whisper.cpp/pkgs/container/whisper.cpp)
 
-The entire high-level implementation of the model is contained in [whisper.h](include/whisper.h) and [whisper.cpp](src/whisper.cpp).
+The entire high-level implementation of the model is contained in [whisper.h](whisper.h) and [whisper.cpp](whisper.cpp).
 The rest of the code is part of the [`ggml`](https://github.com/ggerganov/ggml) machine learning library.
 
 Having such a lightweight implementation of the model allows to easily integrate it in different platforms and applications.
@@ -53,6 +51,18 @@ On Apple Silicon, the inference runs fully on the GPU via Metal:
 
 https://github.com/ggerganov/whisper.cpp/assets/1991296/c82e8f86-60dc-49f2-b048-d2fdbd6b5225
 
+Or you can even run it straight in the browser: [talk.wasm](examples/talk.wasm)
+
+## Implementation details
+
+- The core tensor operations are implemented in C ([ggml.h](ggml.h) / [ggml.c](ggml.c))
+- The transformer model and the high-level C-style API are implemented in C++ ([whisper.h](whisper.h) / [whisper.cpp](whisper.cpp))
+- Sample usage is demonstrated in [main.cpp](examples/main)
+- Sample real-time audio transcription from the microphone is demonstrated in [stream.cpp](examples/stream)
+- Various other examples are available in the [examples](examples) folder
+
+The tensor operators are optimized heavily for Apple silicon CPUs. Depending on the computation size, Arm Neon SIMD intrinsics or CBLAS Accelerate framework routines are used. The latter are especially effective for bigger sizes since the Accelerate framework utilizes the special-purpose AMX coprocessor available in modern Apple products.
+
 ## Quick start
 
 First clone the repository:
@@ -61,38 +71,140 @@ First clone the repository:
 git clone https://github.com/ggerganov/whisper.cpp.git
 ```
 
-Navigate into the directory:
-
-```
-cd whisper.cpp
-```
-
 Then, download one of the Whisper [models](models/README.md) converted in [`ggml` format](#ggml-format). For example:
 
 ```bash
-sh ./models/download-ggml-model.sh base.en
+bash ./models/download-ggml-model.sh base.en
 ```
 
-Now build the [whisper-cli](examples/cli) example and transcribe an audio file like this:
+Now build the [main](examples/main) example and transcribe an audio file like this:
 
 ```bash
-# build the project
+# build the main example
-cmake -B build
+make
-cmake --build build --config Release
 
 # transcribe an audio file
-./build/bin/whisper-cli -f samples/jfk.wav
+./main -f samples/jfk.wav
 ```
 
 ---
 
-For a quick demo, simply run `make base.en`.
+For a quick demo, simply run `make base.en`:
+
+```text
+$ make base.en
+
+cc  -I.              -O3 -std=c11   -pthread -DGGML_USE_ACCELERATE   -c ggml.c -o ggml.o
+c++ -I. -I./examples -O3 -std=c++11 -pthread -c whisper.cpp -o whisper.o
+c++ -I. -I./examples -O3 -std=c++11 -pthread examples/main/main.cpp whisper.o ggml.o -o main  -framework Accelerate
+./main -h
+
+usage: ./main [options] file0.wav file1.wav ...
+
+options:
+  -h,        --help              [default] show this help message and exit
+  -t N,      --threads N         [4      ] number of threads to use during computation
+  -p N,      --processors N      [1      ] number of processors to use during computation
+  -ot N,     --offset-t N        [0      ] time offset in milliseconds
+  -on N,     --offset-n N        [0      ] segment index offset
+  -d  N,     --duration N        [0      ] duration of audio to process in milliseconds
+  -mc N,     --max-context N     [-1     ] maximum number of text context tokens to store
+  -ml N,     --max-len N         [0      ] maximum segment length in characters
+  -sow,      --split-on-word     [false  ] split on word rather than on token
+  -bo N,     --best-of N         [5      ] number of best candidates to keep
+  -bs N,     --beam-size N       [5      ] beam size for beam search
+  -wt N,     --word-thold N      [0.01   ] word timestamp probability threshold
+  -et N,     --entropy-thold N   [2.40   ] entropy threshold for decoder fail
+  -lpt N,    --logprob-thold N   [-1.00  ] log probability threshold for decoder fail
+  -debug,    --debug-mode        [false  ] enable debug mode (eg. dump log_mel)
+  -tr,       --translate         [false  ] translate from source language to english
+  -di,       --diarize           [false  ] stereo audio diarization
+  -tdrz,     --tinydiarize       [false  ] enable tinydiarize (requires a tdrz model)
+  -nf,       --no-fallback       [false  ] do not use temperature fallback while decoding
+  -otxt,     --output-txt        [false  ] output result in a text file
+  -ovtt,     --output-vtt        [false  ] output result in a vtt file
+  -osrt,     --output-srt        [false  ] output result in a srt file
+  -olrc,     --output-lrc        [false  ] output result in a lrc file
+  -owts,     --output-words      [false  ] output script for generating karaoke video
+  -fp,       --font-path         [/System/Library/Fonts/Supplemental/Courier New Bold.ttf] path to a monospace font for karaoke video
+  -ocsv,     --output-csv        [false  ] output result in a CSV file
+  -oj,       --output-json       [false  ] output result in a JSON file
+  -ojf,      --output-json-full  [false  ] include more information in the JSON file
+  -of FNAME, --output-file FNAME [       ] output file path (without file extension)
+  -ps,       --print-special     [false  ] print special tokens
+  -pc,       --print-colors      [false  ] print colors
+  -pp,       --print-progress    [false  ] print progress
+  -nt,       --no-timestamps     [false  ] do not print timestamps
+  -l LANG,   --language LANG     [en     ] spoken language ('auto' for auto-detect)
+  -dl,       --detect-language   [false  ] exit after automatically detecting language
+             --prompt PROMPT     [       ] initial prompt
+  -m FNAME,  --model FNAME       [models/ggml-base.en.bin] model path
+  -f FNAME,  --file FNAME        [       ] input WAV file path
+  -oved D,   --ov-e-device DNAME [CPU    ] the OpenVINO device used for encode inference
+  -ls,       --log-score         [false  ] log best decoder scores of tokens
+  -ng,       --no-gpu            [false  ] disable GPU
+
+
+bash ./models/download-ggml-model.sh base.en
+Downloading ggml model base.en ...
+ggml-base.en.bin               100%[========================>] 141.11M  6.34MB/s    in 24s
+Done! Model 'base.en' saved in 'models/ggml-base.en.bin'
+You can now use it like this:
+
+  $ ./main -m models/ggml-base.en.bin -f samples/jfk.wav
+
+
+===============================================
+Running base.en on all samples in ./samples ...
+===============================================
+
+----------------------------------------------
+[+] Running base.en on samples/jfk.wav ... (run 'ffplay samples/jfk.wav' to listen)
+----------------------------------------------
+
+whisper_init_from_file: loading model from 'models/ggml-base.en.bin'
+whisper_model_load: loading model
+whisper_model_load: n_vocab       = 51864
+whisper_model_load: n_audio_ctx   = 1500
+whisper_model_load: n_audio_state = 512
+whisper_model_load: n_audio_head  = 8
+whisper_model_load: n_audio_layer = 6
+whisper_model_load: n_text_ctx    = 448
+whisper_model_load: n_text_state  = 512
+whisper_model_load: n_text_head   = 8
+whisper_model_load: n_text_layer  = 6
+whisper_model_load: n_mels        = 80
+whisper_model_load: f16           = 1
+whisper_model_load: type          = 2
+whisper_model_load: mem required  =  215.00 MB (+    6.00 MB per decoder)
+whisper_model_load: kv self size  =    5.25 MB
+whisper_model_load: kv cross size =   17.58 MB
+whisper_model_load: adding 1607 extra tokens
+whisper_model_load: model ctx     =  140.60 MB
+whisper_model_load: model size    =  140.54 MB
+
+system_info: n_threads = 4 / 10 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 |
+
+main: processing 'samples/jfk.wav' (176000 samples, 11.0 sec), 4 threads, 1 processors, lang = en, task = transcribe, timestamps = 1 ...
+
+
+[00:00:00.000 --> 00:00:11.000]   And so my fellow Americans, ask not what your country can do for you, ask what you can do for your country.
+
+
+whisper_print_timings:     fallbacks =   0 p /   0 h
+whisper_print_timings:     load time =   113.81 ms
+whisper_print_timings:      mel time =    15.40 ms
+whisper_print_timings:   sample time =    11.58 ms /    27 runs (    0.43 ms per run)
+whisper_print_timings:   encode time =   266.60 ms /     1 runs (  266.60 ms per run)
+whisper_print_timings:   decode time =    66.11 ms /    27 runs (    2.45 ms per run)
+whisper_print_timings:    total time =   476.31 ms
+```
 
 The command downloads the `base.en` model converted to custom `ggml` format and runs the inference on all `.wav` samples in the folder `samples`.
 
-For detailed usage instructions, run: `./build/bin/whisper-cli -h`
+For detailed usage instructions, run: `./main -h`
 
-Note that the [whisper-cli](examples/cli) example currently runs only with 16-bit WAV files, so make sure to convert your input before running the tool.
+Note that the [main](examples/main) example currently runs only with 16-bit WAV files, so make sure to convert your input before running the tool.
 For example, you can use `ffmpeg` like this:
 
 ```bash
@@ -104,7 +216,7 @@ ffmpeg -i input.mp3 -ar 16000 -ac 1 -c:a pcm_s16le output.wav
 If you want some extra audio samples to play with, simply run:
 
 ```
-make -j samples
+make samples
 ```
 
 This will download a few more audio files from Wikipedia and convert them to 16-bit WAV format via `ffmpeg`.
@@ -112,18 +224,17 @@ This will download a few more audio files from Wikipedia and convert them to 16-
 You can download and run the other models as follows:
 
 ```
-make -j tiny.en
+make tiny.en
-make -j tiny
+make tiny
-make -j base.en
+make base.en
-make -j base
+make base
-make -j small.en
+make small.en
-make -j small
+make small
-make -j medium.en
+make medium.en
-make -j medium
+make medium
-make -j large-v1
+make large-v1
-make -j large-v2
+make large-v2
-make -j large-v3
+make large-v3
-make -j large-v3-turbo
 ```
 
 ## Memory usage
@@ -145,12 +256,11 @@ Here are the steps for creating and using a quantized model:
 
 ```bash
 # quantize a model with Q5_0 method
-cmake -B build
+make quantize
-cmake --build build --config Release
+./quantize models/ggml-base.en.bin models/ggml-base.en-q5_0.bin q5_0
-./build/bin/quantize models/ggml-base.en.bin models/ggml-base.en-q5_0.bin q5_0
 
 # run the examples as usual, specifying the quantized model file
-./build/bin/whisper-cli -m models/ggml-base.en-q5_0.bin ./samples/gb0.wav
+./main -m models/ggml-base.en-q5_0.bin ./samples/gb0.wav
 ```
 
 ## Core ML support
@@ -184,6 +294,10 @@ speed-up - more than x3 faster compared with CPU-only execution. Here are the in
 - Build `whisper.cpp` with Core ML support:
 
   ```bash
+  # using Makefile
+  make clean
+  WHISPER_COREML=1 make -j
+
   # using CMake
   cmake -B build -DWHISPER_COREML=1
   cmake --build build -j --config Release
@@ -192,7 +306,7 @@ speed-up - more than x3 faster compared with CPU-only execution. Here are the in
 - Run the examples as usual. For example:
 
   ```text
-  $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav
+  $ ./main -m models/ggml-base.en.bin -f samples/jfk.wav
 
   ...
 
@@ -276,7 +390,7 @@ This can result in significant speedup in encoder performance. Here are the inst
 - Run the examples as usual. For example:
 
   ```text
-  $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav
+  $ ./main -m models/ggml-base.en.bin -f samples/jfk.wav
 
   ...
 
@@ -293,7 +407,7 @@ This can result in significant speedup in encoder performance. Here are the inst
   The first time run on an OpenVINO device is slow, since the OpenVINO framework will compile the IR (Intermediate Representation) model to a device-specific 'blob'. This device-specific blob will get
   cached for the next run.
 
-For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggerganov/whisper.cpp/pull/1037).
+For more information about the Core ML implementation please refer to PR [#1037](https://github.com/ggerganov/whisper.cpp/pull/1037).
 
 ## NVIDIA GPU support
 
@@ -303,18 +417,8 @@ First, make sure you have installed `cuda`: https://developer.nvidia.com/cuda-do
 Now build `whisper.cpp` with CUDA support:
 
 ```
-cmake -B build -DGGML_CUDA=1
+make clean
-cmake --build build -j --config Release
+GGML_CUDA=1 make -j
-```
-
-## Vulkan GPU support
-Cross-vendor solution which allows you to accelerate workload on your GPU.
-First, make sure your graphics card driver provides support for Vulkan API.
-
-Now build `whisper.cpp` with Vulkan support:
-```
-cmake -B build -DGGML_VULKAN=1
-cmake --build build -j --config Release
 ```
 
 ## BLAS CPU support via OpenBLAS
@@ -325,40 +429,56 @@ First, make sure you have installed `openblas`: https://www.openblas.net/
 Now build `whisper.cpp` with OpenBLAS support:
 
 ```
-cmake -B build -DGGML_BLAS=1
+make clean
-cmake --build build -j --config Release
+GGML_OPENBLAS=1 make -j
 ```
 
-## Ascend NPU support
+## BLAS CPU support via Intel MKL
 
-Ascend NPU provides inference acceleration via [`CANN`](https://www.hiascend.com/en/software/cann) and AI cores.
+Encoder processing can be accelerated on the CPU via the BLAS compatible interface of Intel's Math Kernel Library.
+First, make sure you have installed Intel's MKL runtime and development packages: https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl-download.html
 
-First, check if your Ascend NPU device is supported:
+Now build `whisper.cpp` with Intel MKL BLAS support:
-
-**Verified devices**
-| Ascend NPU                    | Status  |
-|:-----------------------------:|:-------:|
-| Atlas 300T A2                 | Support |
-
-Then, make sure you have installed [`CANN toolkit`](https://www.hiascend.com/en/software/cann/community) . The lasted version of CANN is recommanded.
-
-Now build `whisper.cpp` with CANN support:
 
 ```
-cmake -B build -DGGML_CANN=1
+source /opt/intel/oneapi/setvars.sh
-cmake --build build -j --config Release
+mkdir build
+cd build
+cmake -DWHISPER_MKL=ON ..
+WHISPER_MKL=1 make -j
 ```
 
-Run the inference examples as usual, for example:
+## Docker
 
+### Prerequisites
+
+- Docker must be installed and running on your system.
+- Create a folder to store big models & intermediate files (ex. /whisper/models)
+
+### Images
+
+We have two Docker images available for this project:
+
+1. `ghcr.io/ggerganov/whisper.cpp:main`: This image includes the main executable file as well as `curl` and `ffmpeg`. (platforms: `linux/amd64`, `linux/arm64`)
+2. `ghcr.io/ggerganov/whisper.cpp:main-cuda`: Same as `main` but compiled with CUDA support. (platforms: `linux/amd64`)
+
+### Usage
+
+```shell
+# download model and persist it in a local folder
+docker run -it --rm \
+  -v path/to/models:/models \
+  whisper.cpp:main "./models/download-ggml-model.sh base /models"
+# transcribe an audio file
+docker run -it --rm \
+  -v path/to/models:/models \
+  -v path/to/audios:/audios \
+  whisper.cpp:main "./main -m /models/ggml-base.bin -f /audios/jfk.wav"
+# transcribe an audio file in samples folder
+docker run -it --rm \
+  -v path/to/models:/models \
+  whisper.cpp:main "./main -m /models/ggml-base.bin -f ./samples/jfk.wav"
 ```
-./build/bin/whisper-cli -f samples/jfk.wav -m models/ggml-base.en.bin -t 8
-```
-
-*Notes:*
-
-- If you have trouble with Ascend NPU device, please create a issue with **[CANN]** prefix/tag.
-- If you run successfully with your Ascend NPU device, please help update the table `Verified devices`.
 
 ## Installing with Conan
 
@@ -374,6 +494,89 @@ For detailed instructions on how to use Conan, please refer to the [Conan docume
 
 - Inference only
 
+## Another example
+
+Here is another example of transcribing a [3:24 min speech](https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg)
+in about half a minute on a MacBook M1 Pro, using `medium.en` model:
+
+<details>
+  <summary>Expand to see the result</summary>
+
+```text
+$ ./main -m models/ggml-medium.en.bin -f samples/gb1.wav -t 8
+
+whisper_init_from_file: loading model from 'models/ggml-medium.en.bin'
+whisper_model_load: loading model
+whisper_model_load: n_vocab       = 51864
+whisper_model_load: n_audio_ctx   = 1500
+whisper_model_load: n_audio_state = 1024
+whisper_model_load: n_audio_head  = 16
+whisper_model_load: n_audio_layer = 24
+whisper_model_load: n_text_ctx    = 448
+whisper_model_load: n_text_state  = 1024
+whisper_model_load: n_text_head   = 16
+whisper_model_load: n_text_layer  = 24
+whisper_model_load: n_mels        = 80
+whisper_model_load: f16           = 1
+whisper_model_load: type          = 4
+whisper_model_load: mem required  = 1720.00 MB (+   43.00 MB per decoder)
+whisper_model_load: kv self size  =   42.00 MB
+whisper_model_load: kv cross size =  140.62 MB
+whisper_model_load: adding 1607 extra tokens
+whisper_model_load: model ctx     = 1462.35 MB
+whisper_model_load: model size    = 1462.12 MB
+
+system_info: n_threads = 8 / 10 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 |
+
+main: processing 'samples/gb1.wav' (3179750 samples, 198.7 sec), 8 threads, 1 processors, lang = en, task = transcribe, timestamps = 1 ...
+
+
+[00:00:00.000 --> 00:00:08.000]   My fellow Americans, this day has brought terrible news and great sadness to our country.
+[00:00:08.000 --> 00:00:17.000]   At nine o'clock this morning, Mission Control in Houston lost contact with our Space Shuttle Columbia.
+[00:00:17.000 --> 00:00:23.000]   A short time later, debris was seen falling from the skies above Texas.
+[00:00:23.000 --> 00:00:29.000]   The Columbia's lost. There are no survivors.
+[00:00:29.000 --> 00:00:32.000]   On board was a crew of seven.
+[00:00:32.000 --> 00:00:39.000]   Colonel Rick Husband, Lieutenant Colonel Michael Anderson, Commander Laurel Clark,
+[00:00:39.000 --> 00:00:48.000]   Captain David Brown, Commander William McCool, Dr. Kultna Shavla, and Ilan Ramon,
+[00:00:48.000 --> 00:00:52.000]   a colonel in the Israeli Air Force.
+[00:00:52.000 --> 00:00:58.000]   These men and women assumed great risk in the service to all humanity.
+[00:00:58.000 --> 00:01:03.000]   In an age when space flight has come to seem almost routine,
+[00:01:03.000 --> 00:01:07.000]   it is easy to overlook the dangers of travel by rocket
+[00:01:07.000 --> 00:01:12.000]   and the difficulties of navigating the fierce outer atmosphere of the Earth.
+[00:01:12.000 --> 00:01:18.000]   These astronauts knew the dangers, and they faced them willingly,
+[00:01:18.000 --> 00:01:23.000]   knowing they had a high and noble purpose in life.
+[00:01:23.000 --> 00:01:31.000]   Because of their courage and daring and idealism, we will miss them all the more.
+[00:01:31.000 --> 00:01:36.000]   All Americans today are thinking as well of the families of these men and women
+[00:01:36.000 --> 00:01:40.000]   who have been given this sudden shock and grief.
+[00:01:40.000 --> 00:01:45.000]   You're not alone. Our entire nation grieves with you,
+[00:01:45.000 --> 00:01:52.000]   and those you love will always have the respect and gratitude of this country.
+[00:01:52.000 --> 00:01:56.000]   The cause in which they died will continue.
+[00:01:56.000 --> 00:02:04.000]   Mankind is led into the darkness beyond our world by the inspiration of discovery
+[00:02:04.000 --> 00:02:11.000]   and the longing to understand. Our journey into space will go on.
+[00:02:11.000 --> 00:02:16.000]   In the skies today, we saw destruction and tragedy.
+[00:02:16.000 --> 00:02:22.000]   Yet farther than we can see, there is comfort and hope.
+[00:02:22.000 --> 00:02:29.000]   In the words of the prophet Isaiah, "Lift your eyes and look to the heavens
+[00:02:29.000 --> 00:02:35.000]   who created all these. He who brings out the starry hosts one by one
+[00:02:35.000 --> 00:02:39.000]   and calls them each by name."
+[00:02:39.000 --> 00:02:46.000]   Because of His great power and mighty strength, not one of them is missing.
+[00:02:46.000 --> 00:02:55.000]   The same Creator who names the stars also knows the names of the seven souls we mourn today.
+[00:02:55.000 --> 00:03:01.000]   The crew of the shuttle Columbia did not return safely to earth,
+[00:03:01.000 --> 00:03:05.000]   yet we can pray that all are safely home.
+[00:03:05.000 --> 00:03:13.000]   May God bless the grieving families, and may God continue to bless America.
+[00:03:13.000 --> 00:03:19.000]   [Silence]
+
+
+whisper_print_timings:     fallbacks =   1 p /   0 h
+whisper_print_timings:     load time =   569.03 ms
+whisper_print_timings:      mel time =   146.85 ms
+whisper_print_timings:   sample time =   238.66 ms /   553 runs (    0.43 ms per run)
+whisper_print_timings:   encode time = 18665.10 ms /     9 runs ( 2073.90 ms per run)
+whisper_print_timings:   decode time = 13090.93 ms /   549 runs (   23.85 ms per run)
+whisper_print_timings:    total time = 32733.52 ms
+```
+
+</details>
+
 ## Real-time audio input example
 
 This is a naive example of performing real-time inference on audio from your microphone.
@@ -381,9 +584,8 @@ The [stream](examples/stream) tool samples the audio every half a second and run
 More info is available in [issue #10](https://github.com/ggerganov/whisper.cpp/issues/10).
 
 ```bash
-cmake -B build -DWHISPER_SDL2=ON
+make stream
-cmake --build build --config Release
+./stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
-./build/bin/whisper-stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
 ```
 
 https://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a80f-28ba83be7d09.mp4
@@ -394,7 +596,7 @@ Adding the `--print-colors` argument will print the transcribed text using an ex
 to highlight words with high or low confidence:
 
 ```bash
-./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/gb0.wav --print-colors
+./main -m models/ggml-base.en.bin -f samples/gb0.wav --print-colors
 ```
 
 <img width="965" alt="image" src="https://user-images.githubusercontent.com/1991296/197356445-311c8643-9397-4e5e-b46e-0b4b4daa2530.png">
@@ -404,7 +606,7 @@ to highlight words with high or low confidence:
 For example, to limit the line length to a maximum of 16 characters, simply add `-ml 16`:
 
 ```text
-$ ./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 16
+$ ./main -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 16
 
 whisper_model_load: loading model from './models/ggml-base.en.bin'
 ...
@@ -428,7 +630,7 @@ main: processing './samples/jfk.wav' (176000 samples, 11.0 sec), 4 threads, 1 pr
 The `--max-len` argument can be used to obtain word-level timestamps. Simply use `-ml 1`:
 
 ```text
-$ ./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 1
+$ ./main -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 1
 
 whisper_model_load: loading model from './models/ggml-base.en.bin'
 ...
@@ -475,7 +677,7 @@ Sample usage:
 ./models/download-ggml-model.sh small.en-tdrz
 
 # run as usual, adding the "-tdrz" command-line argument
-./build/bin/whisper-cli -f ./samples/a13.wav -m ./models/ggml-small.en-tdrz.bin -tdrz
+./main -f ./samples/a13.wav -m ./models/ggml-small.en-tdrz.bin -tdrz
 ...
 main: processing './samples/a13.wav' (480000 samples, 30.0 sec), 4 threads, 1 processors, lang = en, task = transcribe, tdrz = 1, timestamps = 1 ...
 ...
@@ -492,14 +694,14 @@ main: processing './samples/a13.wav' (480000 samples, 30.0 sec), 4 threads, 1 pr
 
 ## Karaoke-style movie generation (experimental)
 
-The [whisper-cli](examples/cli) example provides support for output of karaoke-style movies, where the
+The [main](examples/main) example provides support for output of karaoke-style movies, where the
 currently pronounced word is highlighted. Use the `-wts` argument and run the generated bash script.
 This requires to have `ffmpeg` installed.
 
 Here are a few _"typical"_ examples:
 
 ```bash
-./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -owts
+./main -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -owts
 source ./samples/jfk.wav.wts
 ffplay ./samples/jfk.wav.mp4
 ```
@@ -509,7 +711,7 @@ https://user-images.githubusercontent.com/1991296/199337465-dbee4b5e-9aeb-48a3-b
 ---
 
 ```bash
-./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/mm0.wav -owts
+./main -m ./models/ggml-base.en.bin -f ./samples/mm0.wav -owts
 source ./samples/mm0.wav.wts
 ffplay ./samples/mm0.wav.mp4
 ```
@@ -519,7 +721,7 @@ https://user-images.githubusercontent.com/1991296/199337504-cc8fd233-0cb7-4920-9
 ---
 
 ```bash
-./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/gb0.wav -owts
+./main -m ./models/ggml-base.en.bin -f ./samples/gb0.wav -owts
 source ./samples/gb0.wav.wts
 ffplay ./samples/gb0.wav.mp4
 ```
@@ -544,12 +746,12 @@ https://user-images.githubusercontent.com/1991296/223206245-2d36d903-cf8e-4f09-8
 ## Benchmarks
 
 In order to have an objective comparison of the performance of the inference across different system configurations,
-use the [whisper-bench](examples/bench) tool. The tool simply runs the Encoder part of the model and prints how much time it
+use the [bench](examples/bench) tool. The tool simply runs the Encoder part of the model and prints how much time it
 took to execute it. The results are summarized in the following Github issue:
 
 [Benchmark results](https://github.com/ggerganov/whisper.cpp/issues/89)
 
-Additionally a script to run whisper.cpp with different models and audio files is provided [bench.py](scripts/bench.py).
+Additionally a script to run whisper.cpp with different models and audio files is provided [bench.py](bench.py).
 
 You can run it with the following command, by default it will run against any standard model in the models folder.
 
@@ -596,7 +798,6 @@ For more details, see the conversion script [models/convert-pt-to-ggml.py](model
   - [stlukey/whispercpp.py](https://github.com/stlukey/whispercpp.py) (Cython)
   - [AIWintermuteAI/whispercpp](https://github.com/AIWintermuteAI/whispercpp) (Updated fork of aarnphm/whispercpp)
   - [aarnphm/whispercpp](https://github.com/aarnphm/whispercpp) (Pybind11)
-  - [abdeladim-s/pywhispercpp](https://github.com/abdeladim-s/pywhispercpp) (Pybind11)
 - [x] R: [bnosac/audio.whisper](https://github.com/bnosac/audio.whisper)
 - [x] Unity: [macoron/whisper.unity](https://github.com/Macoron/whisper.unity)
 
@@ -607,12 +808,13 @@ Some of the examples are even ported to run in the browser using WebAssembly. Ch
 
 | Example                                             | Web                                   | Description                                                                                                                     |
 | --------------------------------------------------- | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
-| [whisper-cli](examples/cli)                         | [whisper.wasm](examples/whisper.wasm) | Tool for translating and transcribing audio using Whisper                                                                       |
+| [main](examples/main)                               | [whisper.wasm](examples/whisper.wasm) | Tool for translating and transcribing audio using Whisper                                                                       |
-| [whisper-bench](examples/bench)                     | [bench.wasm](examples/bench.wasm)     | Benchmark the performance of Whisper on your machine                                                                            |
+| [bench](examples/bench)                             | [bench.wasm](examples/bench.wasm)     | Benchmark the performance of Whisper on your machine                                                                            |
-| [whisper-stream](examples/stream)                   | [stream.wasm](examples/stream.wasm)   | Real-time transcription of raw microphone capture                                                                               |
+| [stream](examples/stream)                           | [stream.wasm](examples/stream.wasm)   | Real-time transcription of raw microphone capture                                                                               |
-| [whisper-command](examples/command)                 | [command.wasm](examples/command.wasm) | Basic voice assistant example for receiving voice commands from the mic                                                         |
+| [command](examples/command)                         | [command.wasm](examples/command.wasm) | Basic voice assistant example for receiving voice commands from the mic                                                         |
-| [whisper-server](examples/server)                   |                                       | HTTP transcription server with OAI-like API                                                                                     |
+| [wchess](examples/wchess)                           | [wchess.wasm](examples/wchess)        | Voice-controlled chess                                                                                                          |
-| [whisper-talk-llama](examples/talk-llama)           |                                       | Talk with a LLaMA bot                                                                                                           |
+| [talk](examples/talk)                               | [talk.wasm](examples/talk.wasm)       | Talk with a GPT-2 bot                                                                                                           |
+| [talk-llama](examples/talk-llama)                   |                                       | Talk with a LLaMA bot                                                                                                           |
 | [whisper.objc](examples/whisper.objc)               |                                       | iOS mobile application using whisper.cpp                                                                                        |
 | [whisper.swiftui](examples/whisper.swiftui)         |                                       | SwiftUI iOS / macOS application using whisper.cpp                                                                               |
 | [whisper.android](examples/whisper.android)         |                                       | Android mobile application using whisper.cpp                                                                                    |
@@ -620,7 +822,7 @@ Some of the examples are even ported to run in the browser using WebAssembly. Ch
 | [generate-karaoke.sh](examples/generate-karaoke.sh) |                                       | Helper script to easily [generate a karaoke video](https://youtu.be/uj7hVta4blM) of raw audio capture                           |
 | [livestream.sh](examples/livestream.sh)             |                                       | [Livestream audio transcription](https://github.com/ggerganov/whisper.cpp/issues/185)                                           |
 | [yt-wsp.sh](examples/yt-wsp.sh)                     |                                       | Download + transcribe and/or translate any VOD [(original)](https://gist.github.com/DaniruKun/96f763ec1a037cc92fe1a059b643b818) |
-| [wchess](examples/wchess)                           | [wchess.wasm](examples/wchess)        | Voice-controlled chess                                                                                                          |
+| [server](examples/server)                           |                                       | HTTP transcription server with OAI-like API                                                                                     |
 
 ## [Discussions](https://github.com/ggerganov/whisper.cpp/discussions)
 

Sources/whisper/module.modulemap

@@ -1,5 +0,0 @@
-module whisper [system] {
-    header "whisper.h"
-    link "whisper"
-    export *
-}

Sources/whisper/whisper.h

@@ -1,4 +0,0 @@
-#pragma once
-
-#include <whisper.h>
-

bindings/go/Makefile

@@ -14,14 +14,9 @@ GGML_METAL_PATH_RESOURCES := $(abspath ../..)
 BUILD_DIR := build
 MODELS_DIR := models
 EXAMPLES_DIR := $(wildcard examples/*)
-INCLUDE_PATH := $(abspath ../../include):$(abspath ../../ggml/include)
+INCLUDE_PATH := $(abspath ../..)
 LIBRARY_PATH := $(abspath ../..)
 
-ifeq ($(GGML_CUDA),1)
-	LIBRARY_PATH := $(LIBRARY_PATH):$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib/
-	BUILD_FLAGS := -ldflags "-extldflags '-lcudart -lcuda -lcublas'"
-endif
-
 ifeq ($(UNAME_S),Darwin)
 	EXT_LDFLAGS := -framework Foundation -framework Metal -framework MetalKit
 endif

bindings/go/README.md

@@ -62,12 +62,6 @@ This will compile a static `libwhisper.a` in a `build` folder, download a model
 make examples
 ```
 
-To build using cuda support add `GGML_CUDA=1`:
-
-```bash
-GGML_CUDA=1 make examples
-```
-
 The examples are placed in the `build` directory. Once built, you can download all the models with the following command:
 
 ```bash

bindings/go/examples/go-model-download/main.go

@@ -24,7 +24,7 @@ const (
 
 var (
 	// The models which will be downloaded, if no model is specified as an argument
-	modelNames = []string{"ggml-tiny.en", "ggml-tiny", "ggml-base.en", "ggml-base", "ggml-small.en", "ggml-small", "ggml-medium.en", "ggml-medium", "ggml-large-v1", "ggml-large-v2", "ggml-large-v3", "large-v3-turbo"}
+	modelNames = []string{"ggml-tiny.en", "ggml-tiny", "ggml-base.en", "ggml-base", "ggml-small.en", "ggml-small", "ggml-medium.en", "ggml-medium", "ggml-large-v1", "ggml-large-v2", "ggml-large-v3"}
 )
 
 var (

bindings/go/go.mod

@@ -1,10 +1,10 @@
 module github.com/ggerganov/whisper.cpp/bindings/go
 
-go 1.23
+go 1.19
 
 require (
 	github.com/go-audio/wav v1.1.0
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.8.1
 )
 
 require (

bindings/go/go.sum

@@ -1,3 +1,4 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-audio/audio v1.0.0 h1:zS9vebldgbQqktK4H0lUqWrG8P0NxCJVqcj7ZpNnwd4=
@@ -8,9 +9,15 @@ github.com/go-audio/wav v1.1.0 h1:jQgLtbqBzY7G+BM8fXF7AHUk1uHUviWS4X39d5rsL2g=
 github.com/go-audio/wav v1.1.0/go.mod h1:mpe9qfwbScEbkd8uybLuIpTgHyrISw/OTuvjUW2iGtE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

bindings/go/params.go

@@ -119,28 +119,6 @@ func (p *Params) SetAudioCtx(n int) {
 	p.audio_ctx = C.int(n)
 }
 
-func (p *Params) SetMaxContext(n int) {
-	p.n_max_text_ctx = C.int(n)
-}
-
-func (p *Params) SetBeamSize(n int) {
-	p.beam_search.beam_size = C.int(n)
-}
-
-func (p *Params) SetEntropyThold(t float32) {
-	p.entropy_thold = C.float(t)
-}
-
-func (p *Params) SetTemperature(t float32) {
-	p.temperature = C.float(t)
-}
-
-// Sets the fallback temperature incrementation
-// Pass -1.0 to disable this feature
-func (p *Params) SetTemperatureFallback(t float32) {
-	p.temperature_inc = C.float(t)
-}
-
 // Set initial prompt
 func (p *Params) SetInitialPrompt(prompt string) {
 	p.initial_prompt = C.CString(prompt)
@@ -171,10 +149,6 @@ func (p *Params) String() string {
 	str += fmt.Sprintf(" duration_ms=%d", p.duration_ms)
 	str += fmt.Sprintf(" audio_ctx=%d", p.audio_ctx)
 	str += fmt.Sprintf(" initial_prompt=%s", C.GoString(p.initial_prompt))
-	str += fmt.Sprintf(" entropy_thold=%f", p.entropy_thold)
-	str += fmt.Sprintf(" temperature=%f", p.temperature)
-	str += fmt.Sprintf(" temperature_inc=%f", p.temperature_inc)
-	str += fmt.Sprintf(" beam_size=%d", p.beam_search.beam_size)
 	if p.translate {
 		str += " translate"
 	}

bindings/go/pkg/whisper/context.go

@@ -125,32 +125,6 @@ func (context *context) SetAudioCtx(n uint) {
 	context.params.SetAudioCtx(int(n))
 }
 
-// Set maximum number of text context tokens to store
-func (context *context) SetMaxContext(n int) {
-	context.params.SetMaxContext(n)
-}
-
-// Set Beam Size
-func (context *context) SetBeamSize(n int) {
-	context.params.SetBeamSize(n)
-}
-
-// Set Entropy threshold
-func (context *context) SetEntropyThold(t float32) {
-	context.params.SetEntropyThold(t)
-}
-
-// Set Temperature
-func (context *context) SetTemperature(t float32) {
-	context.params.SetTemperature(t)
-}
-
-// Set the fallback temperature incrementation
-// Pass -1.0 to disable this feature
-func (context *context) SetTemperatureFallback(t float32) {
-	context.params.SetTemperatureFallback(t)
-}
-
 // Set initial prompt
 func (context *context) SetInitialPrompt(prompt string) {
 	context.params.SetInitialPrompt(prompt)

bindings/go/pkg/whisper/context_test.go

@@ -4,90 +4,52 @@ import (
 	"os"
 	"testing"
 
-	"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper"
+	// Packages
-	"github.com/go-audio/wav"
+	whisper "github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper"
 	assert "github.com/stretchr/testify/assert"
 )
 
-func TestSetLanguage(t *testing.T) {
+const (
-	assert := assert.New(t)
+	ModelPath  = "../../models/ggml-tiny.bin"
+	SamplePath = "../../samples/jfk.wav"
+)
 
+func Test_Whisper_000(t *testing.T) {
+	assert := assert.New(t)
+	if _, err := os.Stat(ModelPath); os.IsNotExist(err) {
+		t.Skip("Skipping test, model not found:", ModelPath)
+	}
+	if _, err := os.Stat(SamplePath); os.IsNotExist(err) {
+		t.Skip("Skipping test, sample not found:", SamplePath)
+	}
+
+	// Load model
+	model, err := whisper.New(ModelPath)
+	assert.NoError(err)
+	assert.NotNil(model)
+	assert.NoError(model.Close())
+
+	t.Log("languages=", model.Languages())
+}
+
+func Test_Whisper_001(t *testing.T) {
+	assert := assert.New(t)
+	if _, err := os.Stat(ModelPath); os.IsNotExist(err) {
+		t.Skip("Skipping test, model not found:", ModelPath)
+	}
+	if _, err := os.Stat(SamplePath); os.IsNotExist(err) {
+		t.Skip("Skipping test, sample not found:", SamplePath)
+	}
+
+	// Load model
 	model, err := whisper.New(ModelPath)
 	assert.NoError(err)
 	assert.NotNil(model)
 	defer model.Close()
 
-	context, err := model.NewContext()
+	// Get context for decoding
+	ctx, err := model.NewContext()
 	assert.NoError(err)
+	assert.NotNil(ctx)
 
-	// This returns an error since
-	// the model 'models/ggml-small.en.bin'
-	// that is loaded is not multilingual
-	err = context.SetLanguage("en")
-	assert.Error(err)
-}
-
-func TestContextModelIsMultilingual(t *testing.T) {
-	assert := assert.New(t)
-
-	model, err := whisper.New(ModelPath)
-	assert.NoError(err)
-	assert.NotNil(model)
-	defer model.Close()
-
-	context, err := model.NewContext()
-	assert.NoError(err)
-
-	isMultilingual := context.IsMultilingual()
-
-	// This returns false since
-	// the model 'models/ggml-small.en.bin'
-	// that is loaded is not multilingual
-	assert.False(isMultilingual)
-}
-
-func TestLanguage(t *testing.T) {
-	assert := assert.New(t)
-
-	model, err := whisper.New(ModelPath)
-	assert.NoError(err)
-	assert.NotNil(model)
-	defer model.Close()
-
-	context, err := model.NewContext()
-	assert.NoError(err)
-
-	// This always returns en since
-	// the model 'models/ggml-small.en.bin'
-	// that is loaded is not multilingual
-	expectedLanguage := "en"
-	actualLanguage := context.Language()
-	assert.Equal(expectedLanguage, actualLanguage)
-}
-
-func TestProcess(t *testing.T) {
-	assert := assert.New(t)
-
-	fh, err := os.Open(SamplePath)
-	assert.NoError(err)
-	defer fh.Close()
-
-	// Decode the WAV file - load the full buffer
-	dec := wav.NewDecoder(fh)
-	buf, err := dec.FullPCMBuffer()
-	assert.NoError(err)
-	assert.Equal(uint16(1), dec.NumChans)
-
-	data := buf.AsFloat32Buffer().Data
-
-	model, err := whisper.New(ModelPath)
-	assert.NoError(err)
-	assert.NotNil(model)
-	defer model.Close()
-
-	context, err := model.NewContext()
-	assert.NoError(err)
-
-	err = context.Process(data, nil, nil)
-	assert.NoError(err)
 }

bindings/go/pkg/whisper/interface.go

@@ -38,22 +38,17 @@ type Context interface {
 	IsMultilingual() bool     // Return true if the model is multilingual.
 	Language() string         // Get language
 
-	SetOffset(time.Duration)          // Set offset
+	SetOffset(time.Duration)        // Set offset
-	SetDuration(time.Duration)        // Set duration
+	SetDuration(time.Duration)      // Set duration
-	SetThreads(uint)                  // Set number of threads to use
+	SetThreads(uint)                // Set number of threads to use
-	SetSplitOnWord(bool)              // Set split on word flag
+	SetSplitOnWord(bool)            // Set split on word flag
-	SetTokenThreshold(float32)        // Set timestamp token probability threshold
+	SetTokenThreshold(float32)      // Set timestamp token probability threshold
-	SetTokenSumThreshold(float32)     // Set timestamp token sum probability threshold
+	SetTokenSumThreshold(float32)   // Set timestamp token sum probability threshold
-	SetMaxSegmentLength(uint)         // Set max segment length in characters
+	SetMaxSegmentLength(uint)       // Set max segment length in characters
-	SetTokenTimestamps(bool)          // Set token timestamps flag
+	SetTokenTimestamps(bool)        // Set token timestamps flag
-	SetMaxTokensPerSegment(uint)      // Set max tokens per segment (0 = no limit)
+	SetMaxTokensPerSegment(uint)    // Set max tokens per segment (0 = no limit)
-	SetAudioCtx(uint)                 // Set audio encoder context
+	SetAudioCtx(uint)               // Set audio encoder context
-	SetMaxContext(n int)              // Set maximum number of text context tokens to store
+	SetInitialPrompt(prompt string) // Set initial prompt
-	SetBeamSize(n int)                // Set Beam Size
-	SetEntropyThold(t float32)        // Set Entropy threshold
-	SetInitialPrompt(prompt string)   // Set initial prompt
-	SetTemperature(t float32)         // Set temperature
-	SetTemperatureFallback(t float32) // Set temperature incrementation
 
 	// Process mono audio data and return any errors.
 	// If defined, newly generated segments are passed to the

bindings/go/pkg/whisper/model_test.go

@@ -1,91 +0,0 @@
-package whisper_test
-
-import (
-	"testing"
-
-	"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper"
-	assert "github.com/stretchr/testify/assert"
-)
-
-func TestNew(t *testing.T) {
-	assert := assert.New(t)
-	t.Run("valid model path", func(t *testing.T) {
-		model, err := whisper.New(ModelPath)
-		assert.NoError(err)
-		assert.NotNil(model)
-		defer model.Close()
-
-	})
-
-	t.Run("invalid model path", func(t *testing.T) {
-		invalidModelPath := "invalid-model-path.bin"
-		model, err := whisper.New(invalidModelPath)
-		assert.Error(err)
-		assert.Nil(model)
-	})
-}
-
-func TestClose(t *testing.T) {
-	assert := assert.New(t)
-
-	model, err := whisper.New(ModelPath)
-	assert.NoError(err)
-	assert.NotNil(model)
-
-	err = model.Close()
-	assert.NoError(err)
-}
-
-func TestNewContext(t *testing.T) {
-	assert := assert.New(t)
-
-	model, err := whisper.New(ModelPath)
-	assert.NoError(err)
-	assert.NotNil(model)
-	defer model.Close()
-
-	context, err := model.NewContext()
-	assert.NoError(err)
-	assert.NotNil(context)
-}
-
-func TestIsMultilingual(t *testing.T) {
-	assert := assert.New(t)
-
-	model, err := whisper.New(ModelPath)
-	assert.NoError(err)
-	assert.NotNil(model)
-	defer model.Close()
-
-	isMultilingual := model.IsMultilingual()
-
-	// This returns false since
-	// the model 'models/ggml-small.en.bin'
-	// that is loaded is not multilingual
-	assert.False(isMultilingual)
-}
-
-func TestLanguages(t *testing.T) {
-	assert := assert.New(t)
-
-	model, err := whisper.New(ModelPath)
-	assert.NoError(err)
-	assert.NotNil(model)
-	defer model.Close()
-
-	expectedLanguages := []string{
-		"en", "zh", "de", "es", "ru", "ko", "fr", "ja", "pt", "tr", "pl",
-		"ca", "nl", "ar", "sv", "it", "id", "hi", "fi", "vi", "he", "uk",
-		"el", "ms", "cs", "ro", "da", "hu", "ta", "no", "th", "ur", "hr",
-		"bg", "lt", "la", "mi", "ml", "cy", "sk", "te", "fa", "lv", "bn",
-		"sr", "az", "sl", "kn", "et", "mk", "br", "eu", "is", "hy", "ne",
-		"mn", "bs", "kk", "sq", "sw", "gl", "mr", "pa", "si", "km", "sn",
-		"yo", "so", "af", "oc", "ka", "be", "tg", "sd", "gu", "am", "yi",
-		"lo", "uz", "fo", "ht", "ps", "tk", "nn", "mt", "sa", "lb", "my",
-		"bo", "tl", "mg", "as", "tt", "haw", "ln", "ha", "ba", "jw", "su",
-	}
-
-	actualLanguages := model.Languages()
-
-	assert.Equal(expectedLanguages, actualLanguages)
-}

bindings/go/pkg/whisper/util_test.go

@@ -1,6 +0,0 @@
-package whisper_test
-
-const (
-	ModelPath  = "../../models/ggml-small.en.bin"
-	SamplePath = "../../samples/jfk.wav"
-)

bindings/go/whisper.go

@@ -9,7 +9,7 @@ import (
 // CGO
 
 /*
-#cgo LDFLAGS: -lwhisper -lm -lstdc++ -fopenmp
+#cgo LDFLAGS: -lwhisper -lm -lstdc++
 #cgo darwin LDFLAGS: -framework Accelerate -framework Metal -framework Foundation -framework CoreGraphics
 #include <whisper.h>
 #include <stdlib.h>

bindings/java/README.md

@@ -67,5 +67,5 @@ copy /y ..\..\build\bin\Release\whisper.dll build\generated\resources\main\win32
 
 ## License
 
-The license for the Java bindings is the same as the license for the rest of the whisper.cpp project, which is the MIT License. See the `LICENSE` file for more details.
+The license for the Go bindings is the same as the license for the rest of the whisper.cpp project, which is the MIT License. See the `LICENSE` file for more details.
 

bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperFullParams.java

@@ -181,11 +181,11 @@ public class WhisperFullParams extends Structure {
     }
 
     /** Flag to suppress non-speech tokens. */
-    public CBool suppress_nst;
+    public CBool suppress_non_speech_tokens;
 
     /** Flag to suppress non-speech tokens. */
     public void suppressNonSpeechTokens(boolean enable) {
-        suppress_nst = enable ? CBool.TRUE : CBool.FALSE;
+        suppress_non_speech_tokens = enable ? CBool.TRUE : CBool.FALSE;
     }
 
     /** Initial decoding temperature. */
@@ -315,7 +315,7 @@ public class WhisperFullParams extends Structure {
                 "print_special", "print_progress", "print_realtime", "print_timestamps",  "token_timestamps",
                 "thold_pt", "thold_ptsum", "max_len", "split_on_word", "max_tokens", "audio_ctx",
                 "tdrz_enable", "suppress_regex", "initial_prompt", "prompt_tokens", "prompt_n_tokens", "language", "detect_language",
-                "suppress_blank", "suppress_nst", "temperature", "max_initial_ts", "length_penalty",
+                "suppress_blank", "suppress_non_speech_tokens", "temperature", "max_initial_ts", "length_penalty",
                 "temperature_inc", "entropy_thold", "logprob_thold", "no_speech_thold", "greedy", "beam_search",
                 "new_segment_callback", "new_segment_callback_user_data",
                 "progress_callback", "progress_callback_user_data",

bindings/javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "whisper.cpp",
-  "version": "1.7.4",
+  "version": "1.6.2",
   "description": "Whisper speech recognition",
   "main": "whisper.js",
   "scripts": {

bindings/ruby/.gitignore

@@ -1,3 +0,0 @@
-LICENSE
-pkg/
-lib/whisper.*

bindings/ruby/README.md

@@ -1,243 +0,0 @@
-whispercpp
-==========
-
-![whisper.cpp](https://user-images.githubusercontent.com/1991296/235238348-05d0f6a4-da44-4900-a1de-d0707e75b763.jpeg)
-
-Ruby bindings for [whisper.cpp][], an interface of automatic speech recognition model.
-
-Installation
-------------
-
-Install the gem and add to the application's Gemfile by executing:
-
-    $ bundle add whispercpp
-
-If bundler is not being used to manage dependencies, install the gem by executing:
-
-    $ gem install whispercpp
-
-Usage
------
-
-```ruby
-require "whisper"
-
-whisper = Whisper::Context.new("base")
-
-params = Whisper::Params.new
-params.language = "en"
-params.offset = 10_000
-params.duration = 60_000
-params.max_text_tokens = 300
-params.translate = true
-params.print_timestamps = false
-params.initial_prompt = "Initial prompt here."
-
-whisper.transcribe("path/to/audio.wav", params) do |whole_text|
-  puts whole_text
-end
-
-```
-
-### Preparing model ###
-
-Some models are prepared up-front:
-
-```ruby
-base_en = Whisper::Model.pre_converted_models["base.en"]
-whisper = Whisper::Context.new(base_en)
-```
-
-At first time you use a model, it is downloaded automatically. After that, downloaded cached file is used. To clear cache, call `#clear_cache`:
-
-```ruby
-Whisper::Model.pre_converted_models["base"].clear_cache
-```
-
-You also can use shorthand for pre-converted models:
-
-```ruby
-whisper = Whisper::Context.new("base.en")
-```
-
-You can see the list of prepared model names by `Whisper::Model.pre_converted_models.keys`:
-
-```ruby
-puts Whisper::Model.pre_converted_models.keys
-# tiny
-# tiny.en
-# tiny-q5_1
-# tiny.en-q5_1
-# tiny-q8_0
-# base
-# base.en
-# base-q5_1
-# base.en-q5_1
-# base-q8_0
-#   :
-#   :
-```
-
-You can also use local model files you prepared:
-
-```ruby
-whisper = Whisper::Context.new("path/to/your/model.bin")
-```
-
-Or, you can download model files:
-
-```ruby
-whisper = Whisper::Context.new("https://example.net/uri/of/your/model.bin")
-# Or
-whisper = Whisper::Context.new(URI("https://example.net/uri/of/your/model.bin"))
-```
-
-See [models][] page for details.
-
-### Preparing audio file ###
-
-Currently, whisper.cpp accepts only 16-bit WAV files.
-
-API
----
-
-### Segments ###
-
-Once `Whisper::Context#transcribe` called, you can retrieve segments by `#each_segment`:
-
-```ruby
-def format_time(time_ms)
-  sec, decimal_part = time_ms.divmod(1000)
-  min, sec = sec.divmod(60)
-  hour, min = min.divmod(60)
-  "%02d:%02d:%02d.%03d" % [hour, min, sec, decimal_part]
-end
-
-whisper.transcribe("path/to/audio.wav", params)
-
-whisper.each_segment.with_index do |segment, index|
-  line = "[%{nth}: %{st} --> %{ed}] %{text}" % {
-    nth: index + 1,
-    st: format_time(segment.start_time),
-    ed: format_time(segment.end_time),
-    text: segment.text
-  }
-  line << " (speaker turned)" if segment.speaker_next_turn?
-  puts line
-end
-
-```
-
-You can also add hook to params called on new segment:
-
-```ruby
-# Add hook before calling #transcribe
-params.on_new_segment do |segment|
-  line = "[%{st} --> %{ed}] %{text}" % {
-    st: format_time(segment.start_time),
-    ed: format_time(segment.end_time),
-    text: segment.text
-  }
-  line << " (speaker turned)" if segment.speaker_next_turn?
-  puts line
-end
-
-whisper.transcribe("path/to/audio.wav", params)
-
-```
-
-### Models ###
-
-You can see model information:
-
-```ruby
-whisper = Whisper::Context.new("base")
-model = whisper.model
-
-model.n_vocab # => 51864
-model.n_audio_ctx # => 1500
-model.n_audio_state # => 512
-model.n_audio_head # => 8
-model.n_audio_layer # => 6
-model.n_text_ctx # => 448
-model.n_text_state # => 512
-model.n_text_head # => 8
-model.n_text_layer # => 6
-model.n_mels # => 80
-model.ftype # => 1
-model.type # => "base"
-
-```
-
-### Logging ###
-
-You can set log callback:
-
-```ruby
-prefix = "[MyApp] "
-log_callback = ->(level, buffer, user_data) {
-  case level
-  when Whisper::LOG_LEVEL_NONE
-    puts "#{user_data}none: #{buffer}"
-  when Whisper::LOG_LEVEL_INFO
-    puts "#{user_data}info: #{buffer}"
-  when Whisper::LOG_LEVEL_WARN
-    puts "#{user_data}warn: #{buffer}"
-  when Whisper::LOG_LEVEL_ERROR
-    puts "#{user_data}error: #{buffer}"
-  when Whisper::LOG_LEVEL_DEBUG
-    puts "#{user_data}debug: #{buffer}"
-  when Whisper::LOG_LEVEL_CONT
-    puts "#{user_data}same to previous: #{buffer}"
-  end
-}
-Whisper.log_set log_callback, prefix
-```
-
-Using this feature, you are also able to suppress log:
-
-```ruby
-Whisper.log_set ->(level, buffer, user_data) {
-  # do nothing
-}, nil
-Whisper::Context.new("base")
-```
-
-### Low-level API to transcribe ###
-
-You can also call `Whisper::Context#full` and `#full_parallel` with a Ruby array as samples. Although `#transcribe` with audio file path is recommended because it extracts PCM samples in C++ and is fast, `#full` and `#full_parallel` give you flexibility.
-
-```ruby
-require "whisper"
-require "wavefile"
-
-reader = WaveFile::Reader.new("path/to/audio.wav", WaveFile::Format.new(:mono, :float, 16000))
-samples = reader.enum_for(:each_buffer).map(&:samples).flatten
-
-whisper = Whisper::Context.new("base")
-whisper.full(Whisper::Params.new, samples)
-whisper.each_segment do |segment|
-  puts segment.text
-end
-```
-
-The second argument `samples` may be an array, an object with `length` and `each` method, or a MemoryView. If you can prepare audio data as C array and export it as a MemoryView, whispercpp accepts and works with it with zero copy.
-
-Development
------------
-
-    % git clone https://github.com/ggerganov/whisper.cpp.git
-    % cd whisper.cpp/bindings/ruby
-    % rake test
-
-First call of `rake test` builds an extension and downloads a model for testing. After that, you add tests in `tests` directory and modify `ext/ruby_whisper.cpp`.
-
-If something seems wrong on build, running `rake clean` solves some cases.
-
-License
--------
-
-The same to [whisper.cpp][].
-
-[whisper.cpp]: https://github.com/ggerganov/whisper.cpp
-[models]: https://github.com/ggerganov/whisper.cpp/tree/master/models

bindings/ruby/Rakefile

@@ -1,64 +1,12 @@
 require 'rake/clean'
-require "bundler/gem_tasks"
+  require 'rubygems/package'
-require "rake/testtask"
-require_relative "extsources"
 
-SOURCES = FileList[]
+desc 'Build gem'
-
+task :package do
-EXTSOURCES.each do |src|
+  spec_source = File.read File.join(File.dirname(__FILE__),'whispercpp.gemspec')
-  basename = src.pathmap("%f")
+  spec = nil
-  dest = basename == "LICENSE" ? basename : src.pathmap("%{../..,ext}p")
+  # see: http://gist.github.com/16215
-  dir = dest.pathmap("%d")
+  Thread.new { spec = eval("#{spec_source}") }.join
-  file src
+  spec.validate
-  directory dir
+  Gem::Package.build(spec)
-  file dest => [src, dir] do |t|
-    cp t.source, t.name
-  end
-  SOURCES.include dest
 end
-
-CLEAN.include SOURCES
-CLEAN.include FileList["ext/*.o", "ext/*.metal", "ext/whisper.{so,bundle,dll}"]
-
-task build: ["ext/Makefile", "ext/ruby_whisper.h", "ext/ruby_whisper.cpp", "whispercpp.gemspec"]
-
-directory "pkg"
-CLOBBER.include "pkg"
-
-LIB_NAME = "whisper".ext(RbConfig::CONFIG["DLEXT"])
-SO_FILE = File.join("ext", LIB_NAME)
-LIB_FILE = File.join("lib", LIB_NAME)
-
-file "ext/Makefile" => ["ext/extconf.rb", "ext/ruby_whisper.h", "ext/ruby_whisper.cpp"] + SOURCES do |t|
-  Dir.chdir "ext" do
-    ruby "extconf.rb"
-  end
-end
-
-file SO_FILE => "ext/Makefile" do |t|
-  Dir.chdir "ext" do
-    sh "make"
-  end
-end
-CLEAN.include SO_FILE
-
-directory "lib"
-file LIB_FILE => [SO_FILE, "lib"] do |t|
-  copy t.source, t.name
-end
-CLEAN.include LIB_FILE
-
-Rake::TestTask.new do |t|
-  t.test_files = FileList["tests/test_*.rb"]
-end
-
-TEST_MEMORY_VIEW = "tests/jfk_reader/jfk_reader.#{RbConfig::CONFIG['DLEXT']}"
-file TEST_MEMORY_VIEW => "tests/jfk_reader/jfk_reader.c" do |t|
-  Dir.chdir "tests/jfk_reader" do
-    ruby "extconf.rb"
-    sh "make"
-  end
-end
-CLEAN.include "tests/jfk_reader/jfk_reader.{o,#{RbConfig::CONFIG['DLEXT']}}"
-
-task test: [LIB_FILE, TEST_MEMORY_VIEW]

bindings/ruby/ext/.gitignore

@@ -1,13 +1,9 @@
 Makefile
-whisper.so
+ggml.c
+ggml.h
+ggml-alloc.c
+ggml-alloc.h
 whisper.bundle
-whisper.dll
+whisper.cpp
-scripts/get-flags.mk
+whisper.h
-*.o
+dr_wav.h
-*.c
-*.cpp
-*.h
-*.m
-*.metal
-!ruby_whisper.cpp
-!ruby_whisper.h

bindings/ruby/ext/cpu.mk

@@ -1,9 +0,0 @@
-ggml/src/ggml-cpu/ggml-cpu-cpp.o: \
-	ggml/src/ggml-cpu/ggml-cpu.cpp \
-	ggml/include/ggml-backend.h \
-	ggml/include/ggml.h \
-	ggml/include/ggml-alloc.h \
-	ggml/src/ggml-backend-impl.h \
-	ggml/include/ggml-cpu.h \
-	ggml/src/ggml-impl.h
-	$(CXX) $(CXXFLAGS)   -c $< -o $@

bindings/ruby/ext/extconf.rb

@@ -1,10 +1,23 @@
 require 'mkmf'
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','whisper.cpp')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','whisper.h')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','whisper-mel.hpp')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml.h')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml.c')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml-impl.h')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml-alloc.h')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml-alloc.c')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml-backend-impl.h')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml-backend.h')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml-backend.c')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml-common.h')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml-quants.h')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','ggml-quants.c')} .")
+system("cp #{File.join(File.dirname(__FILE__),'..','..','..','examples','dr_wav.h')} .")
+
 
 # need to use c++ compiler flags
-$CXXFLAGS << ' -std=c++17'
+$CXXFLAGS << ' -std=c++11'
-
-$LDFLAGS << ' -lstdc++'
-
 # Set to true when building binary gems
 if enable_config('static-stdlib', false)
   $LDFLAGS << ' -static-libgcc -static-libstdc++'
@@ -15,185 +28,4 @@ if enable_config('march-tune-native', false)
   $CXXFLAGS << ' -march=native -mtune=native'
 end
 
-if ENV['WHISPER_METAL']
-  $GGML_METAL ||= true
-  $DEPRECATE_WARNING ||= true
-end
-
-$UNAME_S = `uname -s`.chomp
-$UNAME_P = `uname -p`.chomp
-$UNAME_M = `uname -m`.chomp
-
-if $UNAME_S == 'Darwin'
-  unless ENV['GGML_NO_METAL']
-    $GGML_METAL ||= true
-  end
-  $GGML_NO_OPENMP ||= true
-end
-
-if $GGML_METAL
-  $GGML_METAL_EMBED_LIBRARY = true
-end
-
-$MK_CPPFLAGS = '-Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -Iexamples'
-$MK_CFLAGS   = '-std=c11   -fPIC'
-$MK_CXXFLAGS = '-std=c++17 -fPIC'
-$MK_NVCCFLAGS = '-std=c++17'
-$MK_LDFLAGS = ''
-
-$OBJ_GGML = []
-$OBJ_WHISPER = []
-$OBJ_COMMON = []
-$OBJ_SDL = []
-
-$MK_CPPFLAGS << ' -D_XOPEN_SOURCE=600'
-
-if $UNAME_S == 'Linux'
-  $MK_CPPFLAGS << ' -D_GNU_SOURCE'
-end
-
-if $UNAME_S == 'Darwin'
-  $MK_CPPFLAGS << ' -D_DARWIN_C_SOURCE'
-end
-
-if ENV['WHISPER_DEBUG']
-  $MK_CFLAGS    << ' -O0 -g'
-  $MK_CXXFLAGS  << ' -O0 -g'
-  $MK_LDFLAGS   << ' -g'
-  $MK_NVCCFLAGS << ' -O0 -g'
-else
-  $MK_CPPFLAGS   << ' -DNDEBUG'
-  $MK_CFLAGS     << ' -O3'
-  $MK_CXXFLAGS   << ' -O3'
-  $MK_NVCCFLAGS  << ' -O3'
-end
-
-$WARN_FLAGS =
-  ' -Wall' <<
-  ' -Wextra' <<
-  ' -Wpedantic' <<
-  ' -Wcast-qual' <<
-  ' -Wno-unused-function'
-
-$MK_CFLAGS <<
-  $WARN_FLAGS <<
-  ' -Wshadow' <<
-  ' -Wstrict-prototypes' <<
-  ' -Wpointer-arith' <<
-  ' -Wmissing-prototypes' <<
-  ' -Werror=implicit-int' <<
-  ' -Werror=implicit-function-declaration'
-
-$MK_CXXFLAGS <<
-  $WARN_FLAGS <<
-  ' -Wmissing-declarations' <<
-  ' -Wmissing-noreturn'
-
-unless `#{cc_command} #{$LDFLAGS} -Wl,-v 2>&1`.chomp.include? 'dyld-1015.7'
-  $MK_CPPFLAGS << ' -DHAVE_BUGGY_APPLE_LINKER'
-end
-
-if %w[Linux Darwin FreeBSD NetBSD OpenBSD Haiku].include? $UNAME_S
-  $MK_CFLAGS   << ' -pthread'
-  $MK_CXXFLAGS << ' -pthread'
-end
-
-unless $_WIN32
-  $DSO_EXT = '.so'
-else
-  $DSO_EXT = '.dll'
-end
-
-unless ENV['RISCV']
-  if %w[x86_64 i686 amd64].include? $UNAME_M
-    $HOST_CXXFLAGS ||= ''
-
-    $MK_CFLAGS     << ' -march=native -mtune=native'
-    $HOST_CXXFLAGS << ' -march=native -mtune=native'
-  end
-else
-  $MK_CFLAGS   << ' -march=rv64gcv -mabi=lp64d'
-  $MK_CXXFLAGS << ' -march=rv64gcv -mabi=lp64d'
-end
-
-unless ENV['GGML_NO_ACCELERATE']
-  if $UNAME_S == 'Darwin'
-    $MK_CPPFLAGS << ' -DGGML_USE_ACCELERATE -DGGML_USE_BLAS -DGGML_BLAS_USE_ACCELERATE'
-    $MK_CPPFLAGS << ' -DACCELERATE_NEW_LAPACK'
-    $MK_CPPFLAGS << ' -DACCELERATE_LAPACK_ILP64'
-    $MK_LDFLAGS  << ' -framework Accelerate'
-    $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
-  end
-end
-
-if ENV['GGML_OPENBLAS']
-  $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas`.chomp}"
-  $MK_CFLAGS   << " #{`pkg-config --cflags-only-other openblas)`.chomp}"
-  $MK_LDFLAGS  << " #{`pkg-config --libs openblas`}"
-  $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
-end
-
-if ENV['GGML_OPENBLAS64']
-  $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas64`.chomp}"
-  $MK_CFLAGS   << " #{`pkg-config --cflags-only-other openblas64)`.chomp}"
-  $MK_LDFLAGS  << " #{`pkg-config --libs openblas64`}"
-  $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
-end
-
-if $GGML_METAL
-  $MK_CPPFLAGS << ' -DGGML_USE_METAL'
-  $MK_LDFLAGS  << ' -framework Foundation -framework Metal -framework MetalKit'
-  $OBJ_GGML    << 'ggml/src/ggml-metal/ggml-metal.o'
-
-  if ENV['GGML_METAL_NDEBUG']
-    $MK_CPPFLAGS << ' -DGGML_METAL_NDEBUG'
-  end
-
-  if $GGML_METAL_EMBED_LIBRARY
-    $MK_CPPFLAGS << ' -DGGML_METAL_EMBED_LIBRARY'
-    $OBJ_GGML    << 'ggml/src/ggml-metal/ggml-metal-embed.o'
-  end
-end
-
-$OBJ_GGML <<
-  'ggml/src/ggml.o' <<
-  'ggml/src/ggml-alloc.o' <<
-  'ggml/src/ggml-backend.o' <<
-  'ggml/src/ggml-backend-reg.o' <<
-  'ggml/src/ggml-opt.o' <<
-  'ggml/src/ggml-quants.o' <<
-  'ggml/src/ggml-threading.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-cpp.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-aarch64.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-hbm.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-quants.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-traits.o'
-
-$OBJ_WHISPER <<
-  'src/whisper.o'
-
-$objs = $OBJ_GGML + $OBJ_WHISPER + $OBJ_COMMON + $OBJ_SDL
-$objs << "ruby_whisper.o"
-
-$CPPFLAGS  = "#{$MK_CPPFLAGS} #{$CPPFLAGS}"
-$CFLAGS    = "#{$CPPFLAGS} #{$MK_CFLAGS} #{$GF_CFLAGS} #{$CFLAGS}"
-$BASE_CXXFLAGS = "#{$MK_CXXFLAGS} #{$CXXFLAGS}"
-$CXXFLAGS  = "#{$BASE_CXXFLAGS} #{$HOST_CXXFLAGS} #{$GF_CXXFLAGS} #{$CPPFLAGS}"
-$NVCCFLAGS = "#{$MK_NVCCFLAGS} #{$NVCCFLAGS}"
-$LDFLAGS   = "#{$MK_LDFLAGS} #{$LDFLAGS}"
-
 create_makefile('whisper')
-
-File.open 'Makefile', 'a' do |file|
-  file.puts 'include scripts/get-flags.mk'
-  file.puts 'include cpu.mk'
-
-  if $GGML_METAL
-    file.puts 'include metal.mk'
-
-    if $GGML_METAL_EMBED_LIBRARY
-      file.puts 'include metal-embed.mk'
-    end
-  end
-end

bindings/ruby/ext/ggml-backend-impl.h

@@ -0,0 +1,141 @@
+#pragma once
+
+// ggml-backend internal header
+
+#include "ggml-backend.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+    //
+    // Backend buffer
+    //
+
+    // buffer type
+    typedef void * ggml_backend_buffer_type_context_t;
+
+    struct ggml_backend_buffer_type_i {
+        const char *          (*GGML_CALL get_name)        (ggml_backend_buffer_type_t buft);
+        ggml_backend_buffer_t (*GGML_CALL alloc_buffer)    (ggml_backend_buffer_type_t buft, size_t size);
+        size_t                (*GGML_CALL get_alignment)   (ggml_backend_buffer_type_t buft); // tensor alignment
+        size_t                (*GGML_CALL get_max_size)    (ggml_backend_buffer_type_t buft); // allocation max size
+        size_t                (*GGML_CALL get_alloc_size)  (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); // data size needed to allocate the tensor, including padding
+        bool                  (*GGML_CALL supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend
+        // check if tensor data is in host memory
+        // should be equivalent to supports_backend(buft, ggml_backend_cpu_init())
+        bool                  (*GGML_CALL is_host)         (ggml_backend_buffer_type_t buft);
+    };
+
+    struct ggml_backend_buffer_type {
+        struct ggml_backend_buffer_type_i  iface;
+        ggml_backend_buffer_type_context_t context;
+    };
+
+    // buffer
+    typedef void * ggml_backend_buffer_context_t;
+
+    struct ggml_backend_buffer_i {
+        const char * (*GGML_CALL get_name)   (ggml_backend_buffer_t buffer);
+        void         (*GGML_CALL free_buffer)(ggml_backend_buffer_t buffer);
+        void *       (*GGML_CALL get_base)   (ggml_backend_buffer_t buffer);
+        void         (*GGML_CALL init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
+        void         (*GGML_CALL set_tensor) (ggml_backend_buffer_t buffer,       struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+        void         (*GGML_CALL get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+        bool         (*GGML_CALL cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst); // dst is in the buffer, src may be in any buffer
+        void         (*GGML_CALL clear)      (ggml_backend_buffer_t buffer, uint8_t value);
+        void         (*GGML_CALL reset)      (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras
+    };
+
+    struct ggml_backend_buffer {
+        struct ggml_backend_buffer_i  iface;
+        ggml_backend_buffer_type_t    buft;
+        ggml_backend_buffer_context_t context;
+        size_t size;
+        enum ggml_backend_buffer_usage usage;
+    };
+
+    GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(
+                   ggml_backend_buffer_type_t      buft,
+            struct ggml_backend_buffer_i           iface,
+                   ggml_backend_buffer_context_t   context,
+                   size_t                          size);
+
+    // do not use directly, use ggml_backend_tensor_copy instead
+    bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
+
+    // buffer that contains a collection of buffers
+    GGML_CALL ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
+    GGML_CALL bool                  ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
+    GGML_CALL void                  ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
+
+    //
+    // Backend
+    //
+
+    typedef void * ggml_backend_context_t;
+
+    struct ggml_backend_i {
+        const char * (*GGML_CALL get_name)(ggml_backend_t backend);
+
+        void (*GGML_CALL free)(ggml_backend_t backend);
+
+        // buffer allocation
+        ggml_backend_buffer_type_t (*GGML_CALL get_default_buffer_type)(ggml_backend_t backend);
+
+        // (optional) asynchronous tensor data access
+        void (*GGML_CALL set_tensor_async)(ggml_backend_t backend,       struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+        void (*GGML_CALL get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+        bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
+
+        // (optional) complete all pending operations
+        void (*GGML_CALL synchronize)(ggml_backend_t backend);
+
+        // compute graph with a plan (not used currently)
+        ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);
+        void                      (*GGML_CALL graph_plan_free)   (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+
+        // compute graph with a plan
+        enum ggml_status (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+        // compute graph without a plan (async)
+        enum ggml_status (*GGML_CALL graph_compute)     (ggml_backend_t backend, struct ggml_cgraph * cgraph);
+
+        // check if the backend supports an operation
+        bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op);
+
+        // check if the backend wants to run an operation, even if the weights are allocated in a CPU buffer
+        // these should be expensive operations with large batch sizes that may benefit from running on this backend
+        // even if the weight has to be copied from the CPU temporarily
+        bool (*GGML_CALL offload_op)(ggml_backend_t backend, const struct ggml_tensor * op);
+
+        // (optional) event synchronization
+        ggml_backend_event_t (*GGML_CALL event_new)         (ggml_backend_t backend);
+        void                 (*GGML_CALL event_free)        (ggml_backend_event_t event);
+        void                 (*GGML_CALL event_record)      (ggml_backend_event_t event);
+        void                 (*GGML_CALL event_wait)        (ggml_backend_t backend, ggml_backend_event_t event);
+        void                 (*GGML_CALL event_synchronize) (ggml_backend_event_t event);
+    };
+
+    struct ggml_backend {
+        ggml_guid_t guid;
+
+        struct ggml_backend_i iface;
+        ggml_backend_context_t context;
+    };
+
+    struct ggml_backend_event {
+        ggml_backend_t backend;
+        void * context;
+    };
+
+    //
+    // Backend registry
+    //
+
+    typedef ggml_backend_t (*GGML_CALL ggml_backend_init_fn)(const char * params, void * user_data);
+
+    GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data);
+
+#ifdef  __cplusplus
+}
+#endif

bindings/ruby/ext/ggml-backend.h

@@ -0,0 +1,233 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-alloc.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+    typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
+    typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
+    typedef struct ggml_backend_event * ggml_backend_event_t;
+    typedef struct ggml_backend * ggml_backend_t;
+    typedef void * ggml_backend_graph_plan_t;
+
+    //
+    // Backend buffer
+    //
+
+    // buffer type
+    GGML_API           const char *          ggml_backend_buft_name            (ggml_backend_buffer_type_t buft);
+    GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer    (ggml_backend_buffer_type_t buft, size_t size);
+    GGML_API           size_t                ggml_backend_buft_get_alignment   (ggml_backend_buffer_type_t buft);
+    GGML_API           size_t                ggml_backend_buft_get_max_size    (ggml_backend_buffer_type_t buft);
+    GGML_API GGML_CALL size_t                ggml_backend_buft_get_alloc_size  (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);
+    GGML_API           bool                  ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend);
+    GGML_API           bool                  ggml_backend_buft_is_host         (ggml_backend_buffer_type_t buft);
+
+    // buffer
+    enum ggml_backend_buffer_usage {
+        GGML_BACKEND_BUFFER_USAGE_ANY = 0,
+        GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
+    };
+
+    GGML_API           const char *               ggml_backend_buffer_name          (ggml_backend_buffer_t buffer);
+    GGML_API           void                       ggml_backend_buffer_free          (ggml_backend_buffer_t buffer);
+    GGML_API           void *                     ggml_backend_buffer_get_base      (ggml_backend_buffer_t buffer);
+    GGML_API           size_t                     ggml_backend_buffer_get_size      (ggml_backend_buffer_t buffer);
+    GGML_API GGML_CALL void                       ggml_backend_buffer_init_tensor   (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
+    GGML_API           size_t                     ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);
+    GGML_API           size_t                     ggml_backend_buffer_get_max_size  (ggml_backend_buffer_t buffer);
+    GGML_API           size_t                     ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
+    GGML_API           void                       ggml_backend_buffer_clear         (ggml_backend_buffer_t buffer, uint8_t value);
+    GGML_API           bool                       ggml_backend_buffer_is_host       (ggml_backend_buffer_t buffer);
+    GGML_API           void                       ggml_backend_buffer_set_usage     (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
+    GGML_API           ggml_backend_buffer_type_t ggml_backend_buffer_get_type      (ggml_backend_buffer_t buffer);
+    GGML_API           void                       ggml_backend_buffer_reset         (ggml_backend_buffer_t buffer);
+
+    //
+    // Backend
+    //
+
+    GGML_API ggml_guid_t  ggml_backend_guid(ggml_backend_t backend);
+    GGML_API const char * ggml_backend_name(ggml_backend_t backend);
+    GGML_API void         ggml_backend_free(ggml_backend_t backend);
+
+    GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);
+    GGML_API ggml_backend_buffer_t      ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);
+    GGML_API size_t                     ggml_backend_get_alignment(ggml_backend_t backend);
+    GGML_API size_t                     ggml_backend_get_max_size(ggml_backend_t backend);
+
+    GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend,       struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+    GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+
+    GGML_API GGML_CALL void ggml_backend_tensor_set(      struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+    GGML_API GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+
+    GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
+
+    GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);
+    GGML_API void                      ggml_backend_graph_plan_free  (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+
+    GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
+    GGML_API enum ggml_status ggml_backend_graph_compute      (ggml_backend_t backend, struct ggml_cgraph * cgraph);
+    GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
+    GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
+    GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);
+
+    // tensor copy between different backends
+    GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);
+
+    // asynchronous copy
+    // the copy is performed after all the currently queued operations in backend_src
+    // backend_dst will wait for the copy to complete before performing other operations
+    // automatic fallback to sync copy if async is not supported
+    GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);
+
+    // events
+    GGML_API ggml_backend_event_t   ggml_backend_event_new        (ggml_backend_t backend);
+    GGML_API void                   ggml_backend_event_free       (ggml_backend_event_t event);
+    GGML_API void                   ggml_backend_event_record     (ggml_backend_event_t event);
+    GGML_API void                   ggml_backend_event_synchronize(ggml_backend_event_t event);
+    GGML_API void                   ggml_backend_event_wait       (ggml_backend_t backend, ggml_backend_event_t event); // wait async on event
+
+    //
+    // CPU backend
+    //
+
+    GGML_API ggml_backend_t ggml_backend_cpu_init(void);
+
+    GGML_API GGML_CALL bool ggml_backend_is_cpu                (ggml_backend_t backend);
+    GGML_API           void ggml_backend_cpu_set_n_threads     (ggml_backend_t backend_cpu, int n_threads);
+    GGML_API           void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);
+
+    // Create a backend buffer from an existing pointer
+    GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
+
+    GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);
+
+#ifdef GGML_USE_CPU_HBM
+    GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);
+#endif
+
+    //
+    // Backend registry
+    //
+
+    // The backend registry is a registry of all the available backends, and allows initializing backends in a generic way
+
+    GGML_API size_t                     ggml_backend_reg_get_count(void);
+    GGML_API size_t                     ggml_backend_reg_find_by_name(const char * name);
+    GGML_API ggml_backend_t             ggml_backend_reg_init_backend_from_str(const char * backend_str); // str is name[:params]
+    GGML_API const char *               ggml_backend_reg_get_name(size_t i);
+    GGML_API ggml_backend_t             ggml_backend_reg_init_backend(size_t i, const char * params); // params is backend-specific
+    GGML_API ggml_backend_buffer_type_t ggml_backend_reg_get_default_buffer_type(size_t i);
+    GGML_API ggml_backend_buffer_t      ggml_backend_reg_alloc_buffer(size_t i, size_t size);
+
+    //
+    // Backend scheduler
+    //
+
+    // The backend scheduler allows for multiple backends to be used together
+    // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
+    // The backends are selected based on:
+    // - the backend that supports the operation
+    // - the location of the pre-allocated tensors (e.g. the weights)
+    /*
+      Example usage:
+
+        // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
+        // preferrably to run on the same backend as the buffer
+        ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+
+        sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false);
+
+        // initialize buffers from a max size graph (optional)
+        reserve_graph = build_graph(sched, max_batch_size);
+
+        // manually assign nodes to a backend (optional, should not be needed in most cases)
+        struct ggml_tensor * node = ggml_mul_mat(ctx, ...);
+        ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
+
+        ggml_backend_sched_reserve(sched, reserve_graph);
+
+        // compute
+        graph = build_graph(sched);
+        ggml_backend_sched_graph_compute(sched, graph);
+
+        // if there are graph inputs:
+        ggml_backend_sched_reset(sched);
+        ggml_backend_sched_alloc_graph(sched, graph);
+        ggml_backend_tensor_set(input_tensor, ...);
+        ggml_backend_sched_graph_compute(sched, graph);
+    }
+    */
+
+    struct ggml_backend_sched;
+    typedef struct ggml_backend_sched * ggml_backend_sched_t;
+
+    // when ask == true, the scheduler wants to know if the user wants to observe this node
+    // this allows the scheduler to batch nodes together in order to evaluate them in a single call
+    //
+    // when ask == false, the scheduler is passing the node tensor to the user for observation
+    // if the user returns false, the scheduler will cancel the graph compute
+    //
+    typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
+
+    // Initialize a backend scheduler
+    GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
+    GGML_API void                 ggml_backend_sched_free(ggml_backend_sched_t sched);
+
+    // Initialize backend buffers from a measure graph
+    GGML_API bool                 ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);
+
+    // Get the number of splits of the last graph
+    GGML_API int                  ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);
+    GGML_API int                  ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);
+
+    GGML_API size_t               ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);
+
+    GGML_API void                 ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);
+    GGML_API ggml_backend_t       ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);
+
+    // Allocate and compute graph on the backend scheduler
+    GGML_API bool                 ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
+    GGML_API enum ggml_status     ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
+    GGML_API enum ggml_status     ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
+    GGML_API void                 ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
+
+    // Reset all assignments and allocators - must be called before changing the node backends
+    GGML_API void                 ggml_backend_sched_reset(ggml_backend_sched_t sched);
+
+    // Set a callback to be called for each resulting node during graph compute
+    GGML_API void                 ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
+
+    //
+    // Utils
+    //
+
+    struct ggml_backend_graph_copy {
+        ggml_backend_buffer_t buffer;
+        struct ggml_context * ctx_allocated;
+        struct ggml_context * ctx_unallocated;
+        struct ggml_cgraph * graph;
+    };
+
+    // Copy a graph to a different backend
+    GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);
+    GGML_API void                           ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);
+
+    typedef bool (*GGML_CALL ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);
+
+    // Compare the output of two backends
+    GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);
+
+    // Tensor initialization
+    GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
+    GGML_API void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
+
+
+#ifdef  __cplusplus
+}
+#endif

bindings/ruby/ext/ggml-cuda.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef GGML_USE_HIPBLAS
+#define GGML_CUDA_NAME "ROCm"
+#define GGML_CUBLAS_NAME "hipBLAS"
+#else
+#define GGML_CUDA_NAME "CUDA"
+#define GGML_CUBLAS_NAME "cuBLAS"
+#endif
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#define GGML_CUDA_MAX_DEVICES       16
+
+// backend API
+GGML_API GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device);
+
+GGML_API GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend);
+
+// device buffer
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);
+
+// split tensor buffer that splits matrices by rows across multiple devices
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split);
+
+// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);
+
+GGML_API GGML_CALL int  ggml_backend_cuda_get_device_count(void);
+GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);
+GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
+
+GGML_API GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);
+GGML_API GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer);
+
+#ifdef  __cplusplus
+}
+#endif

bindings/ruby/ext/ggml-impl.h

@@ -0,0 +1,272 @@
+#pragma once
+
+#include "ggml.h"
+
+// GGML internal header
+
+#include <assert.h>
+#include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/
+#include <stddef.h>
+#include <stdbool.h>
+#include <string.h> // memcpy
+#include <math.h>   // fabsf
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// static_assert should be a #define, but if it's not,
+// fall back to the _Static_assert C11 keyword.
+// if C99 - static_assert is noop
+// ref: https://stackoverflow.com/a/53923785/4039976
+#ifndef __cplusplus
+#ifndef static_assert
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)
+#define static_assert(cond, msg) _Static_assert(cond, msg)
+#else
+#define static_assert(cond, msg) struct global_scope_noop_trick
+#endif
+#endif
+#endif
+
+// __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512
+#if defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))
+#ifndef __FMA__
+#define __FMA__
+#endif
+#ifndef __F16C__
+#define __F16C__
+#endif
+#endif
+
+// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available
+#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__))
+#ifndef __SSE3__
+#define __SSE3__
+#endif
+#ifndef __SSSE3__
+#define __SSSE3__
+#endif
+#endif
+
+// 16-bit float
+// on Arm, we use __fp16
+// on x86, we use uint16_t
+#if defined(__ARM_NEON) && !defined(_MSC_VER)
+
+// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:
+//
+//   $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/
+//
+#include <arm_neon.h>
+
+typedef __fp16 ggml_fp16_internal_t;
+
+#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
+#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)
+
+#define GGML_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
+
+static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {
+    ggml_fp16_internal_t tmp;
+    memcpy(&tmp, &h, sizeof(ggml_fp16_t));
+    return (float)tmp;
+}
+
+static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {
+    ggml_fp16_t res;
+    ggml_fp16_internal_t tmp = f;
+    memcpy(&res, &tmp, sizeof(ggml_fp16_t));
+    return res;
+}
+
+#else
+
+typedef uint16_t ggml_fp16_internal_t;
+
+#ifdef __wasm_simd128__
+#include <wasm_simd128.h>
+#else
+#ifdef __POWER9_VECTOR__
+#include <altivec.h>
+#undef bool
+#define bool _Bool
+#else
+#if defined(_MSC_VER) || defined(__MINGW32__)
+#include <intrin.h>
+#else
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) || defined(__SSE3__)
+#if !defined(__riscv)
+#include <immintrin.h>
+#endif
+#endif
+#endif
+#endif
+#endif
+
+#ifdef __riscv_v_intrinsic
+#include <riscv_vector.h>
+#endif
+
+#ifdef __F16C__
+
+#ifdef _MSC_VER
+#define GGML_COMPUTE_FP16_TO_FP32(x) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)))
+#define GGML_COMPUTE_FP32_TO_FP16(x) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0)
+#else
+#define GGML_COMPUTE_FP16_TO_FP32(x) _cvtsh_ss(x)
+#define GGML_COMPUTE_FP32_TO_FP16(x) _cvtss_sh(x, 0)
+#endif
+
+#elif defined(__POWER9_VECTOR__)
+
+#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
+#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)
+/* the inline asm below is about 12% faster than the lookup method */
+#define GGML_FP16_TO_FP32(x) GGML_COMPUTE_FP16_TO_FP32(x)
+#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)
+
+static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {
+    register float f;
+    register double d;
+    __asm__(
+        "mtfprd %0,%2\n"
+        "xscvhpdp %0,%0\n"
+        "frsp %1,%0\n" :
+        /* temp */ "=d"(d),
+        /* out */  "=f"(f):
+        /* in */   "r"(h));
+    return f;
+}
+
+static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {
+    register double d;
+    register ggml_fp16_t r;
+    __asm__( /* xscvdphp can work on double or single precision */
+        "xscvdphp %0,%2\n"
+        "mffprd %1,%0\n" :
+        /* temp */ "=d"(d),
+        /* out */  "=r"(r):
+        /* in */   "f"(f));
+    return r;
+}
+
+#else
+
+// FP16 <-> FP32
+// ref: https://github.com/Maratyszcza/FP16
+
+static inline float fp32_from_bits(uint32_t w) {
+    union {
+        uint32_t as_bits;
+        float as_value;
+    } fp32;
+    fp32.as_bits = w;
+    return fp32.as_value;
+}
+
+static inline uint32_t fp32_to_bits(float f) {
+    union {
+        float as_value;
+        uint32_t as_bits;
+    } fp32;
+    fp32.as_value = f;
+    return fp32.as_bits;
+}
+
+static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {
+    const uint32_t w = (uint32_t) h << 16;
+    const uint32_t sign = w & UINT32_C(0x80000000);
+    const uint32_t two_w = w + w;
+
+    const uint32_t exp_offset = UINT32_C(0xE0) << 23;
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
+    const float exp_scale = 0x1.0p-112f;
+#else
+    const float exp_scale = fp32_from_bits(UINT32_C(0x7800000));
+#endif
+    const float normalized_value = fp32_from_bits((two_w >> 4) + exp_offset) * exp_scale;
+
+    const uint32_t magic_mask = UINT32_C(126) << 23;
+    const float magic_bias = 0.5f;
+    const float denormalized_value = fp32_from_bits((two_w >> 17) | magic_mask) - magic_bias;
+
+    const uint32_t denormalized_cutoff = UINT32_C(1) << 27;
+    const uint32_t result = sign |
+        (two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value));
+    return fp32_from_bits(result);
+}
+
+static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
+    const float scale_to_inf = 0x1.0p+112f;
+    const float scale_to_zero = 0x1.0p-110f;
+#else
+    const float scale_to_inf = fp32_from_bits(UINT32_C(0x77800000));
+    const float scale_to_zero = fp32_from_bits(UINT32_C(0x08800000));
+#endif
+    float base = (fabsf(f) * scale_to_inf) * scale_to_zero;
+
+    const uint32_t w = fp32_to_bits(f);
+    const uint32_t shl1_w = w + w;
+    const uint32_t sign = w & UINT32_C(0x80000000);
+    uint32_t bias = shl1_w & UINT32_C(0xFF000000);
+    if (bias < UINT32_C(0x71000000)) {
+        bias = UINT32_C(0x71000000);
+    }
+
+    base = fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base;
+    const uint32_t bits = fp32_to_bits(base);
+    const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00);
+    const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF);
+    const uint32_t nonsign = exp_bits + mantissa_bits;
+    return (sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign);
+}
+
+#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)
+#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)
+
+#endif // __F16C__
+
+#endif // __ARM_NEON
+
+// precomputed f32 table for f16 (256 KB)
+// defined in ggml.c, initialized in ggml_init()
+extern float ggml_table_f32_f16[1 << 16];
+
+// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32,
+// so we define GGML_FP16_TO_FP32 and GGML_FP32_TO_FP16 elsewhere for NEON.
+// This is also true for POWER9.
+#if !defined(GGML_FP16_TO_FP32)
+inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) {
+    uint16_t s;
+    memcpy(&s, &f, sizeof(uint16_t));
+    return ggml_table_f32_f16[s];
+}
+
+#define GGML_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x)
+#endif
+
+#if !defined(GGML_FP32_TO_FP16)
+#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)
+#endif
+
+#define GGML_HASHTABLE_FULL ((size_t)-1)
+#define GGML_HASHTABLE_ALREADY_EXISTS ((size_t)-2)
+
+struct ggml_hash_set ggml_hash_set_new(size_t size);
+
+bool   ggml_hash_contains      (const struct ggml_hash_set hash_set, struct ggml_tensor * key);
+
+// returns GGML_HASHTABLE_FULL if table is full, otherwise the current index of the key or where it should be inserted
+size_t ggml_hash_find          (const struct ggml_hash_set hash_set, struct ggml_tensor * key);
+
+// returns GGML_HASHTABLE_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full
+size_t ggml_hash_insert        (      struct ggml_hash_set hash_set, struct ggml_tensor * key);
+
+// return index, asserts if table is full
+size_t ggml_hash_find_or_insert(      struct ggml_hash_set hash_set, struct ggml_tensor * key);
+
+#ifdef __cplusplus
+}
+#endif

bindings/ruby/ext/ggml-kompute.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct ggml_vk_device {
+    int index;
+    int type; // same as VkPhysicalDeviceType
+    size_t heapSize;
+    const char * name;
+    const char * vendor;
+    int subgroupSize;
+    uint64_t bufferAlignment;
+    uint64_t maxAlloc;
+};
+
+struct ggml_vk_device * ggml_vk_available_devices(size_t memoryRequired, size_t * count);
+bool ggml_vk_get_device(struct ggml_vk_device * device, size_t memoryRequired, const char * name);
+bool ggml_vk_has_vulkan(void);
+bool ggml_vk_has_device(void);
+struct ggml_vk_device ggml_vk_current_device(void);
+
+//
+// backend API
+//
+
+// forward declaration
+typedef struct ggml_backend * ggml_backend_t;
+
+GGML_API ggml_backend_t ggml_backend_kompute_init(int device);
+
+GGML_API bool ggml_backend_is_kompute(ggml_backend_t backend);
+
+GGML_API ggml_backend_buffer_type_t ggml_backend_kompute_buffer_type(int device);
+
+#ifdef __cplusplus
+}
+#endif

bindings/ruby/ext/ggml-metal.h

@@ -0,0 +1,66 @@
+// An interface allowing to compute ggml_cgraph with Metal
+//
+// This is a fully functional interface that extends ggml with GPU support for Apple devices.
+// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, OpenCL, etc.)
+//
+// How it works?
+//
+// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this
+// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you
+// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.)
+//
+// You only need to make sure that all memory buffers that you used during the graph creation
+// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is
+// used during the graph evaluation to determine the arguments of the compute kernels.
+//
+// Synchronization between device and host memory (for example for input and output tensors)
+// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions.
+//
+
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#include <stddef.h>
+#include <stdbool.h>
+
+// max memory buffers that can be mapped to the device
+#define GGML_METAL_MAX_BUFFERS 64
+
+struct ggml_tensor;
+struct ggml_cgraph;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//
+// backend API
+// user-code should use only these functions
+//
+
+GGML_API void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data);
+
+GGML_API ggml_backend_t ggml_backend_metal_init(void);
+
+GGML_API bool ggml_backend_is_metal(ggml_backend_t backend);
+
+GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size);
+
+GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);
+
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);
+
+// helper to check if the device supports a specific family
+// ideally, the user code should be doing these checks
+// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
+GGML_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family);
+
+// capture all command buffers committed the next time `ggml_backend_graph_compute` is called
+GGML_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);
+
+#ifdef __cplusplus
+}
+#endif
+

bindings/ruby/ext/ggml-opencl.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+GGML_API void ggml_cl_init(void);
+
+GGML_API void   ggml_cl_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
+GGML_API void   ggml_cl_add(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
+GGML_API bool   ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst);
+GGML_API size_t ggml_cl_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
+GGML_API void   ggml_cl_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize);
+
+// GGML_API void * ggml_cl_host_malloc(size_t size);
+// GGML_API void   ggml_cl_host_free(void * ptr);
+
+GGML_API void ggml_cl_free_data(const struct ggml_tensor* tensor);
+
+GGML_API void ggml_cl_transform_tensor(void * data, struct ggml_tensor * tensor);
+
+// backend API
+
+// GGML_API ggml_backend_t ggml_backend_opencl_init(void);
+
+// GGML_API bool ggml_backend_is_opencl(ggml_backend_t backend);
+
+GGML_API ggml_backend_buffer_type_t ggml_backend_opencl_buffer_type(void);
+// GGML_API ggml_backend_buffer_type_t ggml_backend_opencl_host_buffer_type(void);
+
+#ifdef  __cplusplus
+}
+#endif

bindings/ruby/ext/ggml-quants.h

@@ -0,0 +1,133 @@
+#pragma once
+
+#define GGML_COMMON_DECL_C
+#include "ggml-common.h"
+
+#include "ggml.h"
+
+// GGML internal header
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Quantization
+void quantize_row_q4_0_reference(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q4_1_reference(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_0_reference(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_1_reference(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_0_reference(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_1_reference(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_q2_K_reference(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q3_K_reference(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q4_K_reference(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl  * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq4_xs_reference (const float * GGML_RESTRICT x, block_iq4_xs  * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq3_s_reference  (const float * GGML_RESTRICT x, block_iq3_s   * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq2_s_reference  (const float * GGML_RESTRICT x, block_iq2_s   * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq3_s  (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+void quantize_row_iq2_s  (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
+
+// Dequantization
+void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+//void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq2_xs (const block_iq2_xs  * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq2_s  (const block_iq2_s   * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq1_s  (const block_iq1_s   * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq1_m  (const block_iq1_m   * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq4_nl (const block_iq4_nl  * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq4_xs (const block_iq4_xs  * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void dequantize_row_iq3_s  (const block_iq3_s   * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
+// Dot product
+void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq2_s_q8_K  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq1_s_q8_K  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq1_m_q8_K  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_iq3_s_q8_K  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+
+// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")
+size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq2_s  (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq1_s  (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq1_m  (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_iq3_s  (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
+
+void iq2xs_init_impl(enum ggml_type type);
+void iq2xs_free_impl(enum ggml_type type);
+void iq3xs_init_impl(int grid_size);
+void iq3xs_free_impl(int grid_size);
+
+#ifdef __cplusplus
+}
+#endif
+

bindings/ruby/ext/ggml-sycl.h

@@ -0,0 +1,49 @@
+//
+//  MIT license
+//  Copyright (C) 2024 Intel Corporation
+//  SPDX-License-Identifier: MIT
+//
+
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#define GGML_SYCL_MAX_DEVICES       48
+#define GGML_SYCL_NAME "SYCL"
+
+// backend API
+GGML_API ggml_backend_t ggml_backend_sycl_init(int device);
+
+// devide buffer
+GGML_API ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device);
+
+// split tensor buffer that splits matrices by rows across multiple devices
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split);
+
+// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
+GGML_API ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type(void);
+
+GGML_API void   ggml_backend_sycl_print_sycl_devices(void);
+GGML_API GGML_CALL void   ggml_sycl_get_gpu_list(int *id_list, int max_len);
+GGML_API GGML_CALL void   ggml_sycl_get_device_description(int device, char *description, size_t description_size);
+GGML_API GGML_CALL int   ggml_backend_sycl_get_device_count();
+GGML_API GGML_CALL void ggml_backend_sycl_get_device_memory(int device, size_t *free, size_t *total);
+GGML_API GGML_CALL int ggml_backend_sycl_get_device_index(int device_id);
+
+// TODO: these are temporary
+//       ref: https://github.com/ggerganov/llama.cpp/pull/6022#issuecomment-1992615670
+GGML_API GGML_CALL int ggml_backend_sycl_get_device_id(int device_index);
+GGML_API GGML_CALL void ggml_backend_sycl_set_single_device_mode(int main_gpu_id);
+GGML_API GGML_CALL void ggml_backend_sycl_set_mul_device_mode();
+
+// SYCL doesn't support registering host memory, keep here for reference
+// GGML_API GGML_CALL bool ggml_backend_sycl_register_host_buffer(void * buffer, size_t size);
+// GGML_API GGML_CALL void ggml_backend_sycl_unregister_host_buffer(void * buffer);
+#ifdef  __cplusplus
+}
+#endif

bindings/ruby/ext/ggml-vulkan.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#define GGML_VK_NAME "Vulkan"
+#define GGML_VK_MAX_DEVICES 16
+
+GGML_API void ggml_vk_instance_init(void);
+
+// backend API
+GGML_API GGML_CALL ggml_backend_t ggml_backend_vk_init(size_t dev_num);
+
+GGML_API GGML_CALL bool ggml_backend_is_vk(ggml_backend_t backend);
+GGML_API GGML_CALL int  ggml_backend_vk_get_device_count(void);
+GGML_API GGML_CALL void ggml_backend_vk_get_device_description(int device, char * description, size_t description_size);
+GGML_API GGML_CALL void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total);
+
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num);
+// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type(void);
+
+#ifdef  __cplusplus
+}
+#endif

bindings/ruby/ext/metal-embed.mk

@@ -1,17 +0,0 @@
-ggml/src/ggml-metal/ggml-metal-embed.o: \
-	ggml/src/ggml-metal/ggml-metal.metal \
-	ggml/src/ggml-metal/ggml-metal-impl.h \
-	ggml/src/ggml-common.h
-	@echo "Embedding Metal library"
-	@sed -e '/__embed_ggml-common.h__/r      ggml/src/ggml-common.h'                -e '/__embed_ggml-common.h__/d'      < ggml/src/ggml-metal/ggml-metal.metal           > ggml/src/ggml-metal/ggml-metal-embed.metal.tmp
-	@sed -e '/#include "ggml-metal-impl.h"/r ggml/src/ggml-metal/ggml-metal-impl.h' -e '/#include "ggml-metal-impl.h"/d' < ggml/src/ggml-metal/ggml-metal-embed.metal.tmp > ggml/src/ggml-metal/ggml-metal-embed.metal
-	$(eval TEMP_ASSEMBLY=$(shell mktemp -d))
-	@echo ".section __DATA, __ggml_metallib"                       >  $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo ".globl _ggml_metallib_start"                            >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo "_ggml_metallib_start:"                                  >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo ".incbin \"ggml/src/ggml-metal/ggml-metal-embed.metal\"" >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo ".globl _ggml_metallib_end"                              >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo "_ggml_metallib_end:"                                    >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	$(CC) $(CFLAGS) -c $(TEMP_ASSEMBLY)/ggml-metal-embed.s -o $@
-	@rm -f ${TEMP_ASSEMBLY}/ggml-metal-embed.s
-	@rmdir ${TEMP_ASSEMBLY}

bindings/ruby/ext/metal.mk

@@ -1,6 +0,0 @@
-ggml/src/ggml-metal/ggml-metal.o: \
-	ggml/src/ggml-metal/ggml-metal.m \
-	ggml/src/ggml-metal/ggml-metal-impl.h \
-	ggml/include/ggml-metal.h \
-	ggml/include/ggml.h
-	$(CC) $(CFLAGS) -c $< -o $@

bindings/ruby/ext/ruby_whisper.h

@@ -1,15 +1,8 @@
-#ifndef RUBY_WHISPER_H
+#ifndef __RUBY_WHISPER_H
-#define RUBY_WHISPER_H
+#define __RUBY_WHISPER_H
 
 #include "whisper.h"
 
-typedef struct {
-  VALUE *context;
-  VALUE user_data;
-  VALUE callback;
-  VALUE callbacks;
-} ruby_whisper_callback_container;
-
 typedef struct {
   struct whisper_context *context;
 } ruby_whisper;
@@ -17,9 +10,6 @@ typedef struct {
 typedef struct {
   struct whisper_full_params params;
   bool diarize;
-  ruby_whisper_callback_container *new_segment_callback_container;
-  ruby_whisper_callback_container *progress_callback_container;
-  ruby_whisper_callback_container *abort_callback_container;
 } ruby_whisper_params;
 
 #endif

bindings/ruby/extsources.rb

@@ -1,6 +0,0 @@
-require "yaml"
-
-sources = `git ls-files -z ../..`.split("\x0")
-paths = YAML.load_file("../../.github/workflows/bindings-ruby.yml")[true]["push"]["paths"]
-paths.delete "bindings/ruby/**"
-EXTSOURCES = (Dir.glob(paths, base: "../..").collect {|path| "../../#{path}"} << "../../LICENSE") & sources

bindings/ruby/lib/whisper/model/uri.rb

@@ -1,163 +0,0 @@
-require "uri"
-require "net/http"
-require "time"
-require "pathname"
-require "io/console/size"
-
-module Whisper
-  class Model
-    class URI
-      def initialize(uri)
-        @uri = URI(uri)
-      end
-
-      def to_path
-        cache
-        cache_path.to_path
-      end
-
-      def clear_cache
-        path = cache_path
-        path.delete if path.exist?
-      end
-
-      private
-
-      def cache_path
-        base_cache_dir/@uri.host/@uri.path[1..]
-      end
-
-      def base_cache_dir
-        base = case RUBY_PLATFORM
-               when /mswin|mingw/
-                 ENV.key?("LOCALAPPDATA") ? Pathname(ENV["LOCALAPPDATA"]) : Pathname(Dir.home)/"AppData/Local"
-               when /darwin/
-                 Pathname(Dir.home)/"Library/Caches"
-               else
-                 ENV.key?("XDG_CACHE_HOME") ? ENV["XDG_CACHE_HOME"] : Pathname(Dir.home)/".cache"
-               end
-        base/"whisper.cpp"
-      end
-
-      def cache
-        path = cache_path
-        headers = {}
-        headers["if-modified-since"] = path.mtime.httpdate if path.exist?
-        request @uri, headers
-        path
-      end
-
-      def request(uri, headers)
-        Net::HTTP.start uri.host, uri.port, use_ssl: uri.scheme == "https" do |http|
-          request = Net::HTTP::Get.new(uri, headers)
-          http.request request do |response|
-            case response
-            when Net::HTTPNotModified
-            # noop
-            when Net::HTTPOK
-              download response
-            when Net::HTTPRedirection
-              request URI(response["location"]), headers
-            else
-              return if headers.key?("if-modified-since") # Use cache file
-
-              raise "#{response.code} #{response.message}\n#{response.body}"
-            end
-          end
-        end
-      end
-
-      def download(response)
-        path = cache_path
-        path.dirname.mkpath unless path.dirname.exist?
-        downloading_path = Pathname("#{path}.downloading")
-        size = response.content_length
-        downloading_path.open "wb" do |file|
-          downloaded = 0
-          response.read_body do |chunk|
-            file << chunk
-            downloaded += chunk.bytesize
-            show_progress downloaded, size
-          end
-          $stderr.puts
-        end
-        downloading_path.rename path
-      end
-
-      def show_progress(current, size)
-        progress_rate_available = size && $stderr.tty?
-
-        unless @prev
-          @prev = Time.now
-          $stderr.puts "Downloading #{@uri} to #{cache_path}"
-        end
-
-        now = Time.now
-
-        if progress_rate_available
-          return if now - @prev < 1 && current < size
-
-          progress_width = 20
-          progress = current.to_f / size
-          arrow_length = progress * progress_width
-          arrow = "=" * (arrow_length - 1) + ">" + " " * (progress_width - arrow_length)
-          line = "[#{arrow}] (#{format_bytesize(current)} / #{format_bytesize(size)})"
-          padding = ' ' * ($stderr.winsize[1] - line.size)
-          $stderr.print "\r#{line}#{padding}"
-        else
-          return if now - @prev < 1
-
-          $stderr.print "."
-        end
-        @prev = now
-      end
-
-      def format_bytesize(bytesize)
-        return "0.0 B" if bytesize.zero?
-
-        units = %w[B KiB MiB GiB TiB]
-        exp = (Math.log(bytesize) / Math.log(1024)).to_i
-        format("%.1f %s", bytesize.to_f / 1024 ** exp, units[exp])
-      end
-    end
-
-    @pre_converted_models = %w[
-      tiny
-      tiny.en
-      tiny-q5_1
-      tiny.en-q5_1
-      tiny-q8_0
-      base
-      base.en
-      base-q5_1
-      base.en-q5_1
-      base-q8_0
-      small
-      small.en
-      small.en-tdrz
-      small-q5_1
-      small.en-q5_1
-      small-q8_0
-      medium
-      medium.en
-      medium-q5_0
-      medium.en-q5_0
-      medium-q8_0
-      large-v1
-      large-v2
-      large-v2-q5_0
-      large-v2-q8_0
-      large-v3
-      large-v3-q5_0
-      large-v3-turbo
-      large-v3-turbo-q5_0
-      large-v3-turbo-q8_0
-    ].each_with_object({}) {|name, models|
-      models[name] = URI.new("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-#{name}.bin")
-    }
-
-    class << self
-      attr_reader :pre_converted_models
-    end
-  end
-end

bindings/ruby/sig/whisper.rbs

@@ -1,153 +0,0 @@
-module Whisper
-  interface _Samples
-    def length: () -> Integer
-    def each: { (Float) -> void } -> void
-  end
-
-  type log_callback = ^(Integer level, String message, Object user_data) -> void
-  type new_segment_callback = ^(Whisper::Context, void, Integer n_new, Object user_data) -> void
-  type progress_callback = ^(Whisper::Context, void, Integer progress, Object user_data) -> void
-  type abort_callback = ^(Whisper::Context, void, Object user_data) -> boolish
-
-  LOG_LEVEL_NONE: Integer
-  LOG_LEVEL_INFO: Integer
-  LOG_LEVEL_WARN: Integer
-  LOG_LEVEL_ERROR: Integer
-  LOG_LEVEL_DEBUG: Integer
-  LOG_LEVEL_CONT: Integer
-
-  def self.lang_max_id: () -> Integer
-  def self.lang_id: (string name) -> Integer
-  def self.lang_str: (Integer id) -> String
-  def self.lang_str_full: (Integer id) -> String
-  def self.log_set=: (log_callback) -> log_callback
-  def self.finalize_log_callback: (void) -> void # Second argument of ObjectSpace.define_finalizer
-
-  class Context
-    def initialize: (string | _ToPath | ::URI::HTTP ) -> void
-    def transcribe: (string, Params) -> void
-                  | (string, Params) { (String) -> void } -> void
-    def model_n_vocab: () -> Integer
-    def model_n_audio_ctx: () -> Integer
-    def model_n_audio_state: () -> Integer
-    def model_n_text_head: () -> Integer
-    def model_n_text_layer: () -> Integer
-    def model_n_mels: () -> Integer
-    def model_ftype: () -> Integer
-    def model_type: () -> String
-    def full_n_segments: () -> Integer
-    def full_lang_id: () -> Integer
-    def full_get_segment_t0: (Integer) -> Integer
-    def full_get_segment_t1: (Integer) -> Integer
-    def full_get_segment_speaker_turn_next: (Integer) -> (true | false)
-    def full_get_segment_text: (Integer) -> String
-    def full_get_segment_no_speech_prob: (Integer) -> Float
-    def full: (Params, Array[Float], ?Integer) -> void
-            | (Params, _Samples, ?Integer) -> void
-    def full_parallel: (Params, Array[Float], ?Integer) -> void
-                     | (Params, _Samples, ?Integer) -> void
-                     | (Params, _Samples, ?Integer?, Integer) -> void
-    def each_segment: { (Segment) -> void } -> void
-                    | () -> Enumerator[Segment]
-    def model: () -> Model
-  end
-
-  class Params
-    def initialize: () -> void
-    def language=: (String) -> String # TODO: Enumerate lang names
-    def language: () -> String
-    def translate=: (boolish) -> boolish
-    def translate: () -> (true | false)
-    def no_context=: (boolish) -> boolish
-    def no_context: () -> (true | false)
-    def single_segment=: (boolish) -> boolish
-    def single_segment: () -> (true | false)
-    def print_special=: (boolish) -> boolish
-    def print_special: () -> (true | false)
-    def print_progress=: (boolish) -> boolish
-    def print_progress: () -> (true | false)
-    def print_realtime=: (boolish) -> boolish
-    def print_realtime: () -> (true | false)
-    def print_timestamps=: (boolish) -> boolish
-    def print_timestamps: () -> (true | false)
-    def suppress_blank=: (boolish) -> boolish
-    def suppress_blank: () -> (true | false)
-    def suppress_nst=: (boolish) -> boolish
-    def suppress_nst: () -> (true | false)
-    def token_timestamps=: (boolish) -> boolish
-    def token_timestamps: () -> (true | false)
-    def split_on_word=: (boolish) -> boolish
-    def split_on_word: () -> (true | false)
-    def initial_prompt=: (_ToS) -> _ToS
-    def initial_prompt: () -> String
-    def diarize=: (boolish) -> boolish
-    def diarize: () -> (true | false)
-    def offset=: (Integer) -> Integer
-    def offset: () -> Integer
-    def duration=: (Integer) -> Integer
-    def duration: () -> Integer
-    def max_text_tokens=: (Integer) -> Integer
-    def max_text_tokens: () -> Integer
-    def temperature=: (Float) -> Float
-    def temperature: () -> Float
-    def max_initial_ts=: (Float) -> Float
-    def max_initial_ts: () -> Float
-    def length_penalty=: (Float) -> Float
-    def length_penalty: () -> Float
-    def temperature_inc=: (Float) -> Float
-    def temperature_inc: () -> Float
-    def entropy_thold=: (Float) -> Float
-    def entropy_thold: () -> Float
-    def logprob_thold=: (Float) -> Float
-    def logprob_thold: () -> Float
-    def no_speech_thold=: (Float) -> Float
-    def no_speech_thold: () -> Float
-    def new_segment_callback=: (new_segment_callback) -> new_segment_callback
-    def new_segment_callback_user_data=: (Object) -> Object
-    def progress_callback=: (progress_callback) -> progress_callback
-    def progress_callback_user_data=: (Object) -> Object
-    def abort_callback=: (abort_callback) -> abort_callback
-    def abort_callback_user_data=: (Object) -> Object
-    def on_new_segment: { (Segment) -> void } -> void
-    def on_progress: { (Integer) -> void } -> void
-    def abort_on: { (Object) -> boolish } -> void
-  end
-
-  class Model
-    def self.pre_converted_models: () -> Hash[String, Model::URI]
-    def initialize: () -> void
-    def n_vocab: () -> Integer
-    def n_audio_ctx: () -> Integer
-    def n_audio_state: () -> Integer
-    def n_audio_head: () -> Integer
-    def n_audio_layer: () -> Integer
-    def n_text_ctx: () -> Integer
-    def n_text_state: () -> Integer
-    def n_text_head: () -> Integer
-    def n_text_layer: () -> Integer
-    def n_mels: () -> Integer
-    def ftype: () -> Integer
-    def type: () -> String
-
-    class URI
-      def initialize: (string | ::URI::HTTP) -> void
-      def to_path: -> String
-      def clear_cache: -> void
-    end
-  end
-
-  class Segment
-    def initialize: () -> void
-    def start_time: () -> Integer
-    def end_time: () -> Integer
-    def speaker_next_turn?: () -> (true | false)
-    def text: () -> String
-    def no_speech_prob: () -> Float
-  end
-
-  class Error < StandardError
-    attr_reader code: Integer
-
-    def initialize: (Integer) -> void
-  end
-end

bindings/ruby/tests/helper.rb

@@ -1,24 +0,0 @@
-require "test/unit"
-require "whisper"
-require_relative "jfk_reader/jfk_reader"
-
-class TestBase < Test::Unit::TestCase
-  AUDIO = File.join(__dir__, "..", "..", "..", "samples", "jfk.wav")
-
-  class << self
-    attr_reader :whisper
-
-    def startup
-      @whisper = Whisper::Context.new("base.en")
-      params = Whisper::Params.new
-      params.print_timestamps = false
-      @whisper.transcribe(TestBase::AUDIO, params)
-    end
-  end
-
-  private
-
-  def whisper
-    self.class.whisper
-  end
-end

bindings/ruby/tests/jfk_reader/.gitignore

@@ -1,5 +0,0 @@
-Makefile
-jfk_reader.o
-jfk_reader.so
-jfk_reader.bundle
-jfk_reader.dll

bindings/ruby/tests/jfk_reader/extconf.rb

@@ -1,3 +0,0 @@
-require "mkmf"
-
-create_makefile("jfk_reader")

bindings/ruby/tests/jfk_reader/jfk_reader.c

@@ -1,68 +0,0 @@
-#include <ruby.h>
-#include <ruby/memory_view.h>
-#include <ruby/encoding.h>
-
-static VALUE
-jfk_reader_initialize(VALUE self, VALUE audio_path)
-{
-  rb_iv_set(self, "audio_path", audio_path);
-  return Qnil;
-}
-
-static bool
-jfk_reader_get_memory_view(const VALUE obj, rb_memory_view_t *view, int flags)
-{
-  VALUE audio_path = rb_iv_get(obj, "audio_path");
-  const char *audio_path_str = StringValueCStr(audio_path);
-  const int n_samples = 176000;
-  float *data = (float *)malloc(n_samples * sizeof(float));
-  short *samples = (short *)malloc(n_samples * sizeof(short));
-  FILE *file = fopen(audio_path_str, "rb");
-
-  fseek(file, 78, SEEK_SET);
-  fread(samples, sizeof(short), n_samples, file);
-  fclose(file);
-  for (int i = 0; i < n_samples; i++) {
-    data[i] = samples[i]/32768.0;
-  }
-
-  view->obj = obj;
-  view->data = (void *)data;
-  view->byte_size = sizeof(float) * n_samples;
-  view->readonly = true;
-  view->format = "f";
-  view->item_size = sizeof(float);
-  view->item_desc.components = NULL;
-  view->item_desc.length = 0;
-  view->ndim = 1;
-  view->shape = NULL;
-  view->sub_offsets = NULL;
-  view->private_data = NULL;
-
-  return true;
-}
-
-static bool
-jfk_reader_release_memory_view(const VALUE obj, rb_memory_view_t *view)
-{
-  return true;
-}
-
-static bool
-jfk_reader_memory_view_available_p(const VALUE obj)
-{
-  return true;
-}
-
-static const rb_memory_view_entry_t jfk_reader_view_entry = {
-  jfk_reader_get_memory_view,
-  jfk_reader_release_memory_view,
-  jfk_reader_memory_view_available_p
-};
-
-void Init_jfk_reader(void)
-{
-  VALUE cJFKReader = rb_define_class("JFKReader", rb_cObject);
-  rb_memory_view_register(cJFKReader, &jfk_reader_view_entry);
-  rb_define_method(cJFKReader, "initialize", jfk_reader_initialize, 1);
-}

bindings/ruby/tests/test_callback.rb

@@ -1,160 +0,0 @@
-require_relative "helper"
-
-class TestCallback < TestBase
-  def setup
-    GC.start
-    @params = Whisper::Params.new
-    @whisper = Whisper::Context.new("base.en")
-    @audio = File.join(AUDIO)
-  end
-
-  def test_new_segment_callback
-    @params.new_segment_callback = ->(context, state, n_new, user_data) {
-      assert_kind_of Integer, n_new
-      assert n_new > 0
-      assert_same @whisper, context
-
-      n_segments = context.full_n_segments
-      n_new.times do |i|
-        i_segment = n_segments - 1 + i
-        start_time = context.full_get_segment_t0(i_segment) * 10
-        end_time = context.full_get_segment_t1(i_segment) * 10
-        text = context.full_get_segment_text(i_segment)
-
-        assert_kind_of Integer, start_time
-        assert start_time >= 0
-        assert_kind_of Integer, end_time
-        assert end_time > 0
-        assert_match /ask not what your country can do for you, ask what you can do for your country/, text if i_segment == 0
-      end
-    }
-
-    @whisper.transcribe(@audio, @params)
-  end
-
-  def test_new_segment_callback_closure
-    search_word = "what"
-    @params.new_segment_callback = ->(context, state, n_new, user_data) {
-      n_segments = context.full_n_segments
-      n_new.times do |i|
-        i_segment = n_segments - 1 + i
-        text = context.full_get_segment_text(i_segment)
-        if text.include?(search_word)
-          t0 = context.full_get_segment_t0(i_segment)
-          t1 = context.full_get_segment_t1(i_segment)
-          raise "search word '#{search_word}' found at between #{t0} and #{t1}"
-        end
-      end
-    }
-
-    assert_raise RuntimeError do
-      @whisper.transcribe(@audio, @params)
-    end
-  end
-
-  def test_new_segment_callback_user_data
-    udata = Object.new
-    @params.new_segment_callback_user_data = udata
-    @params.new_segment_callback = ->(context, state, n_new, user_data) {
-      assert_same udata, user_data
-    }
-
-    @whisper.transcribe(@audio, @params)
-  end
-
-  def test_new_segment_callback_user_data_gc
-    @params.new_segment_callback_user_data = "My user data"
-    @params.new_segment_callback = ->(context, state, n_new, user_data) {
-      assert_equal "My user data", user_data
-    }
-    GC.start
-
-    assert_same @whisper, @whisper.transcribe(@audio, @params)
-  end
-
-  def test_progress_callback
-    first = nil
-    last = nil
-    @params.progress_callback = ->(context, state, progress, user_data) {
-      assert_kind_of Integer, progress
-      assert 0 <= progress && progress <= 100
-      assert_same @whisper, context
-      first = progress if first.nil?
-      last = progress
-    }
-    @whisper.transcribe(@audio, @params)
-    assert_equal 0, first
-    assert_equal 100, last
-  end
-
-  def test_progress_callback_user_data
-    udata = Object.new
-    @params.progress_callback_user_data = udata
-    @params.progress_callback = ->(context, state, n_new, user_data) {
-      assert_same udata, user_data
-    }
-
-    @whisper.transcribe(@audio, @params)
-  end
-
-  def test_on_progress
-    first = nil
-    last = nil
-    @params.on_progress do |progress|
-      assert_kind_of Integer, progress
-      assert 0 <= progress && progress <= 100
-      first = progress if first.nil?
-      last = progress
-    end
-    @whisper.transcribe(@audio, @params)
-    assert_equal 0, first
-    assert_equal 100, last
-  end
-
-  def test_abort_callback
-    i = 0
-    @params.abort_callback = ->(user_data) {
-      assert_nil user_data
-      i += 1
-      return false
-    }
-    @whisper.transcribe(@audio, @params)
-    assert i > 0
-  end
-
-  def test_abort_callback_abort
-    i = 0
-    @params.abort_callback = ->(user_data) {
-      i += 1
-      return i == 3
-    }
-    @whisper.transcribe(@audio, @params)
-    assert_equal 3, i
-  end
-
-  def test_abort_callback_user_data
-    udata = Object.new
-    @params.abort_callback_user_data = udata
-    yielded = nil
-    @params.abort_callback = ->(user_data) {
-      yielded = user_data
-    }
-    @whisper.transcribe(@audio, @params)
-    assert_same udata, yielded
-  end
-
-  def test_abort_on
-    do_abort = false
-    aborted_from_callback = false
-    @params.on_new_segment do |segment|
-      do_abort = true if segment.text.match? /ask/
-    end
-    i = 0
-    @params.abort_on do
-      i += 1
-      do_abort
-    end
-    @whisper.transcribe(@audio, @params)
-    assert i > 0
-  end
-end

bindings/ruby/tests/test_error.rb

@@ -1,20 +0,0 @@
-require_relative "helper"
-
-class TestError < TestBase
-  def test_error
-    error = Whisper::Error.new(-2)
-    assert_equal "failed to compute log mel spectrogram", error.message
-    assert_equal -2, error.code
-  end
-
-  def test_unknown_error
-    error = Whisper::Error.new(-20)
-    assert_equal "unknown error", error.message
-  end
-
-  def test_non_int_code
-    assert_raise TypeError do
-      error = Whisper::Error.new("non int")
-    end
-  end
-end

bindings/ruby/tests/test_model.rb

@@ -1,109 +0,0 @@
-require_relative "helper"
-require "pathname"
-
-class TestModel < TestBase
-  def test_model
-    whisper = Whisper::Context.new("base.en")
-    assert_instance_of Whisper::Model, whisper.model
-  end
-
-  def test_attributes
-    whisper = Whisper::Context.new("base.en")
-    model = whisper.model
-
-    assert_equal 51864, model.n_vocab
-    assert_equal 1500, model.n_audio_ctx
-    assert_equal 512, model.n_audio_state
-    assert_equal 8, model.n_audio_head
-    assert_equal 6, model.n_audio_layer
-    assert_equal 448, model.n_text_ctx
-    assert_equal 512, model.n_text_state
-    assert_equal 8, model.n_text_head
-    assert_equal 6, model.n_text_layer
-    assert_equal 80, model.n_mels
-    assert_equal 1, model.ftype
-    assert_equal "base", model.type
-  end
-
-  def test_gc
-    model = Whisper::Context.new("base.en").model
-    GC.start
-
-    assert_equal 51864, model.n_vocab
-    assert_equal 1500, model.n_audio_ctx
-    assert_equal 512, model.n_audio_state
-    assert_equal 8, model.n_audio_head
-    assert_equal 6, model.n_audio_layer
-    assert_equal 448, model.n_text_ctx
-    assert_equal 512, model.n_text_state
-    assert_equal 8, model.n_text_head
-    assert_equal 6, model.n_text_layer
-    assert_equal 80, model.n_mels
-    assert_equal 1, model.ftype
-    assert_equal "base", model.type
-  end
-
-  def test_pathname
-    path = Pathname(Whisper::Model.pre_converted_models["base.en"].to_path)
-    whisper = Whisper::Context.new(path)
-    model = whisper.model
-
-    assert_equal 51864, model.n_vocab
-    assert_equal 1500, model.n_audio_ctx
-    assert_equal 512, model.n_audio_state
-    assert_equal 8, model.n_audio_head
-    assert_equal 6, model.n_audio_layer
-    assert_equal 448, model.n_text_ctx
-    assert_equal 512, model.n_text_state
-    assert_equal 8, model.n_text_head
-    assert_equal 6, model.n_text_layer
-    assert_equal 80, model.n_mels
-    assert_equal 1, model.ftype
-    assert_equal "base", model.type
-  end
-
-  def test_auto_download
-    path = Whisper::Model.pre_converted_models["base.en"].to_path
-
-    assert_path_exist path
-    assert_equal 147964211, File.size(path)
-  end
-
-  def test_uri_string
-    path = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin"
-    whisper = Whisper::Context.new(path)
-    model = whisper.model
-
-    assert_equal 51864, model.n_vocab
-    assert_equal 1500, model.n_audio_ctx
-    assert_equal 512, model.n_audio_state
-    assert_equal 8, model.n_audio_head
-    assert_equal 6, model.n_audio_layer
-    assert_equal 448, model.n_text_ctx
-    assert_equal 512, model.n_text_state
-    assert_equal 8, model.n_text_head
-    assert_equal 6, model.n_text_layer
-    assert_equal 80, model.n_mels
-    assert_equal 1, model.ftype
-    assert_equal "base", model.type
-  end
-
-  def test_uri
-    path = URI("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin")
-    whisper = Whisper::Context.new(path)
-    model = whisper.model
-
-    assert_equal 51864, model.n_vocab
-    assert_equal 1500, model.n_audio_ctx
-    assert_equal 512, model.n_audio_state
-    assert_equal 8, model.n_audio_head
-    assert_equal 6, model.n_audio_layer
-    assert_equal 448, model.n_text_ctx
-    assert_equal 512, model.n_text_state
-    assert_equal 8, model.n_text_head
-    assert_equal 6, model.n_text_layer
-    assert_equal 80, model.n_mels
-    assert_equal 1, model.ftype
-    assert_equal "base", model.type
-  end
-end

bindings/ruby/tests/test_package.rb

@@ -1,31 +0,0 @@
-require_relative "helper"
-require 'tempfile'
-require 'tmpdir'
-require 'shellwords'
-
-class TestPackage < TestBase
-  def test_build
-    Tempfile.create do |file|
-      assert system("gem", "build", "whispercpp.gemspec", "--output", file.to_path.shellescape, exception: true)
-      assert file.size > 0
-      assert_path_exist file.to_path
-    end
-  end
-
-  sub_test_case "Building binary on installation" do
-    def setup
-      system "rake", "build", exception: true
-    end
-
-    def test_install
-      match_data = `rake -Tbuild`.match(/(whispercpp-(.+)\.gem)/)
-      filename = match_data[1]
-      version = match_data[2]
-      basename = "whisper.#{RbConfig::CONFIG["DLEXT"]}"
-      Dir.mktmpdir do |dir|
-        system "gem", "install", "--install-dir", dir.shellescape, "--no-document", "pkg/#{filename.shellescape}", exception: true
-        assert_path_exist File.join(dir, "gems/whispercpp-#{version}/lib", basename)
-      end
-    end
-  end
-end

bindings/ruby/tests/test_params.rb

@@ -1,160 +0,0 @@
-require_relative "helper"
-
-class TestParams < TestBase
-  def setup
-    @params  = Whisper::Params.new
-  end
-
-  def test_language
-    @params.language = "en"
-    assert_equal @params.language, "en"
-    @params.language = "auto"
-    assert_equal @params.language, "auto"
-  end
-
-  def test_offset
-    @params.offset = 10_000
-    assert_equal @params.offset, 10_000
-    @params.offset = 0
-    assert_equal @params.offset, 0
-  end
-
-  def test_duration
-    @params.duration = 60_000
-    assert_equal @params.duration, 60_000
-    @params.duration = 0
-    assert_equal @params.duration, 0
-  end
-
-  def test_max_text_tokens
-    @params.max_text_tokens = 300
-    assert_equal @params.max_text_tokens, 300
-    @params.max_text_tokens = 0
-    assert_equal @params.max_text_tokens, 0
-  end
-
-  def test_translate
-    @params.translate = true
-    assert @params.translate
-    @params.translate = false
-    assert !@params.translate
-  end
-
-  def test_no_context
-    @params.no_context = true
-    assert @params.no_context
-    @params.no_context = false
-    assert !@params.no_context
-  end
-
-  def test_single_segment
-    @params.single_segment = true
-    assert @params.single_segment
-    @params.single_segment = false
-    assert !@params.single_segment
-  end
-
-  def test_print_special
-    @params.print_special = true
-    assert @params.print_special
-    @params.print_special = false
-    assert !@params.print_special
-  end
-
-  def test_print_progress
-    @params.print_progress = true
-    assert @params.print_progress
-    @params.print_progress = false
-    assert !@params.print_progress
-  end
-
-  def test_print_realtime
-    @params.print_realtime = true
-    assert @params.print_realtime
-    @params.print_realtime = false
-    assert !@params.print_realtime
-  end
-
-  def test_print_timestamps
-    @params.print_timestamps = true
-    assert @params.print_timestamps
-    @params.print_timestamps = false
-    assert !@params.print_timestamps
-  end
-
-  def test_suppress_blank
-    @params.suppress_blank = true
-    assert @params.suppress_blank
-    @params.suppress_blank = false
-    assert !@params.suppress_blank
-  end
-
-  def test_suppress_nst
-    @params.suppress_nst = true
-    assert @params.suppress_nst
-    @params.suppress_nst = false
-    assert !@params.suppress_nst
-  end
-
-  def test_token_timestamps
-    @params.token_timestamps = true
-    assert @params.token_timestamps
-    @params.token_timestamps = false
-    assert !@params.token_timestamps
-  end
-
-  def test_split_on_word
-    @params.split_on_word = true
-    assert @params.split_on_word
-    @params.split_on_word = false
-    assert !@params.split_on_word
-  end
-
-  def test_initial_prompt
-    assert_nil @params.initial_prompt
-    @params.initial_prompt = "You are a polite person."
-    assert_equal "You are a polite person.", @params.initial_prompt
-  end
-
-  def test_temperature
-    assert_equal 0.0, @params.temperature
-    @params.temperature = 0.5
-    assert_equal 0.5, @params.temperature
-  end
-
-  def test_max_initial_ts
-    assert_equal 1.0, @params.max_initial_ts
-    @params.max_initial_ts = 600.0
-    assert_equal 600.0, @params.max_initial_ts
-  end
-
-  def test_length_penalty
-    assert_equal -1.0, @params.length_penalty
-    @params.length_penalty = 0.5
-    assert_equal 0.5, @params.length_penalty
-  end
-
-  def test_temperature_inc
-    assert_in_delta 0.2, @params.temperature_inc
-    @params.temperature_inc = 0.5
-    assert_in_delta 0.5, @params.temperature_inc
-  end
-
-  def test_entropy_thold
-    assert_in_delta 2.4, @params.entropy_thold
-    @params.entropy_thold = 3.0
-    assert_in_delta 3.0, @params.entropy_thold
-  end
-
-  def test_logprob_thold
-    assert_in_delta -1.0, @params.logprob_thold
-    @params.logprob_thold = -0.5
-    assert_in_delta -0.5, @params.logprob_thold
-  end
-
-  def test_no_speech_thold
-    assert_in_delta 0.6, @params.no_speech_thold
-    @params.no_speech_thold = 0.2
-    assert_in_delta 0.2, @params.no_speech_thold
-  end
-end

bindings/ruby/tests/test_segment.rb

@@ -1,74 +0,0 @@
-require_relative "helper"
-
-class TestSegment < TestBase
-  def test_iteration
-    whisper.each_segment do |segment|
-      assert_instance_of Whisper::Segment, segment
-    end
-  end
-
-  def test_enumerator
-    enum = whisper.each_segment
-    assert_instance_of Enumerator, enum
-    enum.to_a.each_with_index do |segment, index|
-      assert_instance_of Whisper::Segment, segment
-      assert_kind_of Integer, index
-    end
-  end
-
-  def test_start_time
-    i = 0
-    whisper.each_segment do |segment|
-      assert_equal 0, segment.start_time if i == 0
-      i += 1
-    end
-  end
-
-  def test_end_time
-    i = 0
-    whisper.each_segment do |segment|
-      assert_equal whisper.full_get_segment_t1(i) * 10, segment.end_time
-      i += 1
-    end
-  end
-
-  def test_no_speech_prob
-    no_speech_prob = nil
-    whisper.each_segment do |segment|
-      no_speech_prob = segment.no_speech_prob
-    end
-    assert no_speech_prob > 0.0
-  end
-
-  def test_on_new_segment
-    params = Whisper::Params.new
-    seg = nil
-    index = 0
-    params.on_new_segment do |segment|
-      assert_instance_of Whisper::Segment, segment
-      if index == 0
-        seg = segment
-        assert_equal 0, segment.start_time
-        assert_match /ask not what your country can do for you, ask what you can do for your country/, segment.text
-      end
-      index += 1
-    end
-    whisper.transcribe(AUDIO, params)
-    assert_equal 0, seg.start_time
-    assert_match /ask not what your country can do for you, ask what you can do for your country/, seg.text
-  end
-
-  def test_on_new_segment_twice
-    params = Whisper::Params.new
-    seg = nil
-    params.on_new_segment do |segment|
-      seg = segment
-      return
-    end
-    params.on_new_segment do |segment|
-      assert_same seg, segment
-      return
-    end
-    whisper.transcribe(AUDIO, params)
-  end
-end

bindings/ruby/tests/test_whisper.rb

@@ -1,217 +1,131 @@
-require_relative "helper"
+TOPDIR = File.expand_path(File.join(File.dirname(__FILE__), '..'))
-require "stringio"
+EXTDIR = File.join(TOPDIR, 'ext')
-require "etc"
+#$LIBDIR = File.join(TOPDIR, 'lib')
+#$:.unshift(LIBDIR)
+$:.unshift(EXTDIR)
 
-# Exists to detect memory-related bug
+require 'whisper'
-Whisper.log_set ->(level, buffer, user_data) {}, nil
+require 'test/unit'
 
-class TestWhisper < TestBase
+class TestWhisper < Test::Unit::TestCase
   def setup
     @params  = Whisper::Params.new
   end
 
+  def test_language
+    @params.language = "en"
+    assert_equal @params.language, "en"
+    @params.language = "auto"
+    assert_equal @params.language, "auto"
+  end
+
+  def test_offset
+    @params.offset = 10_000
+    assert_equal @params.offset, 10_000
+    @params.offset = 0
+    assert_equal @params.offset, 0
+  end
+
+  def test_duration
+    @params.duration = 60_000
+    assert_equal @params.duration, 60_000
+    @params.duration = 0
+    assert_equal @params.duration, 0
+  end
+
+  def test_max_text_tokens
+    @params.max_text_tokens = 300
+    assert_equal @params.max_text_tokens, 300
+    @params.max_text_tokens = 0
+    assert_equal @params.max_text_tokens, 0
+  end
+
+  def test_translate
+    @params.translate = true
+    assert @params.translate
+    @params.translate = false
+    assert !@params.translate
+  end
+
+  def test_no_context
+    @params.no_context = true
+    assert @params.no_context
+    @params.no_context = false
+    assert !@params.no_context
+  end
+
+  def test_single_segment
+    @params.single_segment = true
+    assert @params.single_segment
+    @params.single_segment = false
+    assert !@params.single_segment
+  end
+
+  def test_print_special
+    @params.print_special = true
+    assert @params.print_special
+    @params.print_special = false
+    assert !@params.print_special
+  end
+
+  def test_print_progress
+    @params.print_progress = true
+    assert @params.print_progress
+    @params.print_progress = false
+    assert !@params.print_progress
+  end
+
+  def test_print_realtime
+    @params.print_realtime = true
+    assert @params.print_realtime
+    @params.print_realtime = false
+    assert !@params.print_realtime
+  end
+
+  def test_print_timestamps
+    @params.print_timestamps = true
+    assert @params.print_timestamps
+    @params.print_timestamps = false
+    assert !@params.print_timestamps
+  end
+
+  def test_suppress_blank
+    @params.suppress_blank = true
+    assert @params.suppress_blank
+    @params.suppress_blank = false
+    assert !@params.suppress_blank
+  end
+
+  def test_suppress_non_speech_tokens
+    @params.suppress_non_speech_tokens = true
+    assert @params.suppress_non_speech_tokens
+    @params.suppress_non_speech_tokens = false
+    assert !@params.suppress_non_speech_tokens
+  end
+
+  def test_token_timestamps
+    @params.token_timestamps = true
+    assert @params.token_timestamps
+    @params.token_timestamps = false
+    assert !@params.token_timestamps
+  end
+
+  def test_split_on_word
+    @params.split_on_word = true
+    assert @params.split_on_word
+    @params.split_on_word = false
+    assert !@params.split_on_word
+  end
+
   def test_whisper
-    @whisper = Whisper::Context.new("base.en")
+    @whisper = Whisper::Context.new(File.join(TOPDIR, '..', '..', 'models', 'ggml-base.en.bin'))
     params  = Whisper::Params.new
     params.print_timestamps = false
 
-    @whisper.transcribe(AUDIO, params) {|text|
+    jfk = File.join(TOPDIR, '..', '..', 'samples', 'jfk.wav')
+    @whisper.transcribe(jfk, params) {|text|
       assert_match /ask not what your country can do for you, ask what you can do for your country/, text
     }
   end
 
-  sub_test_case "After transcription" do
-    def test_full_n_segments
-      assert_equal 1, whisper.full_n_segments
-    end
-
-    def test_full_lang_id
-      assert_equal 0, whisper.full_lang_id
-    end
-
-    def test_full_get_segment_t0
-      assert_equal 0, whisper.full_get_segment_t0(0)
-      assert_raise IndexError do
-        whisper.full_get_segment_t0(whisper.full_n_segments)
-      end
-      assert_raise IndexError do
-        whisper.full_get_segment_t0(-1)
-      end
-    end
-
-    def test_full_get_segment_t1
-      t1 = whisper.full_get_segment_t1(0)
-      assert_kind_of Integer, t1
-      assert t1 > 0
-      assert_raise IndexError do
-        whisper.full_get_segment_t1(whisper.full_n_segments)
-      end
-    end
-
-    def test_full_get_segment_speaker_turn_next
-      assert_false whisper.full_get_segment_speaker_turn_next(0)
-    end
-
-    def test_full_get_segment_text
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, whisper.full_get_segment_text(0)
-    end
-
-    def test_full_get_segment_no_speech_prob
-      prob = whisper.full_get_segment_no_speech_prob(0)
-      assert prob > 0.0
-      assert prob < 1.0
-    end
-  end
-
-  def test_lang_max_id
-    assert_kind_of Integer, Whisper.lang_max_id
-  end
-
-  def test_lang_id
-    assert_equal 0, Whisper.lang_id("en")
-    assert_raise ArgumentError do
-      Whisper.lang_id("non existing language")
-    end
-  end
-
-  def test_lang_str
-    assert_equal "en", Whisper.lang_str(0)
-    assert_raise IndexError do
-      Whisper.lang_str(Whisper.lang_max_id + 1)
-    end
-  end
-
-  def test_lang_str_full
-    assert_equal "english", Whisper.lang_str_full(0)
-    assert_raise IndexError do
-      Whisper.lang_str_full(Whisper.lang_max_id + 1)
-    end
-  end
-
-  def test_log_set
-    user_data = Object.new
-    logs = []
-    log_callback = ->(level, buffer, udata) {
-      logs << [level, buffer, udata]
-    }
-    Whisper.log_set log_callback, user_data
-    Whisper::Context.new("base.en")
-
-    assert logs.length > 30
-    logs.each do |log|
-      assert_include [Whisper::LOG_LEVEL_DEBUG, Whisper::LOG_LEVEL_INFO, Whisper::LOG_LEVEL_WARN], log[0]
-      assert_same user_data, log[2]
-    end
-  end
-
-  def test_log_suppress
-    stderr = $stderr
-    Whisper.log_set ->(level, buffer, user_data) {
-      # do nothing
-    }, nil
-    dev = StringIO.new("")
-    $stderr = dev
-    Whisper::Context.new("base.en")
-    assert_empty dev.string
-  ensure
-    $stderr = stderr
-  end
-
-  sub_test_case "full" do
-    def setup
-      super
-      @whisper = Whisper::Context.new("base.en")
-      @samples = File.read(AUDIO, nil, 78).unpack("s<*").collect {|i| i.to_f / 2**15}
-    end
-
-    def test_full
-      @whisper.full(@params, @samples, @samples.length)
-
-      assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
-    end
-
-    def test_full_without_length
-      @whisper.full(@params, @samples)
-
-      assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
-    end
-
-    def test_full_enumerator
-      samples = @samples.each
-      @whisper.full(@params, samples, @samples.length)
-
-      assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
-    end
-
-    def test_full_enumerator_without_length
-      samples = @samples.each
-      assert_raise ArgumentError do
-        @whisper.full(@params, samples)
-      end
-    end
-
-    def test_full_enumerator_with_too_large_length
-      samples = @samples.each.take(10).to_enum
-      assert_raise StopIteration do
-        @whisper.full(@params, samples, 11)
-      end
-    end
-
-    def test_full_with_memory_view
-      samples = JFKReader.new(AUDIO)
-      @whisper.full(@params, samples)
-
-      assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
-    end
-
-    def test_full_parallel
-      @whisper.full_parallel(@params, @samples, @samples.length, Etc.nprocessors)
-
-      assert_equal Etc.nprocessors, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-
-    def test_full_parallel_with_memory_view
-      samples = JFKReader.new(AUDIO)
-      @whisper.full_parallel(@params, samples, nil, Etc.nprocessors)
-
-      assert_equal Etc.nprocessors, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-
-    def test_full_parallel_without_length_and_n_processors
-      @whisper.full_parallel(@params, @samples)
-
-      assert_equal 1, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-
-    def test_full_parallel_without_length
-      @whisper.full_parallel(@params, @samples, nil, Etc.nprocessors)
-
-      assert_equal Etc.nprocessors, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-
-    def test_full_parallel_without_n_processors
-      @whisper.full_parallel(@params, @samples, @samples.length)
-
-      assert_equal 1, @whisper.full_n_segments
-      text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
-      assert_match /for your country/i, text
-    end
-  end
 end

bindings/ruby/whispercpp.gemspec

@@ -1,36 +1,28 @@
-require_relative "extsources"
-
 Gem::Specification.new do |s|
   s.name    = "whispercpp"
   s.authors = ["Georgi Gerganov", "Todd A. Fisher"]
-  s.version = '1.3.1'
+  s.version = '1.3.0'
-  s.date    = '2024-12-19'
+  s.date    = '2024-05-14'
   s.description = %q{High-performance inference of OpenAI's Whisper automatic speech recognition (ASR) model via Ruby}
   s.email   = 'todd.fisher@gmail.com'
   s.extra_rdoc_files = ['LICENSE', 'README.md']
+  
+  s.files = ["LICENSE", "README.md", "Rakefile", "ext/extconf.rb", "ext/ggml.c", "ext/ruby_whisper.cpp", "ext/whisper.cpp", "ext/dr_wav.h", "ext/ggml.h", "ext/ruby_whisper.h", "ext/whisper.h"]
 
-  s.files = `git ls-files . -z`.split("\x0") +
+  #### Load-time details
-              EXTSOURCES.collect {|file|
+  s.require_paths = ['lib','ext']
-                basename = File.basename(file)
-                if s.extra_rdoc_files.include?(basename)
-                  basename
-                else
-                  file.sub("../..", "ext")
-                end
-              }
-
   s.summary = %q{Ruby whisper.cpp bindings}
-  s.test_files = s.files.select {|file| file.start_with? "tests/"}
+  s.test_files = ["tests/test_whisper.rb"]
-
+  
   s.extensions << 'ext/extconf.rb'
-  s.required_ruby_version = '>= 3.1.0'
+  
 
   #### Documentation and testing.
   s.homepage = 'https://github.com/ggerganov/whisper.cpp'
-  s.rdoc_options = ['--main', 'README.md']
+  s.rdoc_options = ['--main', '../../README.md']
-
 
+  
     s.platform = Gem::Platform::RUBY
-
+  
   s.licenses = ['MIT']
 end

cmake/DefaultTargetOptions.cmake

@@ -13,4 +13,5 @@ set_target_properties(${TARGET}
     PROPERTIES
         EXPORT_COMPILE_COMMANDS ON
         RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin"
+        INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib"
 )

cmake/FindFFmpeg.cmake

@@ -36,7 +36,7 @@ include(FindPackageHandleStandardArgs)
 
 # The default components were taken from a survey over other FindFFMPEG.cmake files
 if (NOT FFmpeg_FIND_COMPONENTS)
-  set(FFmpeg_FIND_COMPONENTS AVFORMAT AVCODEC AVUTIL SWRESAMPLE)
+  set(FFmpeg_FIND_COMPONENTS AVFORMAT AVCODEC AVUTIL SWRESAMPLE) 
 endif()
 
 #
@@ -84,7 +84,7 @@ macro(find_component _component _pkgconfig _library _header)
 
   # CMake's default is to search first for shared libraries and then for static libraries.
   # Todo later: add option to prefer static libs over dynamic:
-  find_library(${_component}_LIBRARIES NAMES ${_library} lib${_library}.a
+  find_library(${_component}_LIBRARIES NAMES ${_library} lib${_library}.a  
       HINTS
       ${PC_${_component}_LIBDIR}
       ${PC_${_component}_LIBRARY_DIRS}

cmake/whisper-config.cmake.in

@@ -1,7 +1,7 @@
-set(WHISPER_VERSION      @WHISPER_INSTALL_VERSION@)
+set(LLAMA_VERSION      @LLAMA_INSTALL_VERSION@)
-set(WHISPER_BUILD_COMMIT @WHISPER_BUILD_COMMIT@)
+set(LLAMA_BUILD_COMMIT @LLAMA_BUILD_COMMIT@)
-set(WHISPER_BUILD_NUMBER @WHISPER_BUILD_NUMBER@)
+set(LLAMA_BUILD_NUMBER @LLAMA_BUILD_NUMBER@)
-set(WHISPER_SHARED_LIB   @BUILD_SHARED_LIBS@)
+set(LLAMA_SHARED_LIB   @BUILD_SHARED_LIBS@)
 
 set(GGML_BLAS       @GGML_BLAS@)
 set(GGML_CUDA       @GGML_CUDA@)
@@ -11,9 +11,9 @@ set(GGML_ACCELERATE @GGML_ACCELERATE@)
 
 @PACKAGE_INIT@
 
-set_and_check(WHISPER_INCLUDE_DIR "@PACKAGE_WHISPER_INCLUDE_INSTALL_DIR@")
+set_and_check(LLAMA_INCLUDE_DIR "@PACKAGE_LLAMA_INCLUDE_INSTALL_DIR@")
-set_and_check(WHISPER_LIB_DIR     "@PACKAGE_WHISPER_LIB_INSTALL_DIR@")
+set_and_check(LLAMA_LIB_DIR     "@PACKAGE_LLAMA_LIB_INSTALL_DIR@")
-set_and_check(WHISPER_BIN_DIR     "@PACKAGE_WHISPER_BIN_INSTALL_DIR@")
+set_and_check(LLAMA_BIN_DIR     "@PACKAGE_LLAMA_BIN_INSTALL_DIR@")
 
 # Ensure transient dependencies satisfied
 
@@ -43,23 +43,23 @@ if (GGML_HIPBLAS)
     find_package(rocblas REQUIRED)
 endif()
 
-find_library(whisper_LIBRARY whisper
+find_library(llama_LIBRARY llama
     REQUIRED
-    HINTS ${WHISPER_LIB_DIR})
+    HINTS ${LLAMA_LIB_DIR})
 
-set(_whisper_link_deps "Threads::Threads" "@WHISPER_EXTRA_LIBS@")
+set(_llama_link_deps "Threads::Threads" "@LLAMA_EXTRA_LIBS@")
-set(_whisper_transient_defines "@WHISPER_TRANSIENT_DEFINES@")
+set(_llama_transient_defines "@LLAMA_TRANSIENT_DEFINES@")
 
-add_library(whisper UNKNOWN IMPORTED)
+add_library(llama UNKNOWN IMPORTED)
 
-set_target_properties(whisper
+set_target_properties(llama
     PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES "${WHISPER_INCLUDE_DIR}"
+        INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}"
-        INTERFACE_LINK_LIBRARIES "${_whisper_link_deps}"
+        INTERFACE_LINK_LIBRARIES "${_llama_link_deps}"
-        INTERFACE_COMPILE_DEFINITIONS "${_whisper_transient_defines}"
+        INTERFACE_COMPILE_DEFINITIONS "${_llama_transient_defines}"
         IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
-        IMPORTED_LOCATION "${whisper_LIBRARY}"
+        IMPORTED_LOCATION "${llama_LIBRARY}"
         INTERFACE_COMPILE_FEATURES cxx_std_11
         POSITION_INDEPENDENT_CODE ON )
 
-check_required_components(whisper)
+check_required_components(Llama)

cmake/whisper.pc.in

@@ -6,5 +6,5 @@ includedir=${prefix}/include
 Name: whisper
 Description: Port of OpenAI's Whisper model in C/C++
 Version: @PROJECT_VERSION@
-Libs: -L${libdir} -lggml  -lggml-base -lwhisper
+Libs: -L${libdir} -lwhisper
 Cflags: -I${includedir}

examples/CMakeLists.txt

@@ -40,7 +40,7 @@ if (WHISPER_FFMPEG)
     message(STATUS "Found ffmpeg libs:       ${FFMPEG_LIBRARIES}")
     message(STATUS "Found ffmpeg headers in: ${FFMPEG_INCLUDE_DIRS}")
     message(STATUS "ffmpeg definitions:      ${FFMPEG_DEFINITIONS}")
-    message(STATUS "Found avformat           ${AVFORMAT_VERSION}")
+    message(STATUS "Found avformat ${AVFORMAT_VERSION}")
 
     include_directories(${FFMPEG_INCLUDE_DIRS})
     add_compile_definitions(WHISPER_FFMPEG)
@@ -97,29 +97,50 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 if (EMSCRIPTEN)
     add_subdirectory(whisper.wasm)
+    set_target_properties(libmain PROPERTIES FOLDER "libs")
     add_subdirectory(stream.wasm)
+    set_target_properties(libstream PROPERTIES FOLDER "libs")
     add_subdirectory(command.wasm)
+    set_target_properties(libcommand PROPERTIES FOLDER "libs")
+    add_subdirectory(talk.wasm)
+    set_target_properties(libtalk PROPERTIES FOLDER "libs")
     add_subdirectory(bench.wasm)
+    set_target_properties(libbench PROPERTIES FOLDER "libs")
 elseif(CMAKE_JS_VERSION)
     add_subdirectory(addon.node)
+    set_target_properties(addon.node PROPERTIES FOLDER "examples")
 else()
-    add_subdirectory(cli)
+    add_subdirectory(main)
-    add_subdirectory(bench)
+    set_target_properties(main PROPERTIES FOLDER "examples")
+if (WHISPER_SDL2)
+    add_subdirectory(stream)
+    set_target_properties(stream PROPERTIES FOLDER "examples")
+endif (WHISPER_SDL2)
     add_subdirectory(server)
+    set_target_properties(server PROPERTIES FOLDER "examples")
+if (WHISPER_SDL2)
+    add_subdirectory(command)
+    set_target_properties(command PROPERTIES FOLDER "examples")
+endif (WHISPER_SDL2)
+    add_subdirectory(bench)
+    set_target_properties(bench PROPERTIES FOLDER "examples")
     add_subdirectory(quantize)
-    if (WHISPER_SDL2)
+    set_target_properties(quantize PROPERTIES FOLDER "examples")
-        add_subdirectory(stream)
+if (WHISPER_SDL2)
-        add_subdirectory(command)
+    add_subdirectory(talk)
-        add_subdirectory(talk-llama)
+    set_target_properties(talk PROPERTIES FOLDER "examples")
-        add_subdirectory(lsp)
+    add_subdirectory(talk-llama)
-        if (GGML_SYCL)
+    set_target_properties(talk-llama PROPERTIES FOLDER "examples")
-            add_subdirectory(sycl)
+    add_subdirectory(lsp)
-        endif()
+    set_target_properties(lsp PROPERTIES FOLDER "examples")
-    endif (WHISPER_SDL2)
+    if (GGML_SYCL)
-
+        add_subdirectory(sycl)
-    add_subdirectory(deprecation-warning)
+        set_target_properties(sycl PROPERTIES FOLDER "examples")
+    endif()
+endif (WHISPER_SDL2)
 endif()
 
 if (WHISPER_SDL2)
     add_subdirectory(wchess)
+    set_target_properties(wchess PROPERTIES FOLDER "examples")
 endif (WHISPER_SDL2)

examples/bench/CMakeLists.txt

@@ -1,8 +1,6 @@
-set(TARGET whisper-bench)
+set(TARGET bench)
 add_executable(${TARGET} bench.cpp)
 
 include(DefaultTargetOptions)
 
 target_link_libraries(${TARGET} PRIVATE whisper ${CMAKE_THREAD_LIBS_INIT})
-
-install(TARGETS ${TARGET} RUNTIME)

examples/bench/README.md

@@ -1,4 +1,4 @@
-# whisper.cpp/examples/bench
+# bench
 
 A very basic tool for benchmarking the inference performance on your device. The tool simply runs the Encoder part of
 the transformer on some random audio data and records the execution time. This way we can have an objective comparison
@@ -7,8 +7,11 @@ of the performance of the model for various setups.
 Benchmark results are tracked in the following Github issue: https://github.com/ggerganov/whisper.cpp/issues/89
 
 ```bash
-# run the bench too on the small.en model using 4 threads
+# build the bench tool
-$ ./build/bin/whisper-bench -m ./models/ggml-small.en.bin -t 4
+$ make bench
+
+# run it on the small.en model using 4 threads
+$ ./bench -m ./models/ggml-small.en.bin -t 4
 
 whisper_model_load: loading model from './models/ggml-small.en.bin'
 whisper_model_load: n_vocab       = 51864

examples/command/CMakeLists.txt

@@ -1,10 +1,9 @@
 if (WHISPER_SDL2)
-    set(TARGET whisper-command)
+    # command
+    set(TARGET command)
     add_executable(${TARGET} command.cpp)
 
     include(DefaultTargetOptions)
 
     target_link_libraries(${TARGET} PRIVATE common common-sdl whisper ${CMAKE_THREAD_LIBS_INIT})
-
-    install(TARGETS ${TARGET} RUNTIME)
 endif ()

examples/command/README.md

@@ -1,14 +1,14 @@
-# whisper.cpp/examples/command
+# command
 
 This is a basic Voice Assistant example that accepts voice commands from the microphone.
 More info is available in [issue #171](https://github.com/ggerganov/whisper.cpp/issues/171).
 
 ```bash
 # Run with default arguments and small model
-./whisper-command -m ./models/ggml-small.en.bin -t 8
+./command -m ./models/ggml-small.en.bin -t 8
 
 # On Raspberry Pi, use tiny or base models + "-ac 768" for better performance
-./whisper-command -m ./models/ggml-tiny.en.bin -ac 768 -t 3 -c 0
+./command -m ./models/ggml-tiny.en.bin -ac 768 -t 3 -c 0
 ```
 
 https://user-images.githubusercontent.com/1991296/204038393-2f846eae-c255-4099-a76d-5735c25c49da.mp4
@@ -23,10 +23,10 @@ Initial tests show that this approach might be extremely efficient in terms of p
 
 ```bash
 # Run in guided mode, the list of allowed commands is in commands.txt
-./whisper-command -m ./models/ggml-base.en.bin -cmd ./examples/command/commands.txt
+./command -m ./models/ggml-base.en.bin -cmd ./examples/command/commands.txt
 
 # On Raspberry Pi, in guided mode you can use "-ac 128" for extra performance
-./whisper-command -m ./models/ggml-tiny.en.bin -cmd ./examples/command/commands.txt -ac 128 -t 3 -c 0
+./command -m ./models/ggml-tiny.en.bin -cmd ./examples/command/commands.txt -ac 128 -t 3 -c 0
 ```
 
 https://user-images.githubusercontent.com/1991296/207435352-8fc4ed3f-bde5-4555-9b8b-aeeb76bee969.mp4
@@ -34,7 +34,7 @@ https://user-images.githubusercontent.com/1991296/207435352-8fc4ed3f-bde5-4555-9
 
 ## Building
 
-The `whisper-command` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:
+The `command` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:
 
 ```bash
 # Install SDL2
@@ -47,6 +47,5 @@ sudo dnf install SDL2 SDL2-devel
 # Install SDL2 on Mac OS
 brew install sdl2
 
-cmake -B build -DWHISPER_SDL2=ON
+make command
-cmake --build build --config Release
 ```

examples/common-ggml.cpp

@@ -209,8 +209,6 @@ bool ggml_common_quantize_0(
                 case GGML_TYPE_IQ4_XS:
                 case GGML_TYPE_IQ1_M:
                 case GGML_TYPE_BF16:
-                case GGML_TYPE_TQ1_0:
-                case GGML_TYPE_TQ2_0:
                 case GGML_TYPE_COUNT:
                     {
                         fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype));

examples/common-sdl.cpp

@@ -1,7 +1,5 @@
 #include "common-sdl.h"
 
-#include <cstdio>
-
 audio_async::audio_async(int len_ms) {
     m_len_ms = len_ms;
 

examples/common.cpp

@@ -147,6 +147,7 @@ std::string gpt_random_prompt(std::mt19937 & rng) {
         case 7: return "He";
         case 8: return "She";
         case 9: return "They";
+        default: return "To";
     }
 
     return "The";

examples/common.h

@@ -9,7 +9,6 @@
 #include <thread>
 #include <ctime>
 #include <fstream>
-#include <sstream>
 
 #define COMMON_SAMPLE_RATE 16000
 
@@ -287,43 +286,12 @@ void sam_print_usage(int argc, char ** argv, const sam_params & params);
 // Terminal utils
 //
 
-#define SQR(X)    ((X) * (X))
-#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40
 
-/**
+// Terminal color map. 10 colors grouped in ranges [0.0, 0.1, ..., 0.9]
- * Quantizes 24-bit RGB to xterm256 code range [16,256).
+// Lowest is red, middle is yellow, highest is green.
- */
-static int rgb2xterm256(int r, int g, int b) {
-    unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377};
-    int av, ir, ig, ib, il, qr, qg, qb, ql;
-    av = r * .299 + g * .587 + b * .114 + .5;
-    ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8;
-    qr = cube[(ir = UNCUBE(r))];
-    qg = cube[(ig = UNCUBE(g))];
-    qb = cube[(ib = UNCUBE(b))];
-    if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <=
-        SQR(ql - r) + SQR(ql - g) + SQR(ql - b))
-        return ir * 36 + ig * 6 + ib + 020;
-    return il + 0350;
-}
-
-static std::string set_xterm256_foreground(int r, int g, int b) {
-    int x = rgb2xterm256(r, g, b);
-    std::ostringstream oss;
-    oss << "\033[38;5;" << x << "m";
-    return oss.str();
-}
-
-// Lowest is red, middle is yellow, highest is green. Color scheme from
-// Paul Tol; it is colorblind friendly https://personal.sron.nl/~pault/
 const std::vector<std::string> k_colors = {
-    set_xterm256_foreground(220,   5,  12),
+    "\033[38;5;196m", "\033[38;5;202m", "\033[38;5;208m", "\033[38;5;214m", "\033[38;5;220m",
-    set_xterm256_foreground(232,  96,  28),
+    "\033[38;5;226m", "\033[38;5;190m", "\033[38;5;154m", "\033[38;5;118m", "\033[38;5;82m",
-    set_xterm256_foreground(241, 147,  45),
-    set_xterm256_foreground(246, 193,  65),
-    set_xterm256_foreground(247, 240,  86),
-    set_xterm256_foreground(144, 201, 135),
-    set_xterm256_foreground( 78, 178, 101),
 };
 
 //

examples/deprecation-warning/CMakeLists.txt

@@ -1,4 +0,0 @@
-add_executable(main ./deprecation-warning.cpp)
-add_executable(bench ./deprecation-warning.cpp)
-add_executable(stream ./deprecation-warning.cpp)
-add_executable(command ./deprecation-warning.cpp)

examples/deprecation-warning/README.md

@@ -1,17 +0,0 @@
-# Migration notice for binary filenames
-
-> [!IMPORTANT]
-[2024 Dec 20] Binaries have been renamed w/ a `whisper-` prefix. `main` is now `whisper-cli`, `server` is `whisper-server`, etc (https://github.com/ggerganov/whisper.cpp/pull/2648)
-
-This migration was important, but it is a breaking change that may not always be immediately obvious to users.
-
-Please update all scripts and workflows to use the new binary names.
-
-| Old Filename | New Filename |
-| ---- | ---- |
-| main | whisper-cli |
-| bench | whisper-bench |
-| stream | whisper-stream |
-| command | whisper-command |
-| server | whisper-server |
-| talk-llama | whisper-talk-llama |

examples/deprecation-warning/deprecation-warning.cpp

@@ -1,38 +0,0 @@
-// Warns users that this filename was deprecated, and provides a link for more information.
-
-#include <cstdio>
-#include <string>
-
-// Main
-int main(int argc, char** argv) {
-    std::string filename = "main";
-    if (argc >= 1) {
-        filename = argv[0];
-    }
-
-    // Get only the program name from the full path
-    size_t pos = filename.find_last_of("/\\");
-    if (pos != std::string::npos) {
-        filename = filename.substr(pos+1);
-    }
-
-    // Append "whisper-" to the beginning of filename to get the replacemnt filename
-    std::string replacement_filename = "whisper-" + filename;
-
-    // The exception is if the filename is "main", then our replacement filename is "whisper-cli"
-    if (filename == "main") {
-        replacement_filename = "whisper-cli";
-    }
-
-    if (filename == "main.exe") {
-        replacement_filename = "whisper-cli.exe";
-    }
-
-    fprintf(stdout, "\n");
-    fprintf(stdout, "WARNING: The binary '%s' is deprecated.\n", filename.c_str());
-    fprintf(stdout, " Please use '%s' instead.\n", replacement_filename.c_str());
-    fprintf(stdout, " See https://github.com/ggerganov/whisper.cpp/tree/master/examples/deprecation-warning/README.md for more information.\n");
-    fprintf(stdout, "\n");
-
-    return EXIT_FAILURE;
-}

examples/ffmpeg-transcode.cpp

@@ -204,6 +204,8 @@ static int decode_audio(struct audio_buffer *audio_buf, s16 **data, int *size)
     const size_t errbuffsize = 1024;
     char errbuff[errbuffsize];
 
+    av_register_all(); // from avformat. Still a must-have call for ffmpeg v3! (can be skipped for later versions)
+
     fmt_ctx = avformat_alloc_context();
     avio_ctx_buffer = (u8*)av_malloc(AVIO_CTX_BUF_SZ);
     LOG("Creating an avio context: AVIO_CTX_BUF_SZ=%d\n", AVIO_CTX_BUF_SZ);
@@ -319,7 +321,7 @@ int ffmpeg_decode_audio(const std::string &ifname, std::vector<uint8_t>& owav_da
         LOG("Couldn't map input file %s\n", ifname.c_str());
         return err;
     }
-    LOG("Mapped input file: %s size: %d\n", ibuf, (int) ibuf_size);
+    LOG("Mapped input file: %x size: %d\n", ibuf, ibuf_size);
     struct audio_buffer inaudio_buf;
     inaudio_buf.ptr = ibuf;
     inaudio_buf.size = ibuf_size;

examples/generate-karaoke.sh

@@ -11,7 +11,7 @@
 # Press Ctrl+C to stop recording
 #
 
-executable="./build/bin/whisper-cli"
+executable="./main"
 model="base.en"
 model_path="models/ggml-$model.bin"
 
@@ -46,7 +46,7 @@ ffmpeg -y -i ./rec.wav -ar 16000 -ac 1 -c:a pcm_s16le ./rec16.wav > /dev/null 2>
 
 # run Whisper
 echo "Processing ..."
-${executable} -m models/ggml-base.en.bin rec16.wav -owts > /dev/null 2>&1
+./main -m models/ggml-base.en.bin rec16.wav -owts > /dev/null 2>&1
 
 # generate Karaoke video
 echo "Generating video ..."

examples/livestream.sh

@@ -14,7 +14,7 @@ model="base.en"
 
 check_requirements()
 {
-    if ! command -v ./build/bin/whisper-cli &>/dev/null; then
+    if ! command -v ./main &>/dev/null; then
         echo "whisper.cpp main executable is required (make)"
         exit 1
     fi
@@ -48,7 +48,7 @@ if [ -n "$3" ]; then
 fi
 
 # Whisper models
-models=( "tiny.en" "tiny" "base.en" "base" "small.en" "small" "medium.en" "medium" "large-v1" "large-v2" "large-v3" "large-v3-turbo" )
+models=( "tiny.en" "tiny" "base.en" "base" "small.en" "small" "medium.en" "medium" "large-v1" "large-v2" "large-v3" )
 
 # list available models
 function list_models {
@@ -100,7 +100,7 @@ while [ $running -eq 1 ]; do
         err=$(cat /tmp/whisper-live.err | wc -l)
     done
 
-    ./build/bin/whisper-cli -t 8 -m ./models/ggml-${model}.bin -f /tmp/whisper-live.wav --no-timestamps -otxt 2> /tmp/whispererr | tail -n 1
+    ./main -t 8 -m ./models/ggml-${model}.bin -f /tmp/whisper-live.wav --no-timestamps -otxt 2> /tmp/whispererr | tail -n 1
 
     while [ $SECONDS -lt $((($i+1)*$step_s)) ]; do
         sleep 1
@@ -109,4 +109,4 @@ while [ $running -eq 1 ]; do
 done
 
 killall -v ffmpeg
-killall -v whisper-cli
+killall -v main

examples/lsp/lsp.cpp

@@ -181,7 +181,7 @@ static json unguided_transcription(struct whisper_context * ctx, audio_async &au
     wparams.n_threads        = params.n_threads;
 
     wparams.audio_ctx        = params.audio_ctx;
-    wparams.suppress_nst     = true;
+    wparams.suppress_non_speech_tokens = true;
     // run the transformer and a single decoding pass
     if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {
         fprintf(stderr, "%s: ERROR: whisper_full() failed\n", __func__);
@@ -225,7 +225,7 @@ static json guided_transcription(struct whisper_context * ctx, audio_async &audi
     wparams.prompt_tokens    = cs.prompt_tokens.data();
     wparams.prompt_n_tokens  = cs.prompt_tokens.size();
     // TODO: properly expose as option
-    wparams.suppress_nst     = true;
+    wparams.suppress_non_speech_tokens = true;
 
     // run the transformer and a single decoding pass
     if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {

examples/main/CMakeLists.txt

@@ -1,8 +1,6 @@
-set(TARGET whisper-cli)
+set(TARGET main)
-add_executable(${TARGET} cli.cpp)
+add_executable(${TARGET} main.cpp)
 
 include(DefaultTargetOptions)
 
 target_link_libraries(${TARGET} PRIVATE common whisper ${FFMPEG_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
-
-install(TARGETS ${TARGET} RUNTIME)

examples/main/README.md

@@ -1,12 +1,12 @@
-# whisper.cpp/examples/cli
+# main
 
 This is the main example demonstrating most of the functionality of the Whisper model.
 It can be used as a reference for using the `whisper.cpp` library in other projects.
 
 ```
-./build/bin/whisper-cli -h
+./main -h
 
-usage: ./build-pkg/bin/whisper-cli [options] file0.wav file1.wav ...
+usage: ./main [options] file0.wav file1.wav ...
 
 options:
   -h,        --help              [default] show this help message and exit
@@ -20,12 +20,9 @@ options:
   -sow,      --split-on-word     [false  ] split on word rather than on token
   -bo N,     --best-of N         [5      ] number of best candidates to keep
   -bs N,     --beam-size N       [5      ] beam size for beam search
-  -ac N,     --audio-ctx N       [0      ] audio context size (0 - all)
   -wt N,     --word-thold N      [0.01   ] word timestamp probability threshold
   -et N,     --entropy-thold N   [2.40   ] entropy threshold for decoder fail
   -lpt N,    --logprob-thold N   [-1.00  ] log probability threshold for decoder fail
-  -tp,       --temperature N     [0.00   ] The sampling temperature, between 0 and 1
-  -tpi,      --temperature-inc N [0.20   ] The increment of temperature, between 0 and 1
   -debug,    --debug-mode        [false  ] enable debug mode (eg. dump log_mel)
   -tr,       --translate         [false  ] translate from source language to english
   -di,       --diarize           [false  ] stereo audio diarization
@@ -41,23 +38,16 @@ options:
   -oj,       --output-json       [false  ] output result in a JSON file
   -ojf,      --output-json-full  [false  ] include more information in the JSON file
   -of FNAME, --output-file FNAME [       ] output file path (without file extension)
-  -np,       --no-prints         [false  ] do not print anything other than the results
   -ps,       --print-special     [false  ] print special tokens
   -pc,       --print-colors      [false  ] print colors
   -pp,       --print-progress    [false  ] print progress
   -nt,       --no-timestamps     [false  ] do not print timestamps
   -l LANG,   --language LANG     [en     ] spoken language ('auto' for auto-detect)
   -dl,       --detect-language   [false  ] exit after automatically detecting language
-             --prompt PROMPT     [       ] initial prompt (max n_text_ctx/2 tokens)
+             --prompt PROMPT     [       ] initial prompt
   -m FNAME,  --model FNAME       [models/ggml-base.en.bin] model path
   -f FNAME,  --file FNAME        [       ] input WAV file path
   -oved D,   --ov-e-device DNAME [CPU    ] the OpenVINO device used for encode inference
-  -dtw MODEL --dtw MODEL         [       ] compute token-level timestamps
   -ls,       --log-score         [false  ] log best decoder scores of tokens
   -ng,       --no-gpu            [false  ] disable GPU
-  -fa,       --flash-attn        [false  ] flash attention
-  --suppress-regex REGEX         [       ] regular expression matching tokens to suppress
-  --grammar GRAMMAR              [       ] GBNF grammar to guide decoding
-  --grammar-rule RULE            [       ] top-level GBNF grammar rule name
-  --grammar-penalty N            [100.0  ] scales down logits of nongrammar tokens
 ```

examples/main/main.cpp

@@ -43,7 +43,6 @@ struct whisper_params {
     float word_thold      =  0.01f;
     float entropy_thold   =  2.40f;
     float logprob_thold   = -1.00f;
-    float no_speech_thold =  0.6f;
     float grammar_penalty = 100.0f;
     float temperature     = 0.0f;
     float temperature_inc = 0.2f;
@@ -71,7 +70,6 @@ struct whisper_params {
     bool log_score       = false;
     bool use_gpu         = true;
     bool flash_attn      = false;
-    bool suppress_nst    = false;
 
     std::string language  = "en";
     std::string prompt;
@@ -106,11 +104,6 @@ static char * whisper_param_turn_lowercase(char * in){
     return in;
 }
 
-static char * requires_value_error(const std::string & arg) {
-    fprintf(stderr, "error: argument %s requires value\n", arg.c_str());
-    exit(0);
-}
-
 static bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
     for (int i = 1; i < argc; i++) {
         std::string arg = argv[i];
@@ -129,23 +122,21 @@ static bool whisper_params_parse(int argc, char ** argv, whisper_params & params
             whisper_print_usage(argc, argv, params);
             exit(0);
         }
-        #define ARGV_NEXT (((i + 1) < argc) ? argv[++i] : requires_value_error(arg))
+        else if (arg == "-t"    || arg == "--threads")         { params.n_threads       = std::stoi(argv[++i]); }
-        else if (arg == "-t"    || arg == "--threads")         { params.n_threads       = std::stoi(ARGV_NEXT); }
+        else if (arg == "-p"    || arg == "--processors")      { params.n_processors    = std::stoi(argv[++i]); }
-        else if (arg == "-p"    || arg == "--processors")      { params.n_processors    = std::stoi(ARGV_NEXT); }
+        else if (arg == "-ot"   || arg == "--offset-t")        { params.offset_t_ms     = std::stoi(argv[++i]); }
-        else if (arg == "-ot"   || arg == "--offset-t")        { params.offset_t_ms     = std::stoi(ARGV_NEXT); }
+        else if (arg == "-on"   || arg == "--offset-n")        { params.offset_n        = std::stoi(argv[++i]); }
-        else if (arg == "-on"   || arg == "--offset-n")        { params.offset_n        = std::stoi(ARGV_NEXT); }
+        else if (arg == "-d"    || arg == "--duration")        { params.duration_ms     = std::stoi(argv[++i]); }
-        else if (arg == "-d"    || arg == "--duration")        { params.duration_ms     = std::stoi(ARGV_NEXT); }
+        else if (arg == "-mc"   || arg == "--max-context")     { params.max_context     = std::stoi(argv[++i]); }
-        else if (arg == "-mc"   || arg == "--max-context")     { params.max_context     = std::stoi(ARGV_NEXT); }
+        else if (arg == "-ml"   || arg == "--max-len")         { params.max_len         = std::stoi(argv[++i]); }
-        else if (arg == "-ml"   || arg == "--max-len")         { params.max_len         = std::stoi(ARGV_NEXT); }
+        else if (arg == "-bo"   || arg == "--best-of")         { params.best_of         = std::stoi(argv[++i]); }
-        else if (arg == "-bo"   || arg == "--best-of")         { params.best_of         = std::stoi(ARGV_NEXT); }
+        else if (arg == "-bs"   || arg == "--beam-size")       { params.beam_size       = std::stoi(argv[++i]); }
-        else if (arg == "-bs"   || arg == "--beam-size")       { params.beam_size       = std::stoi(ARGV_NEXT); }
+        else if (arg == "-ac"   || arg == "--audio-ctx")       { params.audio_ctx       = std::stoi(argv[++i]); }
-        else if (arg == "-ac"   || arg == "--audio-ctx")       { params.audio_ctx       = std::stoi(ARGV_NEXT); }
+        else if (arg == "-wt"   || arg == "--word-thold")      { params.word_thold      = std::stof(argv[++i]); }
-        else if (arg == "-wt"   || arg == "--word-thold")      { params.word_thold      = std::stof(ARGV_NEXT); }
+        else if (arg == "-et"   || arg == "--entropy-thold")   { params.entropy_thold   = std::stof(argv[++i]); }
-        else if (arg == "-et"   || arg == "--entropy-thold")   { params.entropy_thold   = std::stof(ARGV_NEXT); }
+        else if (arg == "-lpt"  || arg == "--logprob-thold")   { params.logprob_thold   = std::stof(argv[++i]); }
-        else if (arg == "-lpt"  || arg == "--logprob-thold")   { params.logprob_thold   = std::stof(ARGV_NEXT); }
+        else if (arg == "-tp"   || arg == "--temperature")     { params.temperature     = std::stof(argv[++i]); }
-        else if (arg == "-nth"  || arg == "--no-speech-thold") { params.no_speech_thold = std::stof(ARGV_NEXT); }
+        else if (arg == "-tpi"  || arg == "--temperature-inc") { params.temperature_inc = std::stof(argv[++i]); }
-        else if (arg == "-tp"   || arg == "--temperature")     { params.temperature     = std::stof(ARGV_NEXT); }
-        else if (arg == "-tpi"  || arg == "--temperature-inc") { params.temperature_inc = std::stof(ARGV_NEXT); }
         else if (arg == "-debug"|| arg == "--debug-mode")      { params.debug_mode      = true; }
         else if (arg == "-tr"   || arg == "--translate")       { params.translate       = true; }
         else if (arg == "-di"   || arg == "--diarize")         { params.diarize         = true; }
@@ -157,31 +148,30 @@ static bool whisper_params_parse(int argc, char ** argv, whisper_params & params
         else if (arg == "-osrt" || arg == "--output-srt")      { params.output_srt      = true; }
         else if (arg == "-owts" || arg == "--output-words")    { params.output_wts      = true; }
         else if (arg == "-olrc" || arg == "--output-lrc")      { params.output_lrc      = true; }
-        else if (arg == "-fp"   || arg == "--font-path")       { params.font_path       = ARGV_NEXT; }
+        else if (arg == "-fp"   || arg == "--font-path")       { params.font_path       = argv[++i]; }
         else if (arg == "-ocsv" || arg == "--output-csv")      { params.output_csv      = true; }
         else if (arg == "-oj"   || arg == "--output-json")     { params.output_jsn      = true; }
         else if (arg == "-ojf"  || arg == "--output-json-full"){ params.output_jsn_full = params.output_jsn = true; }
-        else if (arg == "-of"   || arg == "--output-file")     { params.fname_out.emplace_back(ARGV_NEXT); }
+        else if (arg == "-of"   || arg == "--output-file")     { params.fname_out.emplace_back(argv[++i]); }
         else if (arg == "-np"   || arg == "--no-prints")       { params.no_prints       = true; }
         else if (arg == "-ps"   || arg == "--print-special")   { params.print_special   = true; }
         else if (arg == "-pc"   || arg == "--print-colors")    { params.print_colors    = true; }
         else if (arg == "-pp"   || arg == "--print-progress")  { params.print_progress  = true; }
         else if (arg == "-nt"   || arg == "--no-timestamps")   { params.no_timestamps   = true; }
-        else if (arg == "-l"    || arg == "--language")        { params.language        = whisper_param_turn_lowercase(ARGV_NEXT); }
+        else if (arg == "-l"    || arg == "--language")        { params.language        = whisper_param_turn_lowercase(argv[++i]); }
         else if (arg == "-dl"   || arg == "--detect-language") { params.detect_language = true; }
-        else if (                  arg == "--prompt")          { params.prompt          = ARGV_NEXT; }
+        else if (                  arg == "--prompt")          { params.prompt          = argv[++i]; }
-        else if (arg == "-m"    || arg == "--model")           { params.model           = ARGV_NEXT; }
+        else if (arg == "-m"    || arg == "--model")           { params.model           = argv[++i]; }
-        else if (arg == "-f"    || arg == "--file")            { params.fname_inp.emplace_back(ARGV_NEXT); }
+        else if (arg == "-f"    || arg == "--file")            { params.fname_inp.emplace_back(argv[++i]); }
-        else if (arg == "-oved" || arg == "--ov-e-device")     { params.openvino_encode_device = ARGV_NEXT; }
+        else if (arg == "-oved" || arg == "--ov-e-device")     { params.openvino_encode_device = argv[++i]; }
-        else if (arg == "-dtw"  || arg == "--dtw")             { params.dtw             = ARGV_NEXT; }
+        else if (arg == "-dtw"  || arg == "--dtw")             { params.dtw             = argv[++i]; }
         else if (arg == "-ls"   || arg == "--log-score")       { params.log_score       = true; }
         else if (arg == "-ng"   || arg == "--no-gpu")          { params.use_gpu         = false; }
         else if (arg == "-fa"   || arg == "--flash-attn")      { params.flash_attn      = true; }
-        else if (arg == "-sns"  || arg == "--suppress-nst")    { params.suppress_nst    = true; }
+        else if (                  arg == "--suppress-regex")  { params.suppress_regex  = argv[++i]; }
-        else if (                  arg == "--suppress-regex")  { params.suppress_regex  = ARGV_NEXT; }
+        else if (                  arg == "--grammar")         { params.grammar         = argv[++i]; }
-        else if (                  arg == "--grammar")         { params.grammar         = ARGV_NEXT; }
+        else if (                  arg == "--grammar-rule")    { params.grammar_rule    = argv[++i]; }
-        else if (                  arg == "--grammar-rule")    { params.grammar_rule    = ARGV_NEXT; }
+        else if (                  arg == "--grammar-penalty") { params.grammar_penalty = std::stof(argv[++i]); }
-        else if (                  arg == "--grammar-penalty") { params.grammar_penalty = std::stof(ARGV_NEXT); }
         else {
             fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
             whisper_print_usage(argc, argv, params);
@@ -212,7 +202,6 @@ static void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params
     fprintf(stderr, "  -wt N,     --word-thold N      [%-7.2f] word timestamp probability threshold\n",         params.word_thold);
     fprintf(stderr, "  -et N,     --entropy-thold N   [%-7.2f] entropy threshold for decoder fail\n",           params.entropy_thold);
     fprintf(stderr, "  -lpt N,    --logprob-thold N   [%-7.2f] log probability threshold for decoder fail\n",   params.logprob_thold);
-    fprintf(stderr, "  -nth N,    --no-speech-thold N [%-7.2f] no speech threshold\n",                          params.no_speech_thold);
     fprintf(stderr, "  -tp,       --temperature N     [%-7.2f] The sampling temperature, between 0 and 1\n",    params.temperature);
     fprintf(stderr, "  -tpi,      --temperature-inc N [%-7.2f] The increment of temperature, between 0 and 1\n",params.temperature_inc);
     fprintf(stderr, "  -debug,    --debug-mode        [%-7s] enable debug mode (eg. dump log_mel)\n",           params.debug_mode ? "true" : "false");
@@ -245,7 +234,6 @@ static void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params
     fprintf(stderr, "  -ls,       --log-score         [%-7s] log best decoder scores of tokens\n",              params.log_score?"true":"false");
     fprintf(stderr, "  -ng,       --no-gpu            [%-7s] disable GPU\n",                                    params.use_gpu ? "false" : "true");
     fprintf(stderr, "  -fa,       --flash-attn        [%-7s] flash attention\n",                                params.flash_attn ? "true" : "false");
-    fprintf(stderr, "  -sns,      --suppress-nst      [%-7s] suppress non-speech tokens\n",                     params.suppress_nst ? "true" : "false");
     fprintf(stderr, "  --suppress-regex REGEX         [%-7s] regular expression matching tokens to suppress\n", params.suppress_regex.c_str());
     fprintf(stderr, "  --grammar GRAMMAR              [%-7s] GBNF grammar to guide decoding\n",                 params.grammar.c_str());
     fprintf(stderr, "  --grammar-rule RULE            [%-7s] top-level GBNF grammar rule name\n",               params.grammar_rule.c_str());
@@ -1009,7 +997,6 @@ int main(int argc, char ** argv) {
         if (params.dtw == "large.v1")  cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V1;
         if (params.dtw == "large.v2")  cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V2;
         if (params.dtw == "large.v3")  cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V3;
-        if (params.dtw == "large.v3.turbo")  cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V3_TURBO;
 
         if (cparams.dtw_aheads_preset == WHISPER_AHEADS_NONE) {
             fprintf(stderr, "error: unknown DTW preset '%s'\n", params.dtw.c_str());
@@ -1133,12 +1120,9 @@ int main(int argc, char ** argv) {
 
             wparams.entropy_thold    = params.entropy_thold;
             wparams.logprob_thold    = params.logprob_thold;
-            wparams.no_speech_thold  = params.no_speech_thold;
 
             wparams.no_timestamps    = params.no_timestamps;
 
-            wparams.suppress_nst     = params.suppress_nst;
-
             whisper_print_user_data user_data = { &params, &pcmf32s, 0 };
 
             const auto & grammar_parsed = params.grammar_parsed;

examples/python/whisper_processor.py

@@ -21,7 +21,7 @@ def process_audio(wav_file, model_name="base.en"):
     if not os.path.exists(wav_file):
         raise FileNotFoundError(f"WAV file not found: {wav_file}")
 
-    full_command = f"./main -m {model} -f {wav_file} -nt"
+    full_command = f"./main -m {model} -f {wav_file} -np -nt"
 
     # Execute the command
     process = subprocess.Popen(full_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

examples/server/CMakeLists.txt

@@ -1,4 +1,4 @@
-set(TARGET whisper-server)
+set(TARGET server)
 add_executable(${TARGET} server.cpp httplib.h)
 
 include(DefaultTargetOptions)
@@ -8,5 +8,3 @@ target_link_libraries(${TARGET} PRIVATE common json_cpp whisper ${CMAKE_THREAD_L
 if (WIN32)
     target_link_libraries(${TARGET} PRIVATE ws2_32)
 endif()
-
-install(TARGETS ${TARGET} RUNTIME)

examples/server/README.md

@@ -1,4 +1,4 @@
-# whisper.cpp/examples/server
+# whisper.cpp http server
 
 Simple http server. WAV Files are passed to the inference model via http requests.
 
@@ -7,9 +7,9 @@ https://github.com/ggerganov/whisper.cpp/assets/1991296/e983ee53-8741-4eb5-9048-
 ## Usage
 
 ```
-./build/bin/whisper-server -h
+./server -h
 
-usage: ./build/bin/whisper-server [options]
+usage: ./bin/server [options]
 
 options:
   -h,        --help              [default] show this help message and exit

examples/server/server.cpp

@@ -34,7 +34,6 @@ struct server_params
     std::string hostname = "127.0.0.1";
     std::string public_path = "examples/server/public";
     std::string request_path = "";
-    std::string inference_path = "/inference";
 
     int32_t port          = 8080;
     int32_t read_timeout  = 600;
@@ -61,7 +60,6 @@ struct whisper_params {
     float logprob_thold   = -1.00f;
     float temperature     =  0.00f;
     float temperature_inc =  0.20f;
-    float no_speech_thold = 0.6f;
 
     bool debug_mode      = false;
     bool translate       = false;
@@ -77,7 +75,6 @@ struct whisper_params {
     bool no_timestamps   = false;
     bool use_gpu         = true;
     bool flash_attn      = false;
-    bool suppress_nst    = false;
 
     std::string language        = "en";
     std::string prompt          = "";
@@ -135,10 +132,7 @@ void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & para
     fprintf(stderr, "  --port PORT,                   [%-7d] Port number for the server\n", sparams.port);
     fprintf(stderr, "  --public PATH,                 [%-7s] Path to the public folder\n", sparams.public_path.c_str());
     fprintf(stderr, "  --request-path PATH,           [%-7s] Request path for all requests\n", sparams.request_path.c_str());
-    fprintf(stderr, "  --inference-path PATH,         [%-7s] Inference path for all requests\n", sparams.inference_path.c_str());
+    fprintf(stderr, "  --convert,                     [%-7s] Convert audio to WAV, requires ffmpeg on the server", sparams.ffmpeg_converter ? "true" : "false");
-    fprintf(stderr, "  --convert,                     [%-7s] Convert audio to WAV, requires ffmpeg on the server\n", sparams.ffmpeg_converter ? "true" : "false");
-    fprintf(stderr, "  -sns,      --suppress-nst      [%-7s] suppress non-speech tokens\n", params.suppress_nst ? "true" : "false");
-    fprintf(stderr, "  -nth N,    --no-speech-thold N [%-7.2f] no speech threshold\n",   params.no_speech_thold);
     fprintf(stderr, "\n");
 }
 
@@ -183,15 +177,11 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params, serve
         else if (arg == "-dtw"  || arg == "--dtw")             { params.dtw             = argv[++i]; }
         else if (arg == "-ng"   || arg == "--no-gpu")          { params.use_gpu         = false; }
         else if (arg == "-fa"   || arg == "--flash-attn")      { params.flash_attn      = true; }
-        else if (arg == "-sns"  || arg == "--suppress-nst")    { params.suppress_nst    = true; }
-        else if (arg == "-nth"  || arg == "--no-speech-thold") { params.no_speech_thold = std::stof(argv[++i]); }
-
         // server params
         else if (                  arg == "--port")            { sparams.port        = std::stoi(argv[++i]); }
         else if (                  arg == "--host")            { sparams.hostname    = argv[++i]; }
         else if (                  arg == "--public")          { sparams.public_path = argv[++i]; }
         else if (                  arg == "--request-path")    { sparams.request_path = argv[++i]; }
-        else if (                  arg == "--inference-path")  { sparams.inference_path = argv[++i]; }
         else if (                  arg == "--convert")         { sparams.ffmpeg_converter     = true; }
         else {
             fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
@@ -226,7 +216,7 @@ void check_ffmpeg_availibility() {
 bool convert_to_wav(const std::string & temp_filename, std::string & error_resp) {
     std::ostringstream cmd_stream;
     std::string converted_filename_temp = temp_filename + "_temp.wav";
-    cmd_stream << "ffmpeg -i \"" << temp_filename << "\" -y -ar 16000 -ac 1 -c:a pcm_s16le \"" << converted_filename_temp << "\" 2>&1";
+    cmd_stream << "ffmpeg -i \"" << temp_filename << "\" -ar 16000 -ac 1 -c:a pcm_s16le \"" << converted_filename_temp << "\" 2>&1";
     std::string cmd = cmd_stream.str();
 
     int status = std::system(cmd.c_str());
@@ -479,14 +469,6 @@ void get_req_parameters(const Request & req, whisper_params & params)
     {
         params.temperature_inc = std::stof(req.get_file_value("temperature_inc").content);
     }
-    if (req.has_file("suppress_non_speech"))
-    {
-        params.suppress_nst = parse_str_to_bool(req.get_file_value("suppress_non_speech").content);
-    }
-    if (req.has_file("suppress_nst"))
-    {
-        params.suppress_nst = parse_str_to_bool(req.get_file_value("suppress_nst").content);
-    }
 }
 
 }  // namespace
@@ -662,10 +644,10 @@ int main(int argc, char ** argv) {
         return false;
     });
 
-    svr.Options(sparams.request_path + sparams.inference_path, [&](const Request &, Response &){
+    svr.Options(sparams.request_path + "/inference", [&](const Request &, Response &){
     });
 
-    svr.Post(sparams.request_path + sparams.inference_path, [&](const Request &req, Response &res){
+    svr.Post(sparams.request_path + "/inference", [&](const Request &req, Response &res){
         // acquire whisper model mutex lock
         std::lock_guard<std::mutex> lock(whisper_mutex);
 
@@ -692,9 +674,7 @@ int main(int argc, char ** argv) {
         if (sparams.ffmpeg_converter) {
             // if file is not wav, convert to wav
             // write to temporary file
-            //const std::string temp_filename_base = std::tmpnam(nullptr);
+            const std::string temp_filename = "whisper_server_temp_file.wav";
-            const std::string temp_filename_base = "whisper-server-tmp"; // TODO: this is a hack, remove when the mutext is removed
-            const std::string temp_filename = temp_filename_base + ".wav";
             std::ofstream temp_file{temp_filename, std::ios::binary};
             temp_file << audio_file.content;
             temp_file.close();
@@ -727,6 +707,7 @@ int main(int argc, char ** argv) {
             }
         }
 
+
         printf("Successfully loaded %s\n", filename.c_str());
 
         // print system information
@@ -794,7 +775,6 @@ int main(int argc, char ** argv) {
             wparams.beam_search.beam_size = params.beam_size;
 
             wparams.temperature      = params.temperature;
-            wparams.no_speech_thold = params.no_speech_thold;
             wparams.temperature_inc  = params.temperature_inc;
             wparams.entropy_thold    = params.entropy_thold;
             wparams.logprob_thold    = params.logprob_thold;
@@ -802,8 +782,6 @@ int main(int argc, char ** argv) {
             wparams.no_timestamps    = params.no_timestamps;
             wparams.token_timestamps = !params.no_timestamps && params.response_format == vjson_format;
 
-            wparams.suppress_nst     = params.suppress_nst;
-
             whisper_print_user_data user_data = { &params, &pcmf32s, 0 };
 
             // this callback is called on each new segment
@@ -947,7 +925,7 @@ int main(int argc, char ** argv) {
 
                 // TODO compression_ratio and no_speech_prob are not implemented yet
                 // segment["compression_ratio"] = 0;
-                segment["no_speech_prob"] = whisper_full_get_segment_no_speech_prob(ctx, i);
+                // segment["no_speech_prob"] = 0;
 
                 jres["segments"].push_back(segment);
             }

examples/stream/CMakeLists.txt

@@ -1,10 +1,9 @@
 if (WHISPER_SDL2)
-    set(TARGET whisper-stream)
+    # stream
+    set(TARGET stream)
     add_executable(${TARGET} stream.cpp)
 
     include(DefaultTargetOptions)
 
     target_link_libraries(${TARGET} PRIVATE common common-sdl whisper ${CMAKE_THREAD_LIBS_INIT})
-
-    install(TARGETS ${TARGET} RUNTIME)
 endif ()

examples/stream/README.md

@@ -1,11 +1,11 @@
-# whisper.cpp/examples/stream
+# stream
 
 This is a naive example of performing real-time inference on audio from your microphone.
-The `whisper-stream` tool samples the audio every half a second and runs the transcription continously.
+The `stream` tool samples the audio every half a second and runs the transcription continously.
 More info is available in [issue #10](https://github.com/ggerganov/whisper.cpp/issues/10).
 
 ```bash
-./build/bin/whisper-stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
+./stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
 ```
 
 https://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a80f-28ba83be7d09.mp4
@@ -15,7 +15,7 @@ https://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a
 Setting the `--step` argument to `0` enables the sliding window mode:
 
 ```bash
- ./build/bin/whisper-stream -m ./models/ggml-base.en.bin -t 6 --step 0 --length 30000 -vth 0.6
+ ./stream -m ./models/ggml-small.en.bin -t 6 --step 0 --length 30000 -vth 0.6
 ```
 
 In this mode, the tool will transcribe only after some speech activity is detected. A very
@@ -27,7 +27,7 @@ a transcription block that is suitable for parsing.
 
 ## Building
 
-The `whisper-stream` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:
+The `stream` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:
 
 ```bash
 # Install SDL2
@@ -40,10 +40,21 @@ sudo dnf install SDL2 SDL2-devel
 # Install SDL2 on Mac OS
 brew install sdl2
 
-cmake -B build -DWHISPER_SDL2=ON
+make stream
-cmake --build build --config Release
+```
 
-./build/bin/whisper-stream
+Ensure you are at the root of the repo when running `make stream`. Not within the `examples/stream` dir
+as the libraries needed like `common-sdl.h` are located within `examples`. Attempting to compile within
+`examples/steam` means your compiler cannot find them and it gives an error it cannot find the file.
+
+```bash
+whisper.cpp/examples/stream$ make stream
+g++     stream.cpp   -o stream
+stream.cpp:6:10: fatal error: common/sdl.h: No such file or directory
+    6 | #include "common/sdl.h"
+      |          ^~~~~~~~~~~~~~
+compilation terminated.
+make: *** [<builtin>: stream] Error 1
 ```
 
 ## Web version

examples/sycl/CMakeLists.txt

@@ -5,5 +5,5 @@
 set(TARGET ls-sycl-device)
 add_executable(${TARGET} ls-sycl-device.cpp)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common whisper ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PRIVATE cxx_std_17)
+target_compile_features(${TARGET} PRIVATE cxx_std_17)

examples/sycl/build.sh

@@ -7,16 +7,13 @@ cd build
 source /opt/intel/oneapi/setvars.sh
 
 #for FP16
-#cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DWHISPER_SYCL_F16=ON # faster for long-prompt inference
+#cmake .. -DWHISPER_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DWHISPER_SYCL_F16=ON # faster for long-prompt inference
 
 #for FP32
-cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx
+cmake .. -DWHISPER_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx
-
-#for other features from the examples, e.g. stream and talk link with SDL2:
-#cmake .. -DGGML_SYCL=ON -DWHISPER_SDL2=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx
 
 #build example/main only
 #cmake --build . --config Release --target main
 
 #build all binary
-cmake --build . --config Release -v
+cmake --build . --config Release -v

examples/talk-llama/CMakeLists.txt

@@ -1,12 +1,7 @@
 if (WHISPER_SDL2)
-    set(TARGET whisper-talk-llama)
+    # talk-llama
-    add_executable(${TARGET} talk-llama.cpp
+    set(TARGET talk-llama)
-        llama.cpp
+    add_executable(${TARGET} talk-llama.cpp llama.cpp unicode.cpp unicode-data.cpp)
-        llama-vocab.cpp
-        llama-grammar.cpp
-        llama-sampling.cpp
-        unicode.cpp
-        unicode-data.cpp)
     target_include_directories(${TARGET} PRIVATE ${SDL2_INCLUDE_DIRS})
 
     if (WHISPER_CLBLAST)