wip : initial WASM port

Works but it is very slow because no SIMD is used.
For example, jfk.wav is processed in ~23 seconds using "tiny.en" model

CMakeLists.txt

@@ -7,12 +7,28 @@ set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
 
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     set(WHISPER_STANDALONE ON)
+    include(cmake/GitVars.cmake)
+    include(cmake/BuildTypes.cmake)
 else()
     set(WHISPER_STANDALONE OFF)
 endif()
 
+if (EMSCRIPTEN)
+    set(BUILD_SHARED_LIBS_DEFAULT OFF)
+
+    option(WHISPER_WASM_SINGLE_FILE "whisper: embed WASM inside the generated whisper.js" ON)
+else()
+    if (MINGW)
+        set(BUILD_SHARED_LIBS_DEFAULT OFF)
+    else()
+        set(BUILD_SHARED_LIBS_DEFAULT ON)
+    endif()
+endif()
+
 # options
 
+option(BUILD_SHARED_LIBS               "whisper: build shared libs" ${BUILD_SHARED_LIBS_DEFAULT})
+
 option(WHISPER_ALL_WARNINGS            "whisper: enable all compiler warnings"                   ON)
 option(WHISPER_ALL_WARNINGS_3RD_PARTY  "whisper: enable all compiler warnings in 3rd party libs" OFF)
 
@@ -21,6 +37,7 @@ option(WHISPER_SANITIZE_ADDRESS        "whisper: enable address sanitizer"   OFF
 option(WHISPER_SANITIZE_UNDEFINED      "whisper: enable undefined sanitizer" OFF)
 
 option(WHISPER_BUILD_TESTS             "whisper: build tests"    ${WHISPER_STANDALONE})
+option(WHISPER_BUILD_EXAMPLES          "whisper: build examples" ${WHISPER_STANDALONE})
 
 option(WHISPER_SUPPORT_SDL2            "whisper: support for libSDL2" OFF)
 
@@ -69,16 +86,6 @@ if (APPLE AND NOT WHISPER_NO_ACCELERATE)
     endif()
 endif()
 
-if (WHISPER_SUPPORT_SDL2)
-    # SDL2
-    find_package(SDL2 REQUIRED)
-
-    string(STRIP "${SDL2_LIBRARIES}" SDL2_LIBRARIES)
-
-    message(STATUS "SDL2_INCLUDE_DIRS = ${SDL2_INCLUDE_DIRS}")
-    message(STATUS "SDL2_LIBRARIES = ${SDL2_LIBRARIES}")
-endif()
-
 # compiler flags
 
 if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
@@ -120,6 +127,11 @@ else()
     endif()
 endif()
 
+if (EMSCRIPTEN)
+    set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS}   -pthread -msimd128")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
+endif()
+
 # whisper - this is the main library of the project
 
 set(TARGET whisper)
@@ -154,24 +166,46 @@ install(TARGETS ${TARGET}
     ARCHIVE DESTINATION lib/static
     )
 
+# bindings
+
+add_subdirectory(bindings)
+
 # programs, examples and tests
 
 if (WHISPER_STANDALONE)
+    if (NOT EMSCRIPTEN)
+        # TODO: move to examples
         # main
         set(TARGET main)
         add_executable(${TARGET} main.cpp)
         target_link_libraries(${TARGET} PRIVATE whisper ${CMAKE_THREAD_LIBS_INIT})
 
+        # TODO: move to examples
         if (WHISPER_SUPPORT_SDL2)
+            if (WHISPER_SUPPORT_SDL2)
+                # SDL2
+                find_package(SDL2 REQUIRED)
+
+                string(STRIP "${SDL2_LIBRARIES}" SDL2_LIBRARIES)
+
+                message(STATUS "SDL2_INCLUDE_DIRS = ${SDL2_INCLUDE_DIRS}")
+                message(STATUS "SDL2_LIBRARIES = ${SDL2_LIBRARIES}")
+            endif()
+
             # stream
             set(TARGET stream)
             add_executable(${TARGET} stream.cpp)
             target_include_directories(${TARGET} PRIVATE ${SDL2_INCLUDE_DIRS})
             target_link_libraries(${TARGET} PRIVATE whisper ${SDL2_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
         endif ()
+    endif()
 
     if (WHISPER_BUILD_TESTS)
         enable_testing()
         add_subdirectory(tests)
     endif ()
+
+    if (WHISPER_BUILD_EXAMPLES)
+        add_subdirectory(examples)
+    endif()
 endif ()

bindings/CMakeLists.txt

@@ -0,0 +1,3 @@
+if (EMSCRIPTEN)
+    add_subdirectory(javascript)
+endif()

bindings/javascript/.gitignore

@@ -0,0 +1 @@
+publish.log

bindings/javascript/CMakeLists.txt

@@ -0,0 +1,34 @@
+set(TARGET libwhisper)
+
+add_executable(${TARGET}
+    emscripten.cpp
+    )
+
+target_link_libraries(${TARGET} PRIVATE
+    whisper
+    )
+
+unset(EXTRA_FLAGS)
+if (WHISPER_WASM_SINGLE_FILE)
+    set(EXTRA_FLAGS "-s SINGLE_FILE=1")
+    message(STATUS "Embedding WASM inside whisper.js")
+
+    add_custom_command(
+        TARGET libwhisper POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy
+        ${CMAKE_BINARY_DIR}/bin/libwhisper.js
+        ${CMAKE_CURRENT_SOURCE_DIR}/whisper.js
+        )
+endif()
+
+set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \
+    --bind \
+    -s MODULARIZE=1 \
+    -s ASSERTIONS=1 \
+    -s USE_PTHREADS=1 \
+    -s PTHREAD_POOL_SIZE=8 \
+    -s TOTAL_MEMORY=536870912 \
+    -s FORCE_FILESYSTEM=1 \
+    -s EXPORT_NAME=\"'whisper_factory'\" \
+    ${EXTRA_FLAGS} \
+    ")

bindings/javascript/emscripten.cpp

@@ -0,0 +1,70 @@
+#include "whisper.h"
+
+#include <emscripten.h>
+#include <emscripten/bind.h>
+
+#include <vector>
+
+std::vector<struct whisper_context *> g_contexts(4, nullptr);
+
+EMSCRIPTEN_BINDINGS(whisper) {
+    emscripten::function("init", emscripten::optional_override([](const std::string & path_model) {
+        for (size_t i = 0; i < g_contexts.size(); ++i) {
+            if (g_contexts[i] == nullptr) {
+                g_contexts[i] = whisper_init(path_model.c_str());
+                return i + 1;
+            }
+        }
+
+        return (size_t) 0;
+    }));
+
+    emscripten::function("free", emscripten::optional_override([](size_t index) {
+        --index;
+
+        if (index < g_contexts.size()) {
+            whisper_free(g_contexts[index]);
+            g_contexts[index] = nullptr;
+        }
+    }));
+
+    emscripten::function("full_default", emscripten::optional_override([](size_t index, const emscripten::val & audio) {
+        --index;
+
+        if (index >= g_contexts.size()) {
+            return -1;
+        }
+
+        if (g_contexts[index] == nullptr) {
+            return -2;
+        }
+
+        struct whisper_full_params params = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);
+
+        params.print_realtime       = true;
+        params.print_progress       = false;
+        params.print_timestamps     = true;
+        params.print_special_tokens = false;
+        params.translate            = false;
+        params.language             = "en";
+        params.n_threads            = 4;
+        params.offset_ms            = 0;
+
+        std::vector<float> pcmf32;
+        const int n = audio["length"].as<int>();
+
+        emscripten::val heap = emscripten::val::module_property("HEAPU8");
+        emscripten::val memory = heap["buffer"];
+
+        pcmf32.resize(n);
+
+        emscripten::val memoryView = audio["constructor"].new_(memory, reinterpret_cast<uintptr_t>(pcmf32.data()), n);
+        memoryView.call<void>("set", audio);
+
+        int ret = whisper_full(g_contexts[index], params, pcmf32.data(), pcmf32.size());
+
+        whisper_print_timings(g_contexts[index]);
+
+        return ret;
+    }));
+}

cmake/BuildTypes.cmake

@@ -0,0 +1,54 @@
+# Add new build types
+
+# ReleaseGG - Release with enabled asserts
+
+SET(CMAKE_CXX_FLAGS_RELEASEGG
+    "-O3"
+    CACHE STRING "Flags used by the c++ compiler during release builds with enabled asserts."
+    FORCE )
+SET(CMAKE_C_FLAGS_RELEASEGG
+    "-O3"
+    CACHE STRING "Flags used by the compiler during release builds with enabled asserts."
+    FORCE )
+SET(CMAKE_EXE_LINKER_FLAGS_RELEASEGG
+    ""
+    CACHE STRING "Flags used for linking binaries during release builds with enabled asserts."
+    FORCE )
+SET(CMAKE_SHARED_LINKER_FLAGS_RELEASEGG
+    ""
+    CACHE STRING "Flags used by the shared libraries linker during release builds with enabled asserts."
+    FORCE )
+MARK_AS_ADVANCED(
+    CMAKE_CXX_FLAGS_RELEASEGG
+    CMAKE_C_FLAGS_RELEASEGG
+    CMAKE_EXE_LINKER_FLAGS_RELEASEGG
+    CMAKE_SHARED_LINKER_FLAGS_RELEASEGG )
+
+# RelWithDebInfoGG - RelWithDebInfo with enabled asserts
+
+SET(CMAKE_CXX_FLAGS_RELWITHDEBINFOGG
+    "-O2 -g"
+    CACHE STRING "Flags used by the c++ compiler during release builds with debug symbols and enabled asserts."
+    FORCE )
+SET(CMAKE_C_FLAGS_RELWITHDEBINFOGG
+    "-O2 -g"
+    CACHE STRING "Flags used by the compiler during release builds with debug symbols and enabled asserts."
+    FORCE )
+SET(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFOGG
+    ""
+    CACHE STRING "Flags used for linking binaries during release builds with debug symbols and enabled asserts."
+    FORCE )
+SET(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFOGG
+    ""
+    CACHE STRING "Flags used by the shared libraries linker during release builds with debug symbols and enabled asserts."
+    FORCE )
+MARK_AS_ADVANCED(
+    CMAKE_CXX_FLAGS_RELWITHDEBINFOGG
+    CMAKE_C_FLAGS_RELWITHDEBINFOGG
+    CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFOGG
+    CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFOGG )
+
+if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo" "ReleaseGG" "RelWithDebInfoGG")
+endif()

cmake/GitVars.cmake

@@ -0,0 +1,22 @@
+find_package(Git)
+
+# the commit's SHA1
+execute_process(COMMAND
+    "${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8
+    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    OUTPUT_VARIABLE GIT_SHA1
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+# the date of the commit
+execute_process(COMMAND
+    "${GIT_EXECUTABLE}" log -1 --format=%ad --date=local
+    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    OUTPUT_VARIABLE GIT_DATE
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+# the subject of the commit
+execute_process(COMMAND
+    "${GIT_EXECUTABLE}" log -1 --format=%s
+    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    OUTPUT_VARIABLE GIT_COMMIT_SUBJECT
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)

examples/CMakeLists.txt

@@ -0,0 +1,14 @@
+# dependencies
+
+find_package(Threads REQUIRED)
+
+# third-party
+
+#add_subdirectory(third-party)
+
+# examples
+
+if (EMSCRIPTEN)
+    add_subdirectory(whisper.wasm)
+else()
+endif()

examples/whisper.wasm/CMakeLists.txt

@@ -0,0 +1,4 @@
+set(TARGET whisper.wasm)
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/index-tmpl.html        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/index.html @ONLY)
+configure_file(${CMAKE_SOURCE_DIR}/bindings/javascript/whisper.js ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/whisper.js  COPYONLY)

examples/whisper.wasm/README.md

@@ -0,0 +1,3 @@
+# whisper.wasm
+
+Live demo: https://whisper.ggerganov.com

examples/whisper.wasm/index-tmpl.html

@@ -0,0 +1,153 @@
+<!doctype html>
+<html lang="en-us">
+    <head>
+        <title>whisper.cpp : WASM example</title>
+    </head>
+    <body>
+        <div id="main-container">
+            Minimal <b>whisper.cpp</b> example using Javascript bindings
+
+            <br><br>
+
+            Model:
+            <input type="file" id="file" name="file" onchange="loadFile(event, 'ggml.bin')" />
+            <br><br>
+
+            WAV:
+            <input type="file" id="file" name="file" onchange="loadAudio(event)" />
+            <br><br>
+
+            <button onclick="onTranscribe();">Transcribe</button>
+
+            <br><br>
+
+            <div class="cell-version">
+                <span>
+                    |
+                    Build time: <span class="nav-link">@GIT_DATE@</span> |
+                    Commit hash: <a class="nav-link" href="https://github.com/ggerganov/whisper.cpp/commit/@GIT_SHA1@">@GIT_SHA1@</a> |
+                    Commit subject: <span class="nav-link">@GIT_COMMIT_SUBJECT@</span> |
+                    <a class="nav-link" href="https://github.com/ggerganov/whisper.cpp/tree/master/examples/whisper.wasm">Source Code</a> |
+                </span>
+            </div>
+        </div>
+
+        <script type="text/javascript" src="whisper.js"></script>
+        <script type='text/javascript'>
+            window.AudioContext = window.AudioContext || window.webkitAudioContext;
+            window.OfflineAudioContext = window.OfflineAudioContext || window.webkitOfflineAudioContext;
+
+            // web audio context
+            var context = null;
+
+            // audio data
+            var audio = null;
+
+            // the whisper module instance
+            var whisper = null;
+            var instance = null;
+
+            // instantiate the whisper instance
+            // whisper_factory comes from the whisper.js module
+            whisper_factory().then(function(obj) {
+                whisper = obj;
+            });
+
+            // helper function
+            function convertTypedArray(src, type) {
+                var buffer = new ArrayBuffer(src.byteLength);
+                var baseView = new src.constructor(buffer).set(src);
+                return new type(buffer);
+            }
+
+            // initialize whisper
+            function init() {
+                if (!instance) {
+                    instance = whisper.init('ggml.bin');
+                    if (instance) {
+                        console.log('whisper instance initialized');
+                    }
+                }
+
+                if (!instance) {
+                    console.log('whisper instance initialization failed');
+                    return;
+                }
+
+                if (instance) {
+                    var ret = whisper.full_default(instance, audio);
+                    if (ret) {
+                        console.log('whisper full_default returned: ' + ret);
+                    }
+                }
+            }
+
+            function loadFile(event, fname) {
+                var file = event.target.files[0] || null;
+                if (file == null) {
+                    return;
+                }
+
+                console.log(
+                        "<p>File information: <strong>" + file.name +
+                        "</strong> type: <strong>" + file.type +
+                        "</strong> size: <strong>" + file.size +
+                        "</strong> bytes</p>"
+                );
+
+                var reader = new FileReader();
+                reader.onload = function(event) {
+                    var buf = new Uint8Array(reader.result);
+
+                    // write to WASM file using whisper.FS_createDataFile
+                    whisper.FS_createDataFile("/", fname, buf, true, true);
+                }
+                reader.readAsArrayBuffer(file);
+            }
+
+            function loadAudio(event) {
+                if (!context) {
+                    context = new AudioContext({sampleRate: 16000});
+                }
+
+                var file = event.target.files[0] || null;
+                if (file == null) {
+                    return;
+                }
+
+                console.log(
+                        "<p>Audio information: <strong>" + file.name +
+                        "</strong> type: <strong>" + file.type +
+                        "</strong> size: <strong>" + file.size +
+                        "</strong> bytes</p>"
+                );
+
+                var reader = new FileReader();
+                reader.onload = function(event) {
+                    var buf = new Uint8Array(reader.result);
+
+                    context.decodeAudioData(buf.buffer, function(audioBuffer) {
+                        var offlineContext = new OfflineAudioContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);
+                        var source = offlineContext.createBufferSource();
+                        source.buffer = audioBuffer;
+                        source.connect(offlineContext.destination);
+                        source.start(0);
+
+                        offlineContext.startRendering().then(function(renderedBuffer) {
+                            audio = renderedBuffer.getChannelData(0);
+                            //var audio16 = convertTypedArray(data, Int16Array);
+                        });
+                    });
+                }
+                reader.readAsArrayBuffer(file);
+            }
+            //
+            // Transcribe
+            //
+
+            function onTranscribe() {
+                init();
+            }
+        </script>
+    </body>
+</html>

ggml.c

@@ -7358,7 +7358,7 @@ enum ggml_opt_result ggml_opt_adam(
 
         {
             const int64_t t_end_cpu = ggml_cycles();
-            GGML_PRINT_DEBUG("time iter:      %5.3f s\n", (t_end_cpu - t_start_cpu)/CLOCKS_PER_SEC);
+            GGML_PRINT_DEBUG("time iter:      %5.3f s\n", ((float)(t_end_cpu - t_start_cpu))/CLOCKS_PER_SEC);
             UNUSED(t_end_cpu);
 
             const int64_t t_end_wall = ggml_time_us();

tests/CMakeLists.txt

@@ -1,3 +1,7 @@
+if (EMSCRIPTEN)
+    return()
+endif()
+
 set(TEST_TARGET test-main-tiny)
 add_test(NAME ${TEST_TARGET}
     COMMAND $<TARGET_FILE:main>