forked from extern/whisper.cpp
8de452c18b
* whisper : prepare infra for new decoding strategies * whisper : apply logit filters and compute logprobs * whisper : add whisper_get_logits() * whisper : separate self and cross attention memory Initial step needed for supporting parallel decoders * whisper : move probs_id buffer to whisper_context * whisper : refactor kv cache into separate struct * whisper : move self-attention kv cache to whisper_decoder * whisper : wip decoding parameters + strategies * whisper : wip decoding parameters + strategies (part 2) * whisper : wip decoding parameters + strategies (part 3) * whisper : wip decoding parameters + strategies (part 4) * whisper : fix prompt_past update to not include prompt_init * whisper : temperature + best_of support * whisper : support for compression_ration_threshold We actually use entropy, but it is similar * command : fix example to use logits instead of obsolete probs * whisper : handle empty sequence ranking * whisper : add WHISPER_DEBUG + diagnostic prints + new main args * whisper : minor fixes * whisper : add beam-search support * whisper : bug fix when there no previous context * whisper : add comments * stream : disable temperature fallback For real-time processing, we always want a single decoder running at T=0 * whisper.swiftui : update example - fix paths + add empty folders
358 lines
16 KiB
C
358 lines
16 KiB
C
#ifndef WHISPER_H
|
|
#define WHISPER_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
#include <stdbool.h>
|
|
|
|
#ifdef WHISPER_SHARED
|
|
# ifdef _WIN32
|
|
# ifdef WHISPER_BUILD
|
|
# define WHISPER_API __declspec(dllexport)
|
|
# else
|
|
# define WHISPER_API __declspec(dllimport)
|
|
# endif
|
|
# else
|
|
# define WHISPER_API __attribute__ ((visibility ("default")))
|
|
# endif
|
|
#else
|
|
# define WHISPER_API
|
|
#endif
|
|
|
|
#define WHISPER_SAMPLE_RATE 16000
|
|
#define WHISPER_N_FFT 400
|
|
#define WHISPER_N_MEL 80
|
|
#define WHISPER_HOP_LENGTH 160
|
|
#define WHISPER_CHUNK_SIZE 30
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
//
|
|
// C interface
|
|
//
|
|
// The following interface is thread-safe as long as the sample whisper_context is not used by multiple threads
|
|
// concurrently.
|
|
//
|
|
// Basic usage:
|
|
//
|
|
// #include "whisper.h"
|
|
//
|
|
// ...
|
|
//
|
|
// struct whisper_context * ctx = whisper_init_from_file("/path/to/ggml-base.en.bin");
|
|
//
|
|
// if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {
|
|
// fprintf(stderr, "failed to process audio\n");
|
|
// return 7;
|
|
// }
|
|
//
|
|
// const int n_segments = whisper_full_n_segments(ctx);
|
|
// for (int i = 0; i < n_segments; ++i) {
|
|
// const char * text = whisper_full_get_segment_text(ctx, i);
|
|
// printf("%s", text);
|
|
// }
|
|
//
|
|
// whisper_free(ctx);
|
|
//
|
|
// ...
|
|
//
|
|
// This is a demonstration of the most straightforward usage of the library.
|
|
// "pcmf32" contains the RAW audio data in 32-bit floating point format.
|
|
//
|
|
// The interface also allows for more fine-grained control over the computation, but it requires a deeper
|
|
// understanding of how the model works.
|
|
//
|
|
|
|
struct whisper_context;
|
|
|
|
typedef int whisper_token;
|
|
|
|
typedef struct whisper_token_data {
|
|
whisper_token id; // token id
|
|
whisper_token tid; // forced timestamp token id
|
|
|
|
float p; // probability of the token
|
|
float plog; // log probability of the token
|
|
float pt; // probability of the timestamp token
|
|
float ptsum; // sum of probabilities of all timestamp tokens
|
|
|
|
// token-level timestamp data
|
|
// do not use if you haven't computed token-level timestamps
|
|
int64_t t0; // start time of the token
|
|
int64_t t1; // end time of the token
|
|
|
|
float vlen; // voice length of the token
|
|
} whisper_token_data;
|
|
|
|
typedef struct whisper_model_loader {
|
|
void * context;
|
|
|
|
size_t (*read)(void * ctx, void * output, size_t read_size);
|
|
bool (*eof)(void * ctx);
|
|
void (*close)(void * ctx);
|
|
} whisper_model_loader;
|
|
|
|
// Various functions for loading a ggml whisper model.
|
|
// Allocate (almost) all memory needed for the model.
|
|
// Return NULL on failure
|
|
WHISPER_API struct whisper_context * whisper_init_from_file(const char * path_model);
|
|
WHISPER_API struct whisper_context * whisper_init_from_buffer(void * buffer, size_t buffer_size);
|
|
WHISPER_API struct whisper_context * whisper_init(struct whisper_model_loader * loader);
|
|
|
|
// Frees all memory allocated by the model.
|
|
WHISPER_API void whisper_free(struct whisper_context * ctx);
|
|
|
|
// Convert RAW PCM audio to log mel spectrogram.
|
|
// The resulting spectrogram is stored inside the provided whisper context.
|
|
// Returns 0 on success
|
|
WHISPER_API int whisper_pcm_to_mel(
|
|
struct whisper_context * ctx,
|
|
const float * samples,
|
|
int n_samples,
|
|
int n_threads);
|
|
|
|
// This can be used to set a custom log mel spectrogram inside the provided whisper context.
|
|
// Use this instead of whisper_pcm_to_mel() if you want to provide your own log mel spectrogram.
|
|
// n_mel must be 80
|
|
// Returns 0 on success
|
|
WHISPER_API int whisper_set_mel(
|
|
struct whisper_context * ctx,
|
|
const float * data,
|
|
int n_len,
|
|
int n_mel);
|
|
|
|
// Run the Whisper encoder on the log mel spectrogram stored inside the provided whisper context.
|
|
// Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first.
|
|
// offset can be used to specify the offset of the first frame in the spectrogram.
|
|
// Returns 0 on success
|
|
WHISPER_API int whisper_encode(
|
|
struct whisper_context * ctx,
|
|
int offset,
|
|
int n_threads);
|
|
|
|
// Run the Whisper decoder to obtain the logits and probabilities for the next token.
|
|
// Make sure to call whisper_encode() first.
|
|
// tokens + n_tokens is the provided context for the decoder.
|
|
// n_past is the number of tokens to use from previous decoder calls.
|
|
// Returns 0 on success
|
|
// TODO: add support for multiple decoders
|
|
WHISPER_API int whisper_decode(
|
|
struct whisper_context * ctx,
|
|
const whisper_token * tokens,
|
|
int n_tokens,
|
|
int n_past,
|
|
int n_threads);
|
|
|
|
// Convert the provided text into tokens.
|
|
// The tokens pointer must be large enough to hold the resulting tokens.
|
|
// Returns the number of tokens on success, no more than n_max_tokens
|
|
// Returns -1 on failure
|
|
// TODO: not sure if correct
|
|
WHISPER_API int whisper_tokenize(
|
|
struct whisper_context * ctx,
|
|
const char * text,
|
|
whisper_token * tokens,
|
|
int n_max_tokens);
|
|
|
|
// Largest language id (i.e. number of available languages - 1)
|
|
WHISPER_API int whisper_lang_max_id();
|
|
|
|
// Return the id of the specified language, returns -1 if not found
|
|
// Examples:
|
|
// "de" -> 2
|
|
// "german" -> 2
|
|
WHISPER_API int whisper_lang_id(const char * lang);
|
|
|
|
// Return the short string of the specified language id (e.g. 2 -> "de"), returns nullptr if not found
|
|
WHISPER_API const char * whisper_lang_str(int id);
|
|
|
|
// Use mel data at offset_ms to try and auto-detect the spoken language
|
|
// Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first
|
|
// Returns the top language id or negative on failure
|
|
// If not null, fills the lang_probs array with the probabilities of all languages
|
|
// The array must be whispe_lang_max_id() + 1 in size
|
|
// ref: https://github.com/openai/whisper/blob/main/whisper/decoding.py#L18-L69
|
|
WHISPER_API int whisper_lang_auto_detect(
|
|
struct whisper_context * ctx,
|
|
int offset_ms,
|
|
int n_threads,
|
|
float * lang_probs);
|
|
|
|
WHISPER_API int whisper_n_len (struct whisper_context * ctx); // mel length
|
|
WHISPER_API int whisper_n_vocab (struct whisper_context * ctx);
|
|
WHISPER_API int whisper_n_text_ctx (struct whisper_context * ctx);
|
|
WHISPER_API int whisper_n_audio_ctx (struct whisper_context * ctx);
|
|
WHISPER_API int whisper_is_multilingual(struct whisper_context * ctx);
|
|
|
|
// Token logits obtained from the last call to whisper_decode()
|
|
// The logits for the last token are stored in the last row
|
|
// Rows: n_tokens
|
|
// Cols: n_vocab
|
|
WHISPER_API float * whisper_get_logits(struct whisper_context * ctx);
|
|
|
|
// Token Id -> String. Uses the vocabulary in the provided context
|
|
WHISPER_API const char * whisper_token_to_str(struct whisper_context * ctx, whisper_token token);
|
|
|
|
// Special tokens
|
|
WHISPER_API whisper_token whisper_token_eot (struct whisper_context * ctx);
|
|
WHISPER_API whisper_token whisper_token_sot (struct whisper_context * ctx);
|
|
WHISPER_API whisper_token whisper_token_prev(struct whisper_context * ctx);
|
|
WHISPER_API whisper_token whisper_token_solm(struct whisper_context * ctx);
|
|
WHISPER_API whisper_token whisper_token_not (struct whisper_context * ctx);
|
|
WHISPER_API whisper_token whisper_token_beg (struct whisper_context * ctx);
|
|
WHISPER_API whisper_token whisper_token_lang(struct whisper_context * ctx, int lang_id);
|
|
|
|
// Task tokens
|
|
WHISPER_API whisper_token whisper_token_translate (void);
|
|
WHISPER_API whisper_token whisper_token_transcribe(void);
|
|
|
|
// Performance information
|
|
WHISPER_API void whisper_print_timings(struct whisper_context * ctx);
|
|
WHISPER_API void whisper_reset_timings(struct whisper_context * ctx);
|
|
|
|
// Print system information
|
|
WHISPER_API const char * whisper_print_system_info(void);
|
|
|
|
////////////////////////////////////////////////////////////////////////////
|
|
|
|
// Available sampling strategies
|
|
enum whisper_sampling_strategy {
|
|
WHISPER_SAMPLING_GREEDY, // similar to OpenAI's GreefyDecoder
|
|
WHISPER_SAMPLING_BEAM_SEARCH, // similar to OpenAI's BeamSearchDecoder
|
|
};
|
|
|
|
// Text segment callback
|
|
// Called on every newly generated text segment
|
|
// Use the whisper_full_...() functions to obtain the text segments
|
|
typedef void (*whisper_new_segment_callback)(struct whisper_context * ctx, int n_new, void * user_data);
|
|
|
|
// Encoder begin callback
|
|
// If not NULL, called before the encoder starts
|
|
// If it returns false, the computation is aborted
|
|
typedef bool (*whisper_encoder_begin_callback)(struct whisper_context * ctx, void * user_data);
|
|
|
|
// Parameters for the whisper_full() function
|
|
// If you chnage the order or add new parameters, make sure to update the default values in whisper.cpp:
|
|
// whisper_full_default_params()
|
|
struct whisper_full_params {
|
|
enum whisper_sampling_strategy strategy;
|
|
|
|
int n_threads;
|
|
int n_max_text_ctx; // max tokens to use from past text as prompt for the decoder
|
|
int offset_ms; // start offset in ms
|
|
int duration_ms; // audio duration to process in ms
|
|
|
|
bool translate;
|
|
bool no_context; // do not use initial prompt for the decoder (if any)
|
|
bool single_segment; // force single segment output (useful for streaming)
|
|
bool print_special; // print special tokens (e.g. <SOT>, <EOT>, <BEG>, etc.)
|
|
bool print_progress; // print progress information
|
|
bool print_realtime; // print results from within whisper.cpp (avoid it, use callback instead)
|
|
bool print_timestamps; // print timestamps for each text segment when printing realtime
|
|
|
|
// [EXPERIMENTAL] token-level timestamps
|
|
bool token_timestamps; // enable token-level timestamps
|
|
float thold_pt; // timestamp token probability threshold (~0.01)
|
|
float thold_ptsum; // timestamp token sum probability threshold (~0.01)
|
|
int max_len; // max segment length in characters
|
|
int max_tokens; // max tokens per segment (0 = no limit)
|
|
|
|
// [EXPERIMENTAL] speed-up techniques
|
|
// note: these can significantly reduce the quality of the output
|
|
bool speed_up; // speed-up the audio by 2x using Phase Vocoder
|
|
int audio_ctx; // overwrite the audio context size (0 = use default)
|
|
|
|
// tokens to provide to the whisper decoder as initial prompt
|
|
// these are prepended to any existing text context from a previous call
|
|
const whisper_token * prompt_tokens;
|
|
int prompt_n_tokens;
|
|
|
|
// for auto-detection, set to nullptr, "" or "auto"
|
|
const char * language;
|
|
|
|
// common decoding parameters:
|
|
bool suppress_blank; // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L89
|
|
|
|
float temperature; // initial decoding temperature, ref: https://ai.stackexchange.com/a/32478
|
|
float max_initial_ts; // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L97
|
|
float length_penalty; // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/transcribe.py#L267
|
|
|
|
// fallback parameters
|
|
// ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/transcribe.py#L274-L278
|
|
float temperature_inc;
|
|
float entropy_thold; // similar to OpenAI's "compression_ratio_threshold"
|
|
float logprob_thold;
|
|
float no_speech_thold; // TODO: not implemented
|
|
|
|
struct {
|
|
int best_of; // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/transcribe.py#L264
|
|
} greedy;
|
|
|
|
struct {
|
|
int beam_size; // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/transcribe.py#L265
|
|
|
|
float patience; // TODO: not implemented, ref: https://arxiv.org/pdf/2204.05424.pdf
|
|
} beam_search;
|
|
|
|
// called for every newly generated text segment
|
|
whisper_new_segment_callback new_segment_callback;
|
|
void * new_segment_callback_user_data;
|
|
|
|
// called each time before the encoder starts
|
|
whisper_encoder_begin_callback encoder_begin_callback;
|
|
void * encoder_begin_callback_user_data;
|
|
};
|
|
|
|
WHISPER_API struct whisper_full_params whisper_full_default_params(enum whisper_sampling_strategy strategy);
|
|
|
|
// Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
|
|
// Uses the specified decoding strategy to obtain the text.
|
|
WHISPER_API int whisper_full(
|
|
struct whisper_context * ctx,
|
|
struct whisper_full_params params,
|
|
const float * samples,
|
|
int n_samples);
|
|
|
|
// Split the input audio in chunks and process each chunk separately using whisper_full()
|
|
// It seems this approach can offer some speedup in some cases.
|
|
// However, the transcription accuracy can be worse at the beginning and end of each chunk.
|
|
WHISPER_API int whisper_full_parallel(
|
|
struct whisper_context * ctx,
|
|
struct whisper_full_params params,
|
|
const float * samples,
|
|
int n_samples,
|
|
int n_processors);
|
|
|
|
// Number of generated text segments.
|
|
// A segment can be a few words, a sentence, or even a paragraph.
|
|
WHISPER_API int whisper_full_n_segments(struct whisper_context * ctx);
|
|
|
|
// Get the start and end time of the specified segment.
|
|
WHISPER_API int64_t whisper_full_get_segment_t0(struct whisper_context * ctx, int i_segment);
|
|
WHISPER_API int64_t whisper_full_get_segment_t1(struct whisper_context * ctx, int i_segment);
|
|
|
|
// Get the text of the specified segment.
|
|
WHISPER_API const char * whisper_full_get_segment_text(struct whisper_context * ctx, int i_segment);
|
|
|
|
// Get number of tokens in the specified segment.
|
|
WHISPER_API int whisper_full_n_tokens(struct whisper_context * ctx, int i_segment);
|
|
|
|
// Get the token text of the specified token in the specified segment.
|
|
WHISPER_API const char * whisper_full_get_token_text(struct whisper_context * ctx, int i_segment, int i_token);
|
|
WHISPER_API whisper_token whisper_full_get_token_id (struct whisper_context * ctx, int i_segment, int i_token);
|
|
|
|
// Get token data for the specified token in the specified segment.
|
|
// This contains probabilities, timestamps, etc.
|
|
WHISPER_API whisper_token_data whisper_full_get_token_data(struct whisper_context * ctx, int i_segment, int i_token);
|
|
|
|
// Get the probability of the specified token in the specified segment.
|
|
WHISPER_API float whisper_full_get_token_p(struct whisper_context * ctx, int i_segment, int i_token);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif
|