extern/whisper.cpp
1
0
Fork 1
mirror of https://github.com/ggerganov/whisper.cpp.git synced 2025-07-21 15:57:24 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
b933d17c306e800b6d919e3ee895219c3f64d5cd
whisper.cpp/src/whisper-arch.h
Daniel Bevenius e41bc5c61a vad : add initial Voice Activity Detection (VAD) support (#3065) 
* vad : add initial Voice Activity Detection (VAD) support

This commit add support for Voice Activity Detection (VAD). When enabled
this feature will process the audio input and detect speech segments.
This information is then used to reduce the number of samples that need
to be processed by whisper_full.

Resolves: https://github.com/ggml-org/whisper.cpp/issues/3003

---------

Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2025-05-12 16:10:11 +02:00

198 lines
9.0 KiB
C++
Raw Blame History

#pragma once
#include "ggml.h"
#include <map>
enum asr_tensor {
ASR_TENSOR_ENC_POS_EMBD,
ASR_TENSOR_DEC_POS_EMBD,
ASR_TENSOR_DEC_TOKEN_EMBD_WEIGHT,
ASR_TENSOR_LN_WEIGHT,
ASR_TENSOR_LN_BIAS,
ASR_TENSOR_CONV1_WEIGHT,
ASR_TENSOR_CONV1_BIAS,
ASR_TENSOR_CONV2_WEIGHT,
ASR_TENSOR_CONV2_BIAS,
ASR_TENSOR_LN_POST_WEIGHT,
ASR_TENSOR_LN_POST_BIAS,
ASR_TENSOR_MLP_LN_WEIGHT,
ASR_TENSOR_MLP_LN_BIAS,
ASR_TENSOR_MLP_0_WEIGHT,
ASR_TENSOR_MLP_0_BIAS,
ASR_TENSOR_MLP_2_WEIGHT,
ASR_TENSOR_MLP_2_BIAS,
ASR_TENSOR_ATTN_LN_WEIGHT,
ASR_TENSOR_ATTN_LN_BIAS,
ASR_TENSOR_ATTN_QUERY_WEIGHT,
ASR_TENSOR_ATTN_QUERY_BIAS,
ASR_TENSOR_ATTN_KEY_WEIGHT,
ASR_TENSOR_ATTN_VALUE_WEIGHT,
ASR_TENSOR_ATTN_VALUE_BIAS,
ASR_TENSOR_ATTN_OUT_WEIGHT,
ASR_TENSOR_ATTN_OUT_BIAS,
};
enum asr_system {
ASR_SYSTEM_ENCODER,
ASR_SYSTEM_DECODER,
ASR_SYSTEM_CROSS
};
static const std::map<asr_system, std::map<asr_tensor, const char *>> ASR_TENSOR_NAMES = {
{
ASR_SYSTEM_ENCODER,
{
{ASR_TENSOR_ENC_POS_EMBD, "encoder.positional_embedding"},
{ASR_TENSOR_CONV1_WEIGHT, "encoder.conv1.weight"},
{ASR_TENSOR_CONV1_BIAS, "encoder.conv1.bias"},
{ASR_TENSOR_CONV2_WEIGHT, "encoder.conv2.weight"},
{ASR_TENSOR_CONV2_BIAS, "encoder.conv2.bias"},
{ASR_TENSOR_LN_WEIGHT, "encoder.ln_post.weight"},
{ASR_TENSOR_LN_POST_BIAS, "encoder.ln_post.bias"},
{ASR_TENSOR_MLP_LN_WEIGHT, "encoder.blocks.%d.mlp_ln.weight"},
{ASR_TENSOR_MLP_LN_BIAS, "encoder.blocks.%d.mlp_ln.bias"},
{ASR_TENSOR_MLP_0_WEIGHT, "encoder.blocks.%d.mlp.0.weight"},
{ASR_TENSOR_MLP_0_BIAS, "encoder.blocks.%d.mlp.0.bias"},
{ASR_TENSOR_MLP_2_WEIGHT, "encoder.blocks.%d.mlp.2.weight"},
{ASR_TENSOR_MLP_2_BIAS, "encoder.blocks.%d.mlp.2.bias"},
{ASR_TENSOR_ATTN_LN_WEIGHT, "encoder.blocks.%d.attn_ln.weight"},
{ASR_TENSOR_ATTN_LN_BIAS, "encoder.blocks.%d.attn_ln.bias"},
{ASR_TENSOR_ATTN_QUERY_WEIGHT, "encoder.blocks.%d.attn.query.weight"},
{ASR_TENSOR_ATTN_QUERY_BIAS, "encoder.blocks.%d.attn.query.bias"},
{ASR_TENSOR_ATTN_KEY_WEIGHT, "encoder.blocks.%d.attn.key.weight"},
{ASR_TENSOR_ATTN_VALUE_WEIGHT, "encoder.blocks.%d.attn.value.weight"},
{ASR_TENSOR_ATTN_VALUE_BIAS, "encoder.blocks.%d.attn.value.bias"},
{ASR_TENSOR_ATTN_OUT_WEIGHT, "encoder.blocks.%d.attn.out.weight"},
{ASR_TENSOR_ATTN_OUT_BIAS, "encoder.blocks.%d.attn.out.bias"},
},
},
{
ASR_SYSTEM_DECODER,
{
{ASR_TENSOR_DEC_POS_EMBD, "decoder.positional_embedding"},
{ASR_TENSOR_DEC_TOKEN_EMBD_WEIGHT, "decoder.token_embedding.weight"},
{ASR_TENSOR_LN_WEIGHT, "decoder.ln.weight"},
{ASR_TENSOR_LN_BIAS, "decoder.ln.bias"},
{ASR_TENSOR_MLP_LN_WEIGHT, "decoder.blocks.%d.mlp_ln.weight"},
{ASR_TENSOR_MLP_LN_BIAS, "decoder.blocks.%d.mlp_ln.bias"},
{ASR_TENSOR_MLP_0_WEIGHT, "decoder.blocks.%d.mlp.0.weight"},
{ASR_TENSOR_MLP_0_BIAS, "decoder.blocks.%d.mlp.0.bias"},
{ASR_TENSOR_MLP_2_WEIGHT, "decoder.blocks.%d.mlp.2.weight"},
{ASR_TENSOR_MLP_2_BIAS, "decoder.blocks.%d.mlp.2.bias"},
{ASR_TENSOR_ATTN_LN_WEIGHT, "decoder.blocks.%d.attn_ln.weight"},
{ASR_TENSOR_ATTN_LN_BIAS, "decoder.blocks.%d.attn_ln.bias"},
{ASR_TENSOR_ATTN_QUERY_WEIGHT, "decoder.blocks.%d.attn.query.weight"},
{ASR_TENSOR_ATTN_QUERY_BIAS, "decoder.blocks.%d.attn.query.bias"},
{ASR_TENSOR_ATTN_KEY_WEIGHT, "decoder.blocks.%d.attn.key.weight"},
{ASR_TENSOR_ATTN_VALUE_WEIGHT, "decoder.blocks.%d.attn.value.weight"},
{ASR_TENSOR_ATTN_VALUE_BIAS, "decoder.blocks.%d.attn.value.bias"},
{ASR_TENSOR_ATTN_OUT_WEIGHT, "decoder.blocks.%d.attn.out.weight"},
{ASR_TENSOR_ATTN_OUT_BIAS, "decoder.blocks.%d.attn.out.bias"},
},
},
{
ASR_SYSTEM_CROSS,
{
{ASR_TENSOR_ATTN_LN_WEIGHT, "decoder.blocks.%d.cross_attn_ln.weight"},
{ASR_TENSOR_ATTN_LN_BIAS, "decoder.blocks.%d.cross_attn_ln.bias"},
{ASR_TENSOR_ATTN_QUERY_WEIGHT, "decoder.blocks.%d.cross_attn.query.weight"},
{ASR_TENSOR_ATTN_QUERY_BIAS, "decoder.blocks.%d.cross_attn.query.bias"},
{ASR_TENSOR_ATTN_KEY_WEIGHT, "decoder.blocks.%d.cross_attn.key.weight"},
{ASR_TENSOR_ATTN_VALUE_WEIGHT, "decoder.blocks.%d.cross_attn.value.weight"},
{ASR_TENSOR_ATTN_VALUE_BIAS, "decoder.blocks.%d.cross_attn.value.bias"},
{ASR_TENSOR_ATTN_OUT_WEIGHT, "decoder.blocks.%d.cross_attn.out.weight"},
{ASR_TENSOR_ATTN_OUT_BIAS, "decoder.blocks.%d.cross_attn.out.bias"},
},
},
};
static const std::map<asr_tensor, ggml_op> ASR_TENSOR_INFO = {
{ASR_TENSOR_ENC_POS_EMBD, GGML_OP_ADD},
{ASR_TENSOR_DEC_POS_EMBD, GGML_OP_GET_ROWS},
// Note: ASR_TENSOR_DEC_TOKEN_EMBD_WEIGHT is also used by GGML_OP_MAT_MUL. Need to figure out a way how to handle
// weight tensors that are used by multiple different operators when extra_buffer_type implementations accelerate
// more than just GGML_OP_MUL_MAT.
{ASR_TENSOR_DEC_TOKEN_EMBD_WEIGHT, GGML_OP_GET_ROWS},
{ASR_TENSOR_LN_WEIGHT, GGML_OP_MUL},
{ASR_TENSOR_LN_BIAS, GGML_OP_ADD},
{ASR_TENSOR_CONV1_WEIGHT, GGML_OP_IM2COL},
{ASR_TENSOR_CONV1_BIAS, GGML_OP_ADD},
{ASR_TENSOR_CONV2_WEIGHT, GGML_OP_IM2COL},
{ASR_TENSOR_CONV2_BIAS, GGML_OP_ADD},
{ASR_TENSOR_LN_POST_WEIGHT, GGML_OP_MUL},
{ASR_TENSOR_LN_POST_BIAS, GGML_OP_ADD},
{ASR_TENSOR_MLP_LN_WEIGHT, GGML_OP_MUL},
{ASR_TENSOR_MLP_LN_BIAS, GGML_OP_ADD},
{ASR_TENSOR_MLP_0_WEIGHT, GGML_OP_MUL_MAT},
{ASR_TENSOR_MLP_0_BIAS, GGML_OP_ADD},
{ASR_TENSOR_MLP_2_WEIGHT, GGML_OP_MUL_MAT},
{ASR_TENSOR_MLP_2_BIAS, GGML_OP_ADD},
{ASR_TENSOR_ATTN_LN_WEIGHT, GGML_OP_MUL},
{ASR_TENSOR_ATTN_LN_BIAS, GGML_OP_ADD},
{ASR_TENSOR_ATTN_QUERY_WEIGHT, GGML_OP_MUL_MAT},
{ASR_TENSOR_ATTN_QUERY_BIAS, GGML_OP_ADD},
{ASR_TENSOR_ATTN_KEY_WEIGHT, GGML_OP_MUL_MAT},
{ASR_TENSOR_ATTN_VALUE_WEIGHT, GGML_OP_MUL_MAT},
{ASR_TENSOR_ATTN_VALUE_BIAS, GGML_OP_ADD},
{ASR_TENSOR_ATTN_OUT_WEIGHT, GGML_OP_MUL_MAT},
{ASR_TENSOR_ATTN_OUT_BIAS, GGML_OP_ADD},
};
enum vad_tensor {
VAD_TENSOR_STFT_BASIS,
VAD_TENSOR_ENC_0_WEIGHT,
VAD_TENSOR_ENC_0_BIAS,
VAD_TENSOR_ENC_1_WEIGHT,
VAD_TENSOR_ENC_1_BIAS,
VAD_TENSOR_ENC_2_WEIGHT,
VAD_TENSOR_ENC_2_BIAS,
VAD_TENSOR_ENC_3_WEIGHT,
VAD_TENSOR_ENC_3_BIAS,
VAD_TENSOR_LSTM_WEIGHT_IH,
VAD_TENSOR_LSTM_WEIGHT_HH,
VAD_TENSOR_LSTM_BIAS_IH,
VAD_TENSOR_LSTM_BIAS_HH,
VAD_TENSOR_FINAL_CONV_WEIGHT,
VAD_TENSOR_FINAL_CONV_BIAS,
};
static const std::map<vad_tensor, ggml_op> VAD_TENSOR_OPS = {
{VAD_TENSOR_STFT_BASIS, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_0_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_0_BIAS, GGML_OP_ADD},
{VAD_TENSOR_ENC_1_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_1_BIAS, GGML_OP_ADD},
{VAD_TENSOR_ENC_2_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_2_BIAS, GGML_OP_ADD},
{VAD_TENSOR_ENC_3_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_3_BIAS, GGML_OP_ADD},
{VAD_TENSOR_LSTM_WEIGHT_IH, GGML_OP_MUL_MAT},
{VAD_TENSOR_LSTM_WEIGHT_HH, GGML_OP_MUL_MAT},
{VAD_TENSOR_LSTM_BIAS_IH, GGML_OP_ADD},
{VAD_TENSOR_LSTM_BIAS_HH, GGML_OP_ADD},
{VAD_TENSOR_FINAL_CONV_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_FINAL_CONV_BIAS, GGML_OP_ADD}
};
static const std::map<vad_tensor, const char *> VAD_TENSOR_NAMES = {
{VAD_TENSOR_STFT_BASIS, "_model.stft.forward_basis_buffer"},
{VAD_TENSOR_ENC_0_WEIGHT, "_model.encoder.0.reparam_conv.weight"},
{VAD_TENSOR_ENC_0_BIAS, "_model.encoder.0.reparam_conv.bias"},
{VAD_TENSOR_ENC_1_WEIGHT, "_model.encoder.1.reparam_conv.weight"},
{VAD_TENSOR_ENC_1_BIAS, "_model.encoder.1.reparam_conv.bias"},
{VAD_TENSOR_ENC_2_WEIGHT, "_model.encoder.2.reparam_conv.weight"},
{VAD_TENSOR_ENC_2_BIAS, "_model.encoder.2.reparam_conv.bias"},
{VAD_TENSOR_ENC_3_WEIGHT, "_model.encoder.3.reparam_conv.weight"},
{VAD_TENSOR_ENC_3_BIAS, "_model.encoder.3.reparam_conv.bias"},
{VAD_TENSOR_LSTM_WEIGHT_IH, "_model.decoder.rnn.weight_ih"},
{VAD_TENSOR_LSTM_WEIGHT_HH, "_model.decoder.rnn.weight_hh"},
{VAD_TENSOR_LSTM_BIAS_IH, "_model.decoder.rnn.bias_ih"},
{VAD_TENSOR_LSTM_BIAS_HH, "_model.decoder.rnn.bias_hh"},
{VAD_TENSOR_FINAL_CONV_WEIGHT, "_model.decoder.decoder.2.weight"},
{VAD_TENSOR_FINAL_CONV_BIAS, "_model.decoder.decoder.2.bias"}
};
Reference in New Issue View Git Blame Copy Permalink