forked from extern/whisper.cpp
diarization : try to cluster embedings from last encoder layer
This commit is contained in:
parent
d5d7769fa7
commit
d11f35920e
23
ggml.c
23
ggml.c
@ -8652,16 +8652,16 @@ void ggml_svd_reduce_dims(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// normalize U
|
// normalize U
|
||||||
for (int i = 0; i < n; ++i) {
|
//for (int i = 0; i < n; ++i) {
|
||||||
double sum = 0.0;
|
// double sum = 0.0;
|
||||||
for (int j = 0; j < m; ++j) {
|
// for (int j = 0; j < m; ++j) {
|
||||||
sum += U[i * m + j] * U[i * m + j];
|
// sum += U[i * m + j] * U[i * m + j];
|
||||||
}
|
// }
|
||||||
sum = sqrt(sum);
|
// sum = sqrt(sum);
|
||||||
for (int j = 0; j < m; ++j) {
|
// for (int j = 0; j < m; ++j) {
|
||||||
U[i * m + j] /= sum*sqrt((double) m);
|
// U[i * m + j] /= sum*sqrt((double) m);
|
||||||
}
|
// }
|
||||||
}
|
//}
|
||||||
|
|
||||||
// print U
|
// print U
|
||||||
//printf("U:\n");
|
//printf("U:\n");
|
||||||
@ -8675,9 +8675,10 @@ void ggml_svd_reduce_dims(
|
|||||||
//printf("\n");
|
//printf("\n");
|
||||||
|
|
||||||
|
|
||||||
|
printf("n = %d, m = %d, nd = %d\n", n, m, nd);
|
||||||
// project A0 onto U
|
// project A0 onto U
|
||||||
for (int i = 0; i < n; ++i) {
|
for (int i = 0; i < n; ++i) {
|
||||||
for (int j = 0; j < n; ++j) {
|
for (int j = 0; j < nd; ++j) {
|
||||||
A[i * nd + j] = 0.0f;
|
A[i * nd + j] = 0.0f;
|
||||||
for (int k = 0; k < m; ++k) {
|
for (int k = 0; k < m; ++k) {
|
||||||
A[i * nd + j] += A0[i * m + k] * U[j * m + k];
|
A[i * nd + j] += A0[i * m + k] * U[j * m + k];
|
||||||
|
110
whisper.cpp
110
whisper.cpp
@ -603,6 +603,8 @@ struct whisper_context {
|
|||||||
// [EXPERIMENTAL] speed-up techniques
|
// [EXPERIMENTAL] speed-up techniques
|
||||||
int32_t exp_n_audio_ctx; // 0 - use default
|
int32_t exp_n_audio_ctx; // 0 - use default
|
||||||
|
|
||||||
|
std::vector<float> audio_embd;
|
||||||
|
|
||||||
void use_buf(struct ggml_context * ctx, int i) {
|
void use_buf(struct ggml_context * ctx, int i) {
|
||||||
#if defined(WHISPER_USE_SCRATCH)
|
#if defined(WHISPER_USE_SCRATCH)
|
||||||
size_t last_size = 0;
|
size_t last_size = 0;
|
||||||
@ -1723,17 +1725,35 @@ static bool whisper_encode(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// cur
|
// cur
|
||||||
|
//{
|
||||||
|
// printf("ne0 = %d\n", cur->ne[0]);
|
||||||
|
// printf("ne1 = %d\n", cur->ne[1]);
|
||||||
|
// for (int i = 0; i < 10; ++i) {
|
||||||
|
// printf("%8.4f ", ((float *)(cur->data))[i]);
|
||||||
|
// }
|
||||||
|
// printf("... ");
|
||||||
|
// for (int i = cur->ne[0] - 10; i < cur->ne[0]; ++i) {
|
||||||
|
// printf("%8.4f ", ((float *)(cur->data))[i]);
|
||||||
|
// }
|
||||||
|
// printf("\n");
|
||||||
|
//}
|
||||||
|
|
||||||
{
|
{
|
||||||
//printf("ne0 = %d\n", cur->ne[0]);
|
//const int i0 = std::min(mel_offset, mel_inp.n_len);
|
||||||
//printf("ne1 = %d\n", cur->ne[1]);
|
//const int i1 = std::min(mel_offset + 2*n_ctx, mel_inp.n_len);
|
||||||
//for (int i = 0; i < 10; ++i) {
|
const int i0 = 0;
|
||||||
// printf("%8.4f ", ((float *)(cur->data))[i]);
|
const int i1 = cur->ne[1];
|
||||||
//}
|
|
||||||
//printf("... ");
|
//printf("i0 = %d, i1 = %d, (i1 - i0) = %d, embd size = %d\n", i0, i1, i1 - i0, cur->ne[0]);
|
||||||
//for (int i = cur->ne[0] - 10; i < cur->ne[0]; ++i) {
|
|
||||||
// printf("%8.4f ", ((float *)(cur->data))[i]);
|
wctx.audio_embd.clear();
|
||||||
//}
|
wctx.audio_embd.resize(cur->ne[0], 0.0f);
|
||||||
//printf("\n");
|
for (int j = 0; j < cur->ne[0]; ++j) {
|
||||||
|
for (int i = i0; i < i1; ++i) {
|
||||||
|
wctx.audio_embd[j] += ((float *)(cur->data))[(i - i0)*cur->ne[0] + j];
|
||||||
|
}
|
||||||
|
wctx.audio_embd[j] /= (i1 - i0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// pre-compute cross-attention memory
|
// pre-compute cross-attention memory
|
||||||
@ -4838,6 +4858,28 @@ void whisper_full_cluster_segments(struct whisper_context * ctx) {
|
|||||||
const int n_state = ctx->model.hparams.n_audio_state;
|
const int n_state = ctx->model.hparams.n_audio_state;
|
||||||
const int n_layer = ctx->model.hparams.n_audio_layer;
|
const int n_layer = ctx->model.hparams.n_audio_layer;
|
||||||
|
|
||||||
|
#if 1
|
||||||
|
// use the last layer of the encoder
|
||||||
|
{
|
||||||
|
std::vector<float> embd(n_segments*n_state);
|
||||||
|
|
||||||
|
for (int i = 0; i < n_segments; ++i) {
|
||||||
|
const auto & segment_i = ctx->result_all[i];
|
||||||
|
printf("%s: segment %3d: t0 = %7d, t1 = %7d, text = %s\n", __func__, i, (int) segment_i.t0, (int) segment_i.t1, segment_i.text.c_str());
|
||||||
|
|
||||||
|
ctx->mel.n_len = segment_i.t1;
|
||||||
|
whisper_encode(*ctx, segment_i.t0, 7, true);
|
||||||
|
|
||||||
|
for (int j = 0; j < n_state; ++j) {
|
||||||
|
embd[i*n_state + j] = ctx->audio_embd[j];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const int n_features = std::min(4, n_segments);
|
||||||
|
|
||||||
|
ggml_svd_reduce_dims(n_state, n_segments, embd.data(), n_features);
|
||||||
|
#else
|
||||||
|
// use cross kv cache of various layers
|
||||||
for (int il = 0; il < n_layer; ++il) {
|
for (int il = 0; il < n_layer; ++il) {
|
||||||
std::vector<float> embd(n_segments*n_ctx*n_state);
|
std::vector<float> embd(n_segments*n_ctx*n_state);
|
||||||
|
|
||||||
@ -4856,9 +4898,10 @@ void whisper_full_cluster_segments(struct whisper_context * ctx) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const int n_features = 64;
|
const int n_features = std::min(4, n_segments);
|
||||||
|
|
||||||
ggml_svd_reduce_dims(n_ctx*n_state, n_segments, embd.data(), n_features);
|
ggml_svd_reduce_dims(n_ctx*n_state, n_segments, embd.data(), n_features);
|
||||||
|
#endif
|
||||||
|
|
||||||
std::vector<std::vector<float>> features(n_segments);
|
std::vector<std::vector<float>> features(n_segments);
|
||||||
|
|
||||||
@ -4927,32 +4970,59 @@ void whisper_full_cluster_segments(struct whisper_context * ctx) {
|
|||||||
for (int l = 0; l < n_clusters; ++l) {
|
for (int l = 0; l < n_clusters; ++l) {
|
||||||
//sum += std::pow(whisper_distance(features[j], centroids[k])/whisper_distance(features[j], centroids[l]), 2.0/(2.0 - 1.0));
|
//sum += std::pow(whisper_distance(features[j], centroids[k])/whisper_distance(features[j], centroids[l]), 2.0/(2.0 - 1.0));
|
||||||
|
|
||||||
// use the euclidean distance
|
|
||||||
double d0 = 0.0;
|
double d0 = 0.0;
|
||||||
|
double d1 = 0.0;
|
||||||
|
|
||||||
|
// use the euclidean distance
|
||||||
|
{
|
||||||
for (int m = 0; m < n_features; ++m) {
|
for (int m = 0; m < n_features; ++m) {
|
||||||
d0 += std::pow(features[j][m] - centroids[k][m], 2.0);
|
d0 += std::pow(features[j][m] - centroids[k][m], 2.0);
|
||||||
}
|
}
|
||||||
d0 = std::sqrt(d0);
|
d0 = std::sqrt(d0);
|
||||||
|
|
||||||
double d1 = 0.0;
|
|
||||||
for (int m = 0; m < n_features; ++m) {
|
for (int m = 0; m < n_features; ++m) {
|
||||||
d1 += std::pow(features[j][m] - centroids[l][m], 2.0);
|
d1 += std::pow(features[j][m] - centroids[l][m], 2.0);
|
||||||
}
|
}
|
||||||
d1 = std::sqrt(d1);
|
d1 = std::sqrt(d1);
|
||||||
|
|
||||||
if (d1 == 0.0) {
|
|
||||||
sum += 1.0;
|
|
||||||
} else {
|
|
||||||
sum += std::pow(d0/d1, 2.0/(1.10 - 1.0));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
membership[j][k] = 1.0/sum;
|
// use the cosine distance
|
||||||
|
//{
|
||||||
|
// double dot = 0.0;
|
||||||
|
// double norm0 = 0.0;
|
||||||
|
// double norm1 = 0.0;
|
||||||
|
|
||||||
|
// for (int m = 0; m < n_features; ++m) {
|
||||||
|
// dot += features[j][m]*centroids[k][m];
|
||||||
|
// norm0 += std::pow(features[j][m], 2.0);
|
||||||
|
// norm1 += std::pow(centroids[k][m], 2.0);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// d0 = 1.0 - dot/(std::sqrt(norm0)*std::sqrt(norm1));
|
||||||
|
|
||||||
|
// dot = 0.0;
|
||||||
|
// norm0 = 0.0;
|
||||||
|
// norm1 = 0.0;
|
||||||
|
|
||||||
|
// for (int m = 0; m < n_features; ++m) {
|
||||||
|
// dot += features[j][m]*centroids[l][m];
|
||||||
|
// norm0 += std::pow(features[j][m], 2.0);
|
||||||
|
// norm1 += std::pow(centroids[l][m], 2.0);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// d1 = 1.0 - dot/(std::sqrt(norm0)*std::sqrt(norm1));
|
||||||
|
//}
|
||||||
|
|
||||||
|
sum += std::pow(d0/d1, 2.0/(1.15 - 1.0));
|
||||||
|
}
|
||||||
|
|
||||||
|
membership[j][k] = sum == 0.0 ? 0.0 : 1.0/sum;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// print the membership
|
// print the membership
|
||||||
if (i == niter - 1) {
|
if (i == niter - 1) {
|
||||||
|
//{
|
||||||
for (int i = 0; i < n_segments; ++i) {
|
for (int i = 0; i < n_segments; ++i) {
|
||||||
printf("%s: membership %3d: ", __func__, i);
|
printf("%s: membership %3d: ", __func__, i);
|
||||||
for (int j = 0; j < n_clusters; ++j) {
|
for (int j = 0; j < n_clusters; ++j) {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user