extern/whisper.cpp
1
0
Fork 1
mirror of https://github.com/ggerganov/whisper.cpp.git synced 2025-04-10 10:38:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

whisper : fix the bug related to word splitting errors in the "tokenize" function. (#760)

Browse Source 
Co-authored-by: AfryMask <afrymask@gmail.com>
This commit is contained in:
AfryMask 2023-04-15 01:35:03 +08:00 committed by GitHub
parent 1c5edc3cb3
commit 7e2afa4384
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
whisper.cpp
View File

@ -2449,25 +2449,20 @@ static std::vector<whisper_vocab::id> tokenize(const whisper_vocab & vocab, cons
int n = word.size(); int n = word.size();
while (i < n) { while (i < n) {
int j = n; int j = n;
bool found = false;
while (j > i) { while (j > i) {
auto it = vocab.token_to_id.find(word.substr(i, j-i)); auto sub = word.substr(i, j-i);
auto it = vocab.token_to_id.find(sub);
if (it != vocab.token_to_id.end()) { if (it != vocab.token_to_id.end()) {
tokens.push_back(it->second); tokens.push_back(it->second);
i = j; i = j;
found = true;
break; break;
} }
--j; --j;
} }
if (i == n) { if (!found) {
break; fprintf(stderr, "unknown token \n");
}
if (j == i) {
auto sub = word.substr(i, 1);
if (vocab.token_to_id.find(sub) != vocab.token_to_id.end()) {
tokens.push_back(vocab.token_to_id.at(sub));
} else {
fprintf(stderr, "%s: unknown token '%s'\n", __func__, sub.data());
}
++i; ++i;
} }
} }