holm/whisper.cpp
1
1
Fork 0
forked from extern/whisper.cpp
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #58 from r0y6a3n0/master

Browse Source 
fix decode missing token issue
This commit is contained in:
Georgi Gerganov 2022-10-17 18:06:02 +03:00 committed by GitHub
commit f24d940ca9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
convert-pt-to-ggml.py
View File

@ -271,7 +271,7 @@ byte_decoder = {v:k for k, v in byte_encoder.items()}
fout.write(struct.pack("i", len(tokens))) fout.write(struct.pack("i", len(tokens)))
for key in tokens: for key in tokens:
text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8') text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text))) fout.write(struct.pack("i", len(text)))
fout.write(text) fout.write(text)