extern/whisper.cpp
1
0
Fork 1
mirror of https://github.com/ggerganov/whisper.cpp.git synced 2025-08-16 02:28:16 +02:00
Code Issues Packages Projects Releases Wiki Activity

talk-llama : sync llama.cpp (#2709)

Browse Source
This commit is contained in:
Georgi Gerganov
2025-01-13 08:55:48 +02:00
committed by GitHub
parent 45d3faf961
commit 35d0e02c72
36 changed files with 12960 additions and 11341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
examples/talk-llama/llama-vocab.h
View File

@ -1,6 +1,6 @@
#pragma once
#include "llama-impl.h"
#include "llama.h"
#include <string>
#include <vector>
@ -8,6 +8,18 @@
#include <map>
#include <set>
static const char * llama_model_vocab_type_name(enum llama_vocab_type type){
switch (type) {
case LLAMA_VOCAB_TYPE_NONE: return "no vocab";
case LLAMA_VOCAB_TYPE_SPM: return "SPM";
case LLAMA_VOCAB_TYPE_BPE: return "BPE";
case LLAMA_VOCAB_TYPE_WPM: return "WPM";
case LLAMA_VOCAB_TYPE_UGM: return "UGM";
case LLAMA_VOCAB_TYPE_RWKV: return "RWKV";
default: return "unknown";
}
}
struct llm_tokenizer;
struct llama_vocab {
@ -45,7 +57,7 @@ struct llama_vocab {
id special_unk_id = 0;
id special_sep_id = LLAMA_TOKEN_NULL;
id special_pad_id = LLAMA_TOKEN_NULL;
id special_cls_id = LLAMA_TOKEN_NULL;
id special_cls_id = LLAMA_TOKEN_NULL; // TODO: revisit if this is really needed https://github.com/ggerganov/llama.cpp/pull/10930
id special_mask_id = LLAMA_TOKEN_NULL;
id linefeed_id = 13;