mirror of
https://github.com/ggerganov/whisper.cpp.git
synced 2024-11-07 08:34:37 +01:00
talk-llama : sync llama.cpp
This commit is contained in:
Georgi Gerganov
parent
d9efb664ac
commit
c65d0fd3c8
@ -63,6 +63,30 @@ static void llama_log_softmax(float * array, size_t size) {
|
||||
}
|
||||
*/
|
||||
|
||||
static void llama_sampler_temp_impl(llama_token_data_array * cur_p, float temp) {
|
||||
if (temp <= 0.0f) {
|
||||
// find the token with the highest logit and set the rest to -inf
|
||||
size_t max_i = 0;
|
||||
float max_l = cur_p->data[0].logit;
|
||||
|
||||
for (size_t i = 1; i < cur_p->size; ++i) {
|
||||
if (cur_p->data[i ].logit > max_l) {
|
||||
cur_p->data[max_i].logit = -INFINITY;
|
||||
max_i = i;
|
||||
max_l = cur_p->data[i].logit;
|
||||
} else {
|
||||
cur_p->data[i].logit = -INFINITY;
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
cur_p->data[i].logit /= temp;
|
||||
}
|
||||
}
|
||||
|
||||
static void llama_sampler_softmax_impl(llama_token_data_array * cur_p) {
|
||||
GGML_ASSERT(cur_p->size > 0);
|
||||
|
||||
@ -427,6 +451,9 @@ static const char * llama_sampler_dist_name(const struct llama_sampler * /*smpl*
|
||||
|
||||
static void llama_sampler_dist_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
|
||||
auto * ctx = (llama_sampler_dist *) smpl->ctx;
|
||||
|
||||
llama_sampler_softmax_impl(cur_p);
|
||||
|
||||
cur_p->selected = llama_sample_dist(cur_p, ctx->rng);
|
||||
}
|
||||
|
||||
@ -912,9 +939,8 @@ static const char * llama_sampler_temp_name(const struct llama_sampler * /*smpl*
|
||||
|
||||
static void llama_sampler_temp_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
|
||||
const auto * ctx = (llama_sampler_temp *) smpl->ctx;
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
cur_p->data[i].logit /= ctx->temp;
|
||||
}
|
||||
|
||||
llama_sampler_temp_impl(cur_p, ctx->temp);
|
||||
}
|
||||
|
||||
static struct llama_sampler * llama_sampler_temp_clone(const struct llama_sampler * smpl) {
|
||||
@ -961,6 +987,7 @@ static void llama_sampler_temp_ext_apply(struct llama_sampler * smpl, llama_toke
|
||||
if (ctx->delta > 0) {
|
||||
const float min_temp = std::max(0.0f, ctx->temp - ctx->delta);
|
||||
const float max_temp = ctx->temp + ctx->delta;
|
||||
|
||||
float exponent_val = ctx->exponent;
|
||||
|
||||
// no need to do anything if there is only one (or zero) candidates
|
||||
@ -998,9 +1025,7 @@ static void llama_sampler_temp_ext_apply(struct llama_sampler * smpl, llama_toke
|
||||
#endif
|
||||
|
||||
// Apply the dynamically calculated temperature scaling
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
cur_p->data[i].logit /= dyn_temp;
|
||||
}
|
||||
llama_sampler_temp_impl(cur_p, dyn_temp);
|
||||
|
||||
// Re-compute softmax probabilities after scaling logits with dynamic temperature
|
||||
const double max_l_double = cur_p->data[0].logit;
|
||||
@ -1024,9 +1049,7 @@ static void llama_sampler_temp_ext_apply(struct llama_sampler * smpl, llama_toke
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
cur_p->data[i].logit /= ctx->temp;
|
||||
}
|
||||
llama_sampler_temp_impl(cur_p, ctx->temp);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1059,6 +1082,101 @@ struct llama_sampler * llama_sampler_init_temp_ext(float temp, float delta, floa
|
||||
};
|
||||
}
|
||||
|
||||
// xtc
|
||||
|
||||
struct llama_sampler_xtc {
|
||||
const float probability;
|
||||
const float threshold;
|
||||
const size_t min_keep;
|
||||
|
||||
const uint32_t seed;
|
||||
uint32_t seed_cur;
|
||||
|
||||
std::mt19937 rng;
|
||||
};
|
||||
|
||||
static const char * llama_sampler_xtc_name(const struct llama_sampler * /*smpl*/) {
|
||||
return "xtc";
|
||||
}
|
||||
|
||||
static void llama_sample_xtc_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
|
||||
auto * ctx = (llama_sampler_xtc *) smpl->ctx;
|
||||
|
||||
if (ctx->probability <= 0.0f
|
||||
|| ctx->threshold > 0.5f
|
||||
|| cur_p->size < 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::uniform_real_distribution<float> distribution(0.0f, 1.0f);
|
||||
float chance = distribution(ctx->rng);
|
||||
if (chance > ctx->probability) return;
|
||||
|
||||
// in case it's not sorted/recalculated yet
|
||||
llama_sampler_softmax_impl(cur_p);
|
||||
|
||||
int pos_last = 0;
|
||||
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
if (cur_p->data[i].p >= ctx->threshold) {
|
||||
pos_last = i;
|
||||
} else break;
|
||||
}
|
||||
|
||||
if (cur_p->size - pos_last >= ctx->min_keep && pos_last > 0) {
|
||||
cur_p->data += pos_last;
|
||||
cur_p->size -= pos_last;
|
||||
}
|
||||
}
|
||||
|
||||
static struct llama_sampler * llama_sampler_xtc_clone(const struct llama_sampler * smpl) {
|
||||
const auto * ctx = (const llama_sampler_xtc *) smpl->ctx;
|
||||
auto * result = llama_sampler_init_xtc(ctx->probability, ctx->threshold, ctx->min_keep, ctx->seed);
|
||||
|
||||
// copy the state
|
||||
{
|
||||
auto * result_ctx = (llama_sampler_xtc *) result->ctx;
|
||||
|
||||
result_ctx->rng = ctx->rng;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static void llama_sampler_xtc_free(struct llama_sampler * smpl) {
|
||||
delete (llama_sampler_xtc *) smpl->ctx;
|
||||
}
|
||||
|
||||
static void llama_sampler_xtc_reset(struct llama_sampler * smpl) {
|
||||
auto * ctx = (llama_sampler_xtc *) smpl->ctx;
|
||||
ctx->seed_cur = get_rng_seed(ctx->seed);
|
||||
ctx->rng.seed(ctx->seed_cur);
|
||||
}
|
||||
|
||||
static struct llama_sampler_i llama_sampler_xtc_i = {
|
||||
/* .name = */ llama_sampler_xtc_name,
|
||||
/* .accept = */ nullptr,
|
||||
/* .apply = */ llama_sample_xtc_apply,
|
||||
/* .reset = */ llama_sampler_xtc_reset,
|
||||
/* .clone = */ llama_sampler_xtc_clone,
|
||||
/* .free = */ llama_sampler_xtc_free,
|
||||
};
|
||||
|
||||
struct llama_sampler * llama_sampler_init_xtc(float p, float t, size_t min_keep, uint32_t seed) {
|
||||
auto seed_cur = get_rng_seed(seed);
|
||||
return new llama_sampler {
|
||||
/* .iface = */ &llama_sampler_xtc_i,
|
||||
/* .ctx = */ new llama_sampler_xtc {
|
||||
/* .probability = */ p,
|
||||
/* .threshold = */ t,
|
||||
/* .min_keep = */ min_keep,
|
||||
/* .seed = */ seed,
|
||||
/* .seed_cur = */ seed_cur,
|
||||
/* .rng = */ std::mt19937(seed_cur),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// mirostat
|
||||
|
||||
struct llama_sampler_mirostat {
|
||||
@ -1565,6 +1683,397 @@ struct llama_sampler * llama_sampler_init_penalties(
|
||||
};
|
||||
}
|
||||
|
||||
// DRY
|
||||
|
||||
struct llama_sampler_dry {
|
||||
int32_t total_context_size;
|
||||
|
||||
const float dry_multiplier;
|
||||
const float dry_base;
|
||||
const int32_t dry_allowed_length;
|
||||
const int32_t dry_penalty_last_n;
|
||||
|
||||
std::unordered_multimap<llama_token, std::vector<llama_token>> dry_processed_breakers;
|
||||
std::vector<int> dry_repeat_count;
|
||||
std::unordered_map<llama_token, int> dry_max_token_repeat;
|
||||
ring_buffer<llama_token> last_tokens;
|
||||
};
|
||||
|
||||
// Ported from Koboldcpp, original PR: https://github.com/LostRuins/koboldcpp/pull/982 (Original author: pi6am)
|
||||
static void get_overlapping_token_sequences(const llama_vocab & vocab, const std::string& str, std::unordered_multimap<llama_token, std::vector<llama_token>>& token_sequences, int max_tail_len = -1) {
|
||||
for (llama_token token_id = 0; token_id < (llama_token)vocab.n_vocab; token_id++) {
|
||||
std::string word = llama_detokenize(vocab, {token_id}, true);
|
||||
if (word.find(str) != std::string::npos) {
|
||||
token_sequences.emplace(token_id, std::vector<llama_token>());
|
||||
} else {
|
||||
size_t word_len = word.size(), str_len = str.size();
|
||||
size_t pos = -1;
|
||||
while ((pos = word.find(str[0], pos + 1)) != std::string::npos) {
|
||||
bool match = true;
|
||||
size_t i;
|
||||
for (i = 1; i < str_len && i + pos < word_len; ++i) {
|
||||
if (word[pos + i] != str[i]) {
|
||||
match = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match) {
|
||||
std::vector<llama_token> tokenization = llama_tokenize_internal(vocab, str.substr(i), false, false);
|
||||
if (max_tail_len >= 0 && tokenization.size() > (size_t)max_tail_len) {
|
||||
tokenization.resize(max_tail_len);
|
||||
}
|
||||
|
||||
// Ensure we don't already have a duplicate matching tokenization
|
||||
auto its = token_sequences.equal_range(token_id);
|
||||
bool found = false;
|
||||
for (auto it = its.first; it != its.second; ++it) {
|
||||
if (tokenization == it->second) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
token_sequences.emplace(token_id, tokenization);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static const char * llama_sampler_dry_name(const struct llama_sampler * /*smpl*/) {
|
||||
return "dry";
|
||||
}
|
||||
|
||||
static void llama_sampler_dry_accept(struct llama_sampler * smpl, llama_token token) {
|
||||
auto * ctx = (llama_sampler_dry *) smpl->ctx;
|
||||
if (ctx->dry_multiplier == 0.0f || ctx->dry_base < 1.0f || ctx->dry_penalty_last_n == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
ctx->last_tokens.push_back(token);
|
||||
}
|
||||
|
||||
// Ported from Koboldcpp, original PR: https://github.com/LostRuins/koboldcpp/pull/982 (Original author: pi6am)
|
||||
static void llama_sampler_dry_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
|
||||
auto * ctx = (llama_sampler_dry *) smpl->ctx;
|
||||
|
||||
if (ctx->dry_multiplier == 0.0f || ctx->dry_base < 1.0f || ctx->dry_penalty_last_n == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int32_t effective_dry_penalty_last_n = (ctx->dry_penalty_last_n == -1) ? ctx->total_context_size : std::max(ctx->dry_penalty_last_n, 0);
|
||||
int last_n_repeat = std::min(std::min((int)ctx->last_tokens.size(), effective_dry_penalty_last_n), ctx->total_context_size);
|
||||
|
||||
if (last_n_repeat <= ctx->dry_allowed_length) {
|
||||
return;
|
||||
}
|
||||
|
||||
ctx->dry_repeat_count.assign(last_n_repeat, 0);
|
||||
ctx->dry_max_token_repeat.clear();
|
||||
|
||||
// Step 1: Look for restart sequences to limit the maximum repetition length.
|
||||
// Work backwards through the context looking for any token that begins a restart sequence.
|
||||
//
|
||||
// The collection `restart_sequences` is a mapping from a "head" token to all "tail"
|
||||
// sequences that together comprise a restart sequence. This allows us to quickly check
|
||||
// whether each token is the head of a complete sequence. Most restart sequences are actually
|
||||
// a single token, and for these the "tail" is an empty vector.
|
||||
//
|
||||
// If the token is a "head", test all restart sequences that begin with this token
|
||||
// (there will often only be one sequence for each token, but if sequences like 'aaaq1' and
|
||||
// 'aaa1' are used as restart strings, both could start with 'aaa' when tokenized). The
|
||||
// longest matching sequence (if any) is used to limit the maximum repetition length.
|
||||
//
|
||||
// Note that in the case case of a short sequence contained in a longer one, this might fail to
|
||||
// find the smallest value for `rep_limit`. For example, if 'amniotic' and 'ni' are both used as
|
||||
// restart sequences, 'ni' will be found first, and since it's shorter it will fail to suppress
|
||||
// 'otic'. This is a minor issue since fully contained restart sequences are likely to be rare.
|
||||
//
|
||||
// This is theoretically worst-case O(N^2) for arbitrary restart sequences, which is why we
|
||||
// have already clamped the maximum tail sequence length when generating `restart_sequences`.
|
||||
// With clamping, this scan is O(N) in the context length.
|
||||
|
||||
int rep_limit = last_n_repeat;
|
||||
for (int i = 0; i < last_n_repeat; ++i) {
|
||||
llama_token token = ctx->last_tokens.rat(i);
|
||||
auto its = ctx->dry_processed_breakers.equal_range(token);
|
||||
if (its.first == ctx->dry_processed_breakers.end()) {
|
||||
continue;
|
||||
}
|
||||
int longest_match = -1;
|
||||
for (auto it = its.first; it != its.second; ++it) {
|
||||
// Note that (*it) does not contain the head character, so seq_len will be
|
||||
// the restart sequence length minus 1.
|
||||
// In the common case of a single-token restart sequence, (*it) will be empty
|
||||
// and we will trivially match.
|
||||
int seq_len = (int)it->second.size();
|
||||
if (seq_len > longest_match && seq_len <= (int)i) {
|
||||
bool match = true;
|
||||
for (int offset = 0; offset < seq_len; ++offset) {
|
||||
// The -1 when indexing `last_tokens` is because we already matched the head.
|
||||
if (it->second[offset] != ctx->last_tokens.rat(i - offset - 1)) {
|
||||
match = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match) {
|
||||
longest_match = seq_len;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (longest_match >= 0) {
|
||||
// We found a restart sequence starting `i` tokens from the end and continuing for
|
||||
// `longest_match` tokens.
|
||||
rep_limit = i - longest_match;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (rep_limit < ctx->dry_allowed_length) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 2: Iterate in reverse over the last N tokens of the context, using the "Z-algorithm" (in
|
||||
// the reverse direction) to efficiently compute the positions and lengths of suffixes appearing
|
||||
// elsewhere in the context. We limit the suffix length to `rep_limit` to respect restart sequences.
|
||||
//
|
||||
// This algorithm is not currently documented on Wikipedia, but there is a clear description here:
|
||||
// https://ivanyu.me/blog/2014/10/15/z-algorithm/
|
||||
//
|
||||
// The code below is adapted from the public domain implementation by the same author here:
|
||||
// https://github.com/ivanyu/string-algorithms/blob/master/z_algorithm.py
|
||||
//
|
||||
// Example:
|
||||
// Last N tokens: a b c c b c y a b c
|
||||
// Repeat counts: 0 0 3 1 0 2 0 0 0 0
|
||||
// ^
|
||||
// This `3` means that the last three tokens of the context (a b c) also appear here.
|
||||
//
|
||||
// This step is worst case O(N) since the Z-algorithm is linear, despite the appearance of nested
|
||||
// for/while loops. This can be seen by observing that the `lt` and `rt` bounds are set after each
|
||||
// repeated suffix is detected (i.e. after each while loop when n > 0). These bound variables
|
||||
// ensure that the inner while loops only examine each token in the context once as the outer
|
||||
// for loop iterates over the context.
|
||||
|
||||
{
|
||||
const int last = last_n_repeat - 1;
|
||||
int rt = 0, lt = 0;
|
||||
|
||||
for (int k = 1; k < last_n_repeat; ++k) {
|
||||
if (k > rt) {
|
||||
// If k is outside the current Z-box, do naive computation.
|
||||
int n = 0;
|
||||
while (n + k < last_n_repeat && ctx->last_tokens.rat(n) == ctx->last_tokens.rat(n+k)) {
|
||||
++n;
|
||||
}
|
||||
ctx->dry_repeat_count[last - k] = std::min(n, rep_limit);
|
||||
if (n > 0) {
|
||||
lt = k;
|
||||
rt = k+n-1;
|
||||
}
|
||||
} else {
|
||||
// If k is inside the current Z-box, consider two cases.
|
||||
|
||||
int p = k - lt; // Pair index.
|
||||
int right_part_len = rt - k + 1;
|
||||
|
||||
if (ctx->dry_repeat_count[last - p] < right_part_len) {
|
||||
int n = std::min(ctx->dry_repeat_count[last - p], rep_limit);
|
||||
ctx->dry_repeat_count[last - k] = n;
|
||||
} else {
|
||||
int i = rt + 1;
|
||||
while (i < last_n_repeat && ctx->last_tokens.rat(i) == ctx->last_tokens.rat(i - k)) {
|
||||
i += 1;
|
||||
}
|
||||
|
||||
int n = std::min(i - k, rep_limit);
|
||||
ctx->dry_repeat_count[last - k] = n;
|
||||
lt = k;
|
||||
rt = i - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Iterate over dry_repeat_count and last_tokens, examining the maximum repeat length
|
||||
// that would be generated by emitting each new token that would extend a sequence.
|
||||
//
|
||||
// Following the same example as above:
|
||||
// Last N tokens: a b c c b c y a b c
|
||||
// Repeat counts: 0 0 3 1 0 2 0 0 0 0
|
||||
//
|
||||
// For each non-zero, look ahead one token. This token, if emitted, would extend the repetition.
|
||||
// c: 3 -> 4 (from `a b c` to `a b c c`)
|
||||
// b: 1 -> 2 (from `c` to `c b`)
|
||||
// y: 2 -> 3 (from `b c` to `b c y`)
|
||||
|
||||
for (int i = 0; i < last_n_repeat - 1; ++i) {
|
||||
int repeat_len = ctx->dry_repeat_count[i];
|
||||
if (repeat_len >= ctx->dry_allowed_length) {
|
||||
// This token ends a repeat, so the next token would continue one.
|
||||
// By convention, the value of `repeat_len` only includes the tokens currently
|
||||
// in the context, not the new token that would be added.
|
||||
llama_token token = ctx->last_tokens.rat(last_n_repeat - 2 - i);
|
||||
// Track the maximum sequence ending in this token.
|
||||
const auto& it = ctx->dry_max_token_repeat.find(token);
|
||||
if (it == ctx->dry_max_token_repeat.end() || it->second < repeat_len) {
|
||||
ctx->dry_max_token_repeat[token] = repeat_len;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Apply logit penalties based on the maximum repeat length for relevant tokens.
|
||||
|
||||
// Prevent floating point overflow in `pow(penalty_base, exponent)` by clamping to `max_exponent`.
|
||||
// Compute it from `penalty_base` and the approximate log of `std::numeric_limits<float>::max()`
|
||||
const float FLOAT_MAX_LOG = 88.7228391f;
|
||||
int max_exponent = 0;
|
||||
if (ctx->dry_base > 1.000001f) {
|
||||
max_exponent = FLOAT_MAX_LOG / std::log(ctx->dry_base);
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
const auto& af_kvp = ctx->dry_max_token_repeat.find(cur_p->data[i].id);
|
||||
if (af_kvp != ctx->dry_max_token_repeat.end()) {
|
||||
// Check all sequence breakers starting with this token
|
||||
auto range = ctx->dry_processed_breakers.equal_range(cur_p->data[i].id);
|
||||
bool is_single_token_breaker = false;
|
||||
|
||||
for (auto it = range.first; it != range.second; ++it) {
|
||||
if (it->second.empty()) {
|
||||
is_single_token_breaker = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply penalty only if it's not a single-token sequence breaker
|
||||
if (!is_single_token_breaker) {
|
||||
int repeat_exp = af_kvp->second - ctx->dry_allowed_length;
|
||||
if (max_exponent > 0 && repeat_exp > max_exponent) {
|
||||
repeat_exp = max_exponent;
|
||||
}
|
||||
float penalty = ctx->dry_multiplier * std::pow(ctx->dry_base, repeat_exp);
|
||||
cur_p->data[i].logit -= penalty;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cur_p->sorted = false;
|
||||
}
|
||||
|
||||
static void llama_sampler_dry_reset(struct llama_sampler * smpl) {
|
||||
auto * ctx = (llama_sampler_dry *) smpl->ctx;
|
||||
ctx->last_tokens.clear();
|
||||
ctx->dry_repeat_count.clear();
|
||||
ctx->dry_max_token_repeat.clear();
|
||||
}
|
||||
|
||||
static struct llama_sampler * llama_sampler_dry_clone(const struct llama_sampler * smpl) {
|
||||
const auto * ctx = (llama_sampler_dry *) smpl->ctx;
|
||||
|
||||
// nullptr is passed as vocab because it is only needed for raw sequence breaker processing, which we have already done and will be copying
|
||||
auto * result = llama_sampler_init_dry(nullptr, ctx->dry_multiplier, ctx->dry_base, ctx->dry_allowed_length, ctx->dry_penalty_last_n, NULL, 0);
|
||||
// Copy the state, including the processed breakers
|
||||
{
|
||||
auto * result_ctx = (llama_sampler_dry *) result->ctx;
|
||||
result_ctx->dry_processed_breakers = ctx->dry_processed_breakers;
|
||||
result_ctx->dry_repeat_count = ctx->dry_repeat_count;
|
||||
result_ctx->dry_max_token_repeat = ctx->dry_max_token_repeat;
|
||||
result_ctx->last_tokens = ctx->last_tokens;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static void llama_sampler_dry_free(struct llama_sampler * smpl) {
|
||||
delete (llama_sampler_dry *) smpl->ctx;
|
||||
}
|
||||
|
||||
static struct llama_sampler_i llama_sampler_dry_i = {
|
||||
/* .name = */ llama_sampler_dry_name,
|
||||
/* .accept = */ llama_sampler_dry_accept,
|
||||
/* .apply = */ llama_sampler_dry_apply,
|
||||
/* .reset = */ llama_sampler_dry_reset,
|
||||
/* .clone = */ llama_sampler_dry_clone,
|
||||
/* .free = */ llama_sampler_dry_free,
|
||||
};
|
||||
|
||||
struct llama_sampler * llama_sampler_init_dry_impl(const struct llama_vocab & vocab, int32_t context_size, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const char** seq_breakers, size_t num_breakers) {
|
||||
int32_t effective_dry_penalty_last_n = (dry_penalty_last_n == -1) ? context_size : std::max(dry_penalty_last_n, 0);
|
||||
std::unordered_multimap<llama_token, std::vector<llama_token>> processed_breakers;
|
||||
const int MAX_CHAR_LEN = 40;
|
||||
const int MAX_SEQ_LEN = 20;
|
||||
|
||||
const bool dry_enabled = (dry_multiplier != 0.0f && dry_base >= 1.0f && dry_penalty_last_n != 0);
|
||||
|
||||
if (dry_enabled && seq_breakers != nullptr && num_breakers > 0) {
|
||||
// Process sequence breakers
|
||||
for (size_t i = 0; i < num_breakers; ++i) {
|
||||
if (seq_breakers[i] == nullptr || std::strlen(seq_breakers[i]) == 0) {
|
||||
LLAMA_LOG_WARN("skipping null or empty DRY sequence breaker at index %zu\n", i);
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string sequence_break(seq_breakers[i]);
|
||||
if (sequence_break.empty()) {
|
||||
LLAMA_LOG_WARN("skipping empty DRY sequence breaker\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (sequence_break.size() > MAX_CHAR_LEN) {
|
||||
LLAMA_LOG_WARN("truncating DRY sequence breaker to %d characters\n", MAX_CHAR_LEN);
|
||||
sequence_break.resize(MAX_CHAR_LEN);
|
||||
}
|
||||
|
||||
get_overlapping_token_sequences(vocab, sequence_break, processed_breakers, MAX_SEQ_LEN);
|
||||
}
|
||||
}
|
||||
|
||||
return new llama_sampler {
|
||||
/* .iface = */ &llama_sampler_dry_i,
|
||||
/* .ctx = */ new llama_sampler_dry {
|
||||
/* .total_context_size = */ context_size,
|
||||
/* .dry_multiplier = */ dry_multiplier,
|
||||
/* .dry_base = */ dry_base,
|
||||
/* .dry_allowed_length = */ dry_allowed_length,
|
||||
/* .dry_penalty_last_n = */ dry_penalty_last_n,
|
||||
/* .dry_processed_breakers = */ std::move(processed_breakers),
|
||||
/* .dry_repeat_count = */ dry_enabled ? std::vector<int>(effective_dry_penalty_last_n, 0) : std::vector<int>{},
|
||||
/* .dry_max_token_repeat = */ {},
|
||||
/* .last_tokens = */ dry_enabled ? ring_buffer<llama_token>(effective_dry_penalty_last_n) : ring_buffer<llama_token>(0),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// wrapper for test-sampling.cpp
|
||||
struct llama_sampler * llama_sampler_init_dry_testing(int32_t context_size, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector<std::vector<llama_token>>& seq_breakers) {
|
||||
llama_vocab dummy_vocab;
|
||||
auto * result = llama_sampler_init_dry_impl(dummy_vocab, context_size, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, NULL, 0);
|
||||
auto * ctx = (llama_sampler_dry *) result->ctx;
|
||||
|
||||
// Process the token-based sequence breakers
|
||||
ctx->dry_processed_breakers.clear();
|
||||
if (seq_breakers.empty()) {
|
||||
LLAMA_LOG_WARN("empty DRY sequence breakers list in llama_sampler_init_dry_testing\n");
|
||||
} else {
|
||||
for (const auto& breaker : seq_breakers) {
|
||||
if (breaker.empty()) {
|
||||
LLAMA_LOG_WARN("skipping DRY empty sequence breaker\n");
|
||||
continue;
|
||||
}
|
||||
llama_token head_token = breaker[0];
|
||||
std::vector<llama_token> tail_tokens(breaker.begin() + 1, breaker.end());
|
||||
ctx->dry_processed_breakers.emplace(head_token, std::move(tail_tokens));
|
||||
}
|
||||
|
||||
if (ctx->dry_processed_breakers.empty()) {
|
||||
LLAMA_LOG_WARN("no valid DRY sequence breakers processed in llama_sampler_init_dry_testing\n");
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// logit-bias
|
||||
|
||||
struct llama_sampler_logit_bias {
|
||||
@ -1644,6 +2153,229 @@ struct llama_sampler * llama_sampler_init_logit_bias(
|
||||
};
|
||||
}
|
||||
|
||||
// infill
|
||||
|
||||
//#define GGML_DEBUG_SAMPLER_INFILL
|
||||
|
||||
struct llama_sampler_infill {
|
||||
const struct llama_vocab * vocab;
|
||||
|
||||
std::vector<char> buf0;
|
||||
std::vector<char> buf1;
|
||||
};
|
||||
|
||||
static const char * llama_sampler_infill_name(const struct llama_sampler * /*smpl*/) {
|
||||
return "infill";
|
||||
}
|
||||
|
||||
static void llama_sampler_infill_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
|
||||
auto * ctx = (llama_sampler_infill *) smpl->ctx;
|
||||
|
||||
llama_sampler_softmax_impl(cur_p);
|
||||
|
||||
#if defined(GGML_DEBUG_SAMPLER_INFILL)
|
||||
#define LOG_DBG_CUR LLAMA_LOG_DEBUG
|
||||
#else
|
||||
#define LOG_DBG_CUR(...)
|
||||
#endif
|
||||
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
LOG_DBG_CUR("%s: cur_p[%3zu] = { id: %6d, p: %.6f, logit: %6.3f }\n", __func__, i, cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
|
||||
}
|
||||
|
||||
float p_txt_sum = 0.0f;
|
||||
float p_eog_sum = 0.0f;
|
||||
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
if (llama_token_is_eog_impl(*ctx->vocab, cur_p->data[i].id)) {
|
||||
p_eog_sum += cur_p->data[i].p;
|
||||
} else {
|
||||
p_txt_sum += cur_p->data[i].p;
|
||||
}
|
||||
}
|
||||
|
||||
const float rat = p_eog_sum == 0.0 ? INFINITY : p_txt_sum / p_eog_sum; GGML_UNUSED(rat);
|
||||
|
||||
LOG_DBG_CUR("%s: p_txt_sum = %.2f, p_eog_sum = %.2f, rat = %.2f, n = %zu\n", __func__, p_txt_sum, p_eog_sum, rat, cur_p->size);
|
||||
|
||||
if (3*p_eog_sum*cur_p->size > p_txt_sum) {
|
||||
LOG_DBG_CUR("%s: the ratio p_txt/p_eog = %.2f is too low -> sampling EOG\n", __func__, p_txt_sum/p_eog_sum);
|
||||
|
||||
// keep just the EOG tokens
|
||||
const auto size_org = cur_p->size;
|
||||
|
||||
cur_p->size = 0;
|
||||
|
||||
float p_sum = 0.0f;
|
||||
|
||||
for (size_t i = 0; i < size_org; ++i) {
|
||||
if (llama_token_is_eog_impl(*ctx->vocab, cur_p->data[i].id)) {
|
||||
p_sum += cur_p->data[i].p;
|
||||
|
||||
cur_p->data[cur_p->size++] = cur_p->data[i];
|
||||
}
|
||||
}
|
||||
|
||||
// normalize probs
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
cur_p->data[i].p /= p_sum;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
size_t n_combined = 0; GGML_UNUSED(n_combined);
|
||||
|
||||
// combine tokens with common prefix
|
||||
for (size_t i0 = 0; i0 < cur_p->size; ++i0) {
|
||||
for (size_t i1 = 0; i1 < cur_p->size; ++i1) {
|
||||
if (cur_p->data[i0].logit == -INFINITY) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (i0 == i1 || cur_p->data[i1].logit == -INFINITY) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int len0 = llama_token_to_piece_impl(*ctx->vocab, cur_p->data[i0].id, ctx->buf0.data(), ctx->buf0.size(), 0, false);
|
||||
if (len0 < 0) {
|
||||
ctx->buf0.resize(len0);
|
||||
len0 = llama_token_to_piece_impl(*ctx->vocab, cur_p->data[i0].id, ctx->buf0.data(), ctx->buf0.size(), 0, false);
|
||||
assert(len0 > 0);
|
||||
}
|
||||
|
||||
int len1 = llama_token_to_piece_impl(*ctx->vocab, cur_p->data[i1].id, ctx->buf1.data(), ctx->buf1.size(), 0, false);
|
||||
if (len1 < 0) {
|
||||
ctx->buf1.resize(len1);
|
||||
len1 = llama_token_to_piece_impl(*ctx->vocab, cur_p->data[i1].id, ctx->buf1.data(), ctx->buf1.size(), 0, false);
|
||||
assert(len1 > 0);
|
||||
}
|
||||
|
||||
// token i0 is a prefix of token i1
|
||||
if (len0 > 0 && len0 <= len1 && memcmp(ctx->buf0.data(), ctx->buf1.data(), len0) == 0) {
|
||||
int dst = i0;
|
||||
int src = i1;
|
||||
|
||||
// merge into the token with higher probability
|
||||
if (cur_p->data[i1].p > cur_p->data[i0].p) {
|
||||
std::swap(dst, src);
|
||||
}
|
||||
|
||||
cur_p->data[dst].p += cur_p->data[src].p;
|
||||
cur_p->data[src].logit = -INFINITY;
|
||||
cur_p->data[src].p = 0.0f;
|
||||
|
||||
n_combined++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t n_non_eog = 0;
|
||||
|
||||
size_t size_org = cur_p->size;
|
||||
|
||||
float p_sum = 0.0f;
|
||||
float thold = 0.2f;
|
||||
|
||||
cur_p->size = 0;
|
||||
|
||||
LOG_DBG_CUR("%s: n_combined = %zu, applying thold = %.3f\n", __func__, n_combined, thold);
|
||||
|
||||
for (size_t i = 0; i < size_org; ++i) {
|
||||
const bool is_eog = llama_token_is_eog_impl(*ctx->vocab, cur_p->data[i].id);
|
||||
|
||||
if (cur_p->data[i].p < thold && !is_eog) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!is_eog) {
|
||||
++n_non_eog;
|
||||
}
|
||||
|
||||
p_sum += cur_p->data[i].p;
|
||||
|
||||
// keep this token
|
||||
cur_p->data[cur_p->size++] = cur_p->data[i];
|
||||
}
|
||||
|
||||
LOG_DBG_CUR("%s: n_non_eog = %zu\n", __func__, n_non_eog);
|
||||
|
||||
// if no non-EOG tokens are left -> reduce cur_p to single EOT token
|
||||
if (n_non_eog == 0) {
|
||||
cur_p->size = 1;
|
||||
cur_p->data[0].id = llama_token_eot_impl(*ctx->vocab);
|
||||
cur_p->data[0].logit = 1.0f;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// normalize probs
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
cur_p->data[i].p /= p_sum;
|
||||
|
||||
LOG_DBG_CUR("%s: cur_p[%3zu] = { id: %6d, p: %.6f, logit: %6.3f }\n", __func__, i, cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
|
||||
}
|
||||
|
||||
size_org = cur_p->size;
|
||||
p_sum = 0.0f;
|
||||
thold = 1.0/(n_non_eog + 1);
|
||||
|
||||
cur_p->size = 0;
|
||||
|
||||
LOG_DBG_CUR("%s: applying thold = %.3f\n", __func__, thold);
|
||||
|
||||
for (size_t i = 0; i < size_org; ++i) {
|
||||
const bool is_eog = llama_token_is_eog_impl(*ctx->vocab, cur_p->data[i].id);
|
||||
|
||||
if (cur_p->data[i].p < thold && !is_eog) {
|
||||
continue;
|
||||
}
|
||||
|
||||
p_sum += cur_p->data[i].p;
|
||||
|
||||
cur_p->data[cur_p->size++] = cur_p->data[i];
|
||||
}
|
||||
|
||||
// normalize probs
|
||||
for (size_t i = 0; i < cur_p->size; ++i) {
|
||||
cur_p->data[i].p /= p_sum;
|
||||
|
||||
LOG_DBG_CUR("%s: cur_p[%3zu] = { id: %6d, p: %.6f, logit: %6.3f }\n", __func__, i, cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
|
||||
}
|
||||
|
||||
#undef LOG_DBG_CUR
|
||||
}
|
||||
|
||||
static struct llama_sampler * llama_sampler_infill_clone(const struct llama_sampler * smpl) {
|
||||
const auto * ctx = (const llama_sampler_infill *) smpl->ctx;
|
||||
return llama_sampler_init_infill_impl(*ctx->vocab);
|
||||
}
|
||||
|
||||
static void llama_sampler_infill_free(struct llama_sampler * smpl) {
|
||||
delete (llama_sampler_infill *) smpl->ctx;
|
||||
}
|
||||
|
||||
static struct llama_sampler_i llama_sampler_infill_i = {
|
||||
/* .name = */ llama_sampler_infill_name,
|
||||
/* .accept = */ nullptr,
|
||||
/* .apply = */ llama_sampler_infill_apply,
|
||||
/* .reset = */ nullptr,
|
||||
/* .clone = */ llama_sampler_infill_clone,
|
||||
/* .free = */ llama_sampler_infill_free,
|
||||
};
|
||||
|
||||
struct llama_sampler * llama_sampler_init_infill_impl(
|
||||
const struct llama_vocab & vocab) {
|
||||
return new llama_sampler {
|
||||
/* .iface = */ &llama_sampler_infill_i,
|
||||
/* .ctx = */ new llama_sampler_infill {
|
||||
/* .vocab = */ &vocab,
|
||||
/* .buf0 = */ std::vector<char>(512),
|
||||
/* .buf1 = */ std::vector<char>(512),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// utils
|
||||
|
||||
uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl) {
|
||||
|
||||
@ -4,8 +4,6 @@
|
||||
|
||||
#include "llama-grammar.h"
|
||||
|
||||
#include <unordered_map>
|
||||
|
||||
struct llama_vocab;
|
||||
struct llama_grammar;
|
||||
|
||||
@ -27,3 +25,24 @@ struct llama_sampler * llama_sampler_init_grammar_impl(
|
||||
const struct llama_vocab & vocab,
|
||||
const char * grammar_str,
|
||||
const char * grammar_root);
|
||||
|
||||
struct llama_sampler * llama_sampler_init_infill_impl(
|
||||
const struct llama_vocab & vocab);
|
||||
|
||||
struct llama_sampler * llama_sampler_init_dry_impl(
|
||||
const struct llama_vocab & vocab,
|
||||
int32_t context_size,
|
||||
float dry_multiplier,
|
||||
float dry_base,
|
||||
int32_t dry_allowed_length,
|
||||
int32_t dry_penalty_last_n,
|
||||
const char ** seq_breakers,
|
||||
size_t num_breakers);
|
||||
|
||||
struct llama_sampler * llama_sampler_init_dry_testing(
|
||||
int32_t context_size,
|
||||
float dry_multiplier,
|
||||
float dry_base,
|
||||
int32_t dry_allowed_length,
|
||||
int32_t dry_penalty_last_n,
|
||||
const std::vector<std::vector<llama_token>>& seq_breakers);
|
||||
|
||||
@ -221,7 +221,7 @@ struct llm_tokenizer_spm_session {
|
||||
}
|
||||
|
||||
// seed the work queue with all possible 2-character tokens.
|
||||
for (size_t i = 1; i < symbols.size(); ++i) {
|
||||
for (int i = 1; i < (int) symbols.size(); ++i) {
|
||||
try_add_bigram(i - 1, i);
|
||||
}
|
||||
|
||||
@ -563,7 +563,7 @@ struct llm_tokenizer_bpe_session {
|
||||
index++;
|
||||
symbols.emplace_back(sym);
|
||||
}
|
||||
for (size_t i = 1; i < symbols.size(); ++i) {
|
||||
for (int i = 1; i < (int) symbols.size(); ++i) {
|
||||
add_new_bigram(i - 1, i);
|
||||
}
|
||||
|
||||
@ -1663,6 +1663,14 @@ llama_token llama_token_eos_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_eos_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_eot_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_eot_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_eom_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_eom_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_cls_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_cls_id;
|
||||
}
|
||||
@ -1688,23 +1696,39 @@ bool llama_add_eos_token_impl(const struct llama_vocab & vocab) {
|
||||
}
|
||||
|
||||
llama_token llama_token_prefix_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_prefix_id;
|
||||
return vocab.special_fim_pre_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_middle_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_middle_id;
|
||||
return vocab.special_fim_mid_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_suffix_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_suffix_id;
|
||||
return vocab.special_fim_suf_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_eot_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_eot_id;
|
||||
llama_token llama_token_fim_pre_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_fim_pre_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_eom_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_eom_id;
|
||||
llama_token llama_token_fim_suf_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_fim_suf_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_fim_mid_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_fim_mid_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_fim_pad_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_fim_pad_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_fim_rep_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_fim_rep_id;
|
||||
}
|
||||
|
||||
llama_token llama_token_fim_sep_impl(const struct llama_vocab & vocab) {
|
||||
return vocab.special_fim_sep_id;
|
||||
}
|
||||
|
||||
int32_t llama_tokenize_impl(
|
||||
@ -1942,3 +1966,19 @@ int32_t llama_detokenize_impl(
|
||||
|
||||
return total <= text_len_max ? total : -total;
|
||||
}
|
||||
|
||||
std::string llama_detokenize(const struct llama_vocab & vocab, const std::vector<llama_token> & tokens, bool special) {
|
||||
std::string text;
|
||||
text.resize(std::max(text.capacity(), tokens.size()));
|
||||
int32_t n_chars = llama_detokenize_impl(vocab, tokens.data(), (int32_t)tokens.size(), &text[0], (int32_t)text.size(), false, special);
|
||||
if (n_chars < 0) {
|
||||
text.resize(-n_chars);
|
||||
n_chars = llama_detokenize_impl(vocab, tokens.data(), (int32_t)tokens.size(), &text[0], (int32_t)text.size(), false, special);
|
||||
GGML_ASSERT(n_chars <= (int32_t)text.size()); // whitespace trimming is performed after per-token detokenization
|
||||
}
|
||||
|
||||
text.resize(n_chars);
|
||||
|
||||
// NOTE: the original tokenizer decodes bytes after collecting the pieces.
|
||||
return text;
|
||||
}
|
||||
|
||||
@ -37,20 +37,26 @@ struct llama_vocab {
|
||||
std::map<std::pair<std::string, std::string>, int> bpe_ranks;
|
||||
|
||||
// default LLaMA special tokens
|
||||
// TODO: should we set all of these to LLAMA_TOKEN_NULL?
|
||||
id special_bos_id = 1;
|
||||
id special_eos_id = 2;
|
||||
id special_eot_id = LLAMA_TOKEN_NULL;
|
||||
id special_eom_id = LLAMA_TOKEN_NULL;
|
||||
id special_unk_id = 0;
|
||||
id special_sep_id = -1;
|
||||
id special_pad_id = -1;
|
||||
id special_cls_id = -1;
|
||||
id special_mask_id = -1;
|
||||
id special_sep_id = LLAMA_TOKEN_NULL;
|
||||
id special_pad_id = LLAMA_TOKEN_NULL;
|
||||
id special_cls_id = LLAMA_TOKEN_NULL;
|
||||
id special_mask_id = LLAMA_TOKEN_NULL;
|
||||
|
||||
id linefeed_id = 13;
|
||||
id special_prefix_id = -1;
|
||||
id special_suffix_id = -1;
|
||||
id special_middle_id = -1;
|
||||
id special_eot_id = -1; // TODO: move above after "eos_id", and here add "file separator" token
|
||||
id special_eom_id = -1;
|
||||
id linefeed_id = 13;
|
||||
|
||||
// fim tokens
|
||||
id special_fim_pre_id = LLAMA_TOKEN_NULL;
|
||||
id special_fim_suf_id = LLAMA_TOKEN_NULL;
|
||||
id special_fim_mid_id = LLAMA_TOKEN_NULL;
|
||||
id special_fim_pad_id = LLAMA_TOKEN_NULL;
|
||||
id special_fim_rep_id = LLAMA_TOKEN_NULL; // repo
|
||||
id special_fim_sep_id = LLAMA_TOKEN_NULL; // file separator
|
||||
|
||||
// set of all tokens that cause "end of generation"
|
||||
std::set<id> special_eog_ids;
|
||||
@ -104,19 +110,26 @@ bool llama_token_is_control_impl(const struct llama_vocab & vocab, llama_token t
|
||||
|
||||
llama_token llama_token_bos_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_eos_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_eot_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_eom_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_cls_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_sep_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_nl_impl (const struct llama_vocab & vocab);
|
||||
llama_token llama_token_pad_impl(const struct llama_vocab & vocab);
|
||||
|
||||
bool llama_add_bos_token_impl(const struct llama_vocab & vocab);
|
||||
bool llama_add_eos_token_impl(const struct llama_vocab & vocab);
|
||||
|
||||
llama_token llama_token_prefix_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_middle_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_suffix_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_eot_impl (const struct llama_vocab & vocab);
|
||||
llama_token llama_token_eom_impl (const struct llama_vocab & vocab);
|
||||
|
||||
llama_token llama_token_fim_pre_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_fim_suf_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_fim_mid_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_fim_pad_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_fim_rep_impl(const struct llama_vocab & vocab);
|
||||
llama_token llama_token_fim_sep_impl(const struct llama_vocab & vocab);
|
||||
|
||||
bool llama_add_bos_token_impl(const struct llama_vocab & vocab);
|
||||
bool llama_add_eos_token_impl(const struct llama_vocab & vocab);
|
||||
|
||||
int32_t llama_tokenize_impl(
|
||||
const struct llama_vocab & vocab,
|
||||
@ -136,6 +149,12 @@ int32_t llama_token_to_piece_impl(
|
||||
int32_t lstrip,
|
||||
bool special);
|
||||
|
||||
// check if token0 is contained as a prefix in token1
|
||||
bool llama_token_is_prefix_impl(
|
||||
const struct llama_vocab & vocab,
|
||||
llama_token token0,
|
||||
llama_token token1);
|
||||
|
||||
int32_t llama_detokenize_impl(
|
||||
const struct llama_vocab & vocab,
|
||||
const llama_token * tokens,
|
||||
@ -144,3 +163,8 @@ int32_t llama_detokenize_impl(
|
||||
int32_t text_len_max,
|
||||
bool remove_special,
|
||||
bool unparse_special);
|
||||
|
||||
std::string llama_detokenize(
|
||||
const struct llama_vocab & vocab,
|
||||
const std::vector<llama_token> & tokens,
|
||||
bool special);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -217,6 +217,7 @@ extern "C" {
|
||||
|
||||
typedef struct llama_token_data_array {
|
||||
// TODO: consider SoA
|
||||
// NOTE: this pointer can be modified by the samplers
|
||||
llama_token_data * data;
|
||||
size_t size;
|
||||
int64_t selected; // this is the index in the data array (i.e. not the token id)
|
||||
@ -232,8 +233,11 @@ extern "C" {
|
||||
// - token : the token ids of the input (used when embd is NULL)
|
||||
// - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
|
||||
// - pos : the positions of the respective token in the sequence
|
||||
// (if set to NULL, the token position will be tracked automatically by llama_decode)
|
||||
// - seq_id : the sequence to which the respective token belongs
|
||||
// (if set to NULL, the sequence ID will be assumed to be 0)
|
||||
// - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output
|
||||
// (if set to NULL, only the logits for last token will be returned)
|
||||
//
|
||||
typedef struct llama_batch {
|
||||
int32_t n_tokens;
|
||||
@ -244,15 +248,6 @@ extern "C" {
|
||||
int32_t * n_seq_id;
|
||||
llama_seq_id ** seq_id;
|
||||
int8_t * logits; // TODO: rename this to "output"
|
||||
|
||||
// NOTE: helpers for smooth API transition - can be deprecated in the future
|
||||
// for future-proof code, use the above fields instead and ignore everything below
|
||||
//
|
||||
// pos[i] = all_pos_0 + i*all_pos_1
|
||||
//
|
||||
llama_pos all_pos_0; // used if pos == NULL
|
||||
llama_pos all_pos_1; // used if pos == NULL
|
||||
llama_seq_id all_seq_id; // used if seq_id == NULL
|
||||
} llama_batch;
|
||||
|
||||
enum llama_model_kv_override_type {
|
||||
@ -433,6 +428,7 @@ extern "C" {
|
||||
LLAMA_API bool llama_supports_mmap (void);
|
||||
LLAMA_API bool llama_supports_mlock (void);
|
||||
LLAMA_API bool llama_supports_gpu_offload(void);
|
||||
LLAMA_API bool llama_supports_rpc (void);
|
||||
|
||||
LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);
|
||||
LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);
|
||||
@ -775,15 +771,15 @@ extern "C" {
|
||||
// Decoding
|
||||
//
|
||||
|
||||
// Return batch for single sequence of tokens starting at pos_0
|
||||
// Return batch for single sequence of tokens
|
||||
// The sequence ID will be fixed to 0
|
||||
// The position of the tokens will be tracked automatically by llama_decode
|
||||
//
|
||||
// NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
|
||||
//
|
||||
LLAMA_API struct llama_batch llama_batch_get_one(
|
||||
llama_token * tokens,
|
||||
int32_t n_tokens,
|
||||
llama_pos pos_0,
|
||||
llama_seq_id seq_id);
|
||||
int32_t n_tokens);
|
||||
|
||||
// Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
|
||||
// Each token can be assigned up to n_seq_max sequence ids
|
||||
@ -896,6 +892,7 @@ extern "C" {
|
||||
// Special tokens
|
||||
LLAMA_API llama_token llama_token_bos(const struct llama_model * model); // beginning-of-sentence
|
||||
LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentence
|
||||
LLAMA_API llama_token llama_token_eot(const struct llama_model * model); // end-of-turn
|
||||
LLAMA_API llama_token llama_token_cls(const struct llama_model * model); // classification
|
||||
LLAMA_API llama_token llama_token_sep(const struct llama_model * model); // sentence separator
|
||||
LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line
|
||||
@ -904,11 +901,17 @@ extern "C" {
|
||||
LLAMA_API bool llama_add_bos_token(const struct llama_model * model);
|
||||
LLAMA_API bool llama_add_eos_token(const struct llama_model * model);
|
||||
|
||||
// Codellama infill tokens
|
||||
LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix
|
||||
LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle
|
||||
LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix
|
||||
LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle
|
||||
// infill tokens
|
||||
DEPRECATED(LLAMA_API llama_token llama_token_prefix(const struct llama_model * model), "use llama_token_fim_pre instead");
|
||||
DEPRECATED(LLAMA_API llama_token llama_token_middle(const struct llama_model * model), "use llama_token_fim_mid instead");
|
||||
DEPRECATED(LLAMA_API llama_token llama_token_suffix(const struct llama_model * model), "use llama_token_fim_suf instead");
|
||||
|
||||
LLAMA_API llama_token llama_token_fim_pre(const struct llama_model * model);
|
||||
LLAMA_API llama_token llama_token_fim_suf(const struct llama_model * model);
|
||||
LLAMA_API llama_token llama_token_fim_mid(const struct llama_model * model);
|
||||
LLAMA_API llama_token llama_token_fim_pad(const struct llama_model * model);
|
||||
LLAMA_API llama_token llama_token_fim_rep(const struct llama_model * model);
|
||||
LLAMA_API llama_token llama_token_fim_sep(const struct llama_model * model);
|
||||
|
||||
//
|
||||
// Tokenization
|
||||
@ -1067,12 +1070,13 @@ extern "C" {
|
||||
|
||||
// available samplers:
|
||||
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_greedy (void);
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed);
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void);
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed);
|
||||
|
||||
/// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.
|
||||
/// NOTE: Avoid using on the full vocabulary as the sorting can become slow. For example, apply top-k or top-p sampling first.
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void);
|
||||
DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void),
|
||||
"will be removed in the future (see https://github.com/ggerganov/llama.cpp/pull/9896#discussion_r1800920915)");
|
||||
|
||||
/// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k);
|
||||
@ -1088,11 +1092,16 @@ extern "C" {
|
||||
|
||||
/// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep);
|
||||
|
||||
/// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t);
|
||||
|
||||
/// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772.
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent);
|
||||
|
||||
/// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed);
|
||||
|
||||
/// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
|
||||
/// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
|
||||
/// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
|
||||
@ -1132,11 +1141,43 @@ extern "C" {
|
||||
bool penalize_nl, // consider newlines as a repeatable token
|
||||
bool ignore_eos); // ignore the end-of-sequence token
|
||||
|
||||
/// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_dry(
|
||||
const struct llama_model * model,
|
||||
float dry_multiplier,
|
||||
float dry_base,
|
||||
int32_t dry_allowed_length,
|
||||
int32_t dry_penalty_last_n,
|
||||
const char ** seq_breakers,
|
||||
size_t num_breakers);
|
||||
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias(
|
||||
int32_t n_vocab,
|
||||
int32_t n_logit_bias,
|
||||
const llama_logit_bias * logit_bias);
|
||||
|
||||
// this sampler is meant to be used for fill-in-the-middle infilling
|
||||
// it's supposed to be used after top_k + top_p sampling
|
||||
//
|
||||
// 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG
|
||||
// 2. combine probs of tokens that have the same prefix
|
||||
//
|
||||
// example:
|
||||
//
|
||||
// - before:
|
||||
// "hel": 0.5
|
||||
// "hell": 0.2
|
||||
// "hello": 0.1
|
||||
// "dummy": 0.1
|
||||
//
|
||||
// - after:
|
||||
// "hel": 0.8
|
||||
// "dummy": 0.1
|
||||
//
|
||||
// 3. discard non-EOG tokens with low prob
|
||||
// 4. if no tokens are left -> pick EOT
|
||||
//
|
||||
LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_model * model);
|
||||
|
||||
// Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise
|
||||
LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl);
|
||||
|
||||
@ -2311,7 +2311,7 @@ const std::unordered_set<uint32_t> unicode_set_whitespace = {
|
||||
0x003000,
|
||||
};
|
||||
|
||||
// list is always in ascending order, to enable binary searh
|
||||
// list is always in ascending order, to enable binary search
|
||||
const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_lowercase = {
|
||||
{0x000041, 0x000061},
|
||||
{0x000042, 0x000062},
|
||||
@ -3748,7 +3748,7 @@ const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_lowercase
|
||||
{0x01E921, 0x01E943},
|
||||
};
|
||||
|
||||
// list is always in ascending order, to enable binary searh
|
||||
// list is always in ascending order, to enable binary search
|
||||
const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_uppercase = {
|
||||
{0x000061, 0x000041},
|
||||
{0x000062, 0x000042},
|
||||
|
||||
Loading…
Reference in New Issue
Block a user