|
179 | 179 |
|
180 | 180 |
|
181 | 181 | # enum llama_vocab_type {
|
182 |
| -# LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab |
183 |
| -# LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback |
184 |
| -# LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE |
185 |
| -# LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece |
186 |
| -# LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram |
187 |
| -# LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization |
| 182 | +# LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab |
| 183 | +# LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback |
| 184 | +# LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE |
| 185 | +# LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece |
| 186 | +# LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram |
| 187 | +# LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization |
| 188 | +# LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming |
188 | 189 | # };
|
189 | 190 | LLAMA_VOCAB_TYPE_NONE = 0
|
190 | 191 | """For models without vocab"""
|
|
198 | 199 | """T5 tokenizer based on Unigram"""
|
199 | 200 | LLAMA_VOCAB_TYPE_RWKV = 5
|
200 | 201 | """RWKV tokenizer based on greedy tokenization"""
|
| 202 | +LLAMA_VOCAB_TYPE_PLAMO2 = 6 |
| 203 | +"""PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming""" |
201 | 204 |
|
202 | 205 |
|
203 | 206 | # NOTE: Deprecated and will be removed in the future. (already gone in llama.cpp)
|
@@ -2171,7 +2174,7 @@ def llama_kv_self_seq_add(
|
2171 | 2174 | # // - lazily on next llama_decode()
|
2172 | 2175 | # // p0 < 0 : [0, p1]
|
2173 | 2176 | # // p1 < 0 : [p0, inf)
|
2174 |
| -# DEPRECATED(void llama_kv_self_seq_div( |
| 2177 | +# DEPRECATED(LLAMA_API void llama_kv_self_seq_div( |
2175 | 2178 | # struct llama_context * ctx,
|
2176 | 2179 | # llama_seq_id seq_id,
|
2177 | 2180 | # llama_pos p0,
|
|
0 commit comments