from __future__ import annotations
import ctypes
import enum
import os
import pathlib
from ._ggml import (
ggml_abort_callback,
ggml_backend_sched_eval_callback,
ggml_log_callback,
ggml_opt_get_optimizer_params,
)
from typing import (
Callable,
Union,
NewType,
Optional,
TYPE_CHECKING,
)
from llama_cpp._ctypes_extensions import (
load_shared_library,
byref,
ctypes_function_for_shared_library,
)
if TYPE_CHECKING:
from llama_cpp._ctypes_extensions import (
CtypesCData,
CtypesArray,
CtypesPointer,
CtypesVoidPointer,
CtypesRef,
CtypesPointerOrRef,
CtypesFuncPointer,
)
# Specify the base name of the shared library to load
_lib_base_name = "llama"
_override_base_path = os.environ.get("LLAMA_CPP_LIB_PATH")
_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _override_base_path is None else pathlib.Path(_override_base_path)
# Load the library
_lib = load_shared_library(_lib_base_name, _base_path)
ctypes_function = ctypes_function_for_shared_library(_lib)
# llama.h bindings
_lib.llama_max_devices.argtypes = []
_lib.llama_max_devices.restype = ctypes.c_size_t
LLAMA_MAX_DEVICES = _lib.llama_max_devices()
# define LLAMA_DEFAULT_SEED 0xFFFFFFFF
LLAMA_DEFAULT_SEED = 0xFFFFFFFF
# define LLAMA_TOKEN_NULL -1
LLAMA_TOKEN_NULL = -1
# define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'
LLAMA_FILE_MAGIC_GGLA = 0x67676C61
# define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'
LLAMA_FILE_MAGIC_GGSN = 0x6767736E
# define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'
LLAMA_FILE_MAGIC_GGSQ = 0x67677371
# define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN
# define LLAMA_SESSION_VERSION 9
LLAMA_SESSION_VERSION = 9
# define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ
LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ
# define LLAMA_STATE_SEQ_VERSION 2
LLAMA_STATE_SEQ_VERSION = 2
# struct llama_vocab;
llama_vocab_p = NewType("llama_vocab_p", int)
llama_vocab_p_ctypes = ctypes.c_void_p
# struct llama_model;
llama_model_p = NewType("llama_model_p", int)
llama_model_p_ctypes = ctypes.c_void_p
# struct llama_context;
llama_context_p = NewType("llama_context_p", int)
llama_context_p_ctypes = ctypes.c_void_p
# # struct llama_sampler;
# llama_sampler_p = NewType("llama_sampler_p", int)
# llama_sampler_p_ctypes = ctypes.c_void_p
# struct llama_opt_params;
llama_opt_params_p = NewType("llama_opt_params_p", int)
llama_opt_params_p_ctypes = ctypes.c_void_p
# typedef struct llama_memory_i * llama_memory_t;
llama_memory_i_p = NewType("llama_memory_i_p", int)
llama_memory_i_p_ctypes = ctypes.c_void_p
# typedef int32_t llama_pos;
llama_pos = ctypes.c_int32
# typedef int32_t llama_token;
llama_token = ctypes.c_int32
llama_token_p = ctypes.POINTER(llama_token)
# typedef int32_t llama_seq_id;
llama_seq_id = ctypes.c_int32
# enum llama_vocab_type {
# LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab
# LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback
# LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE
# LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece
# LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram
# LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization
# LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming
# };
LLAMA_VOCAB_TYPE_NONE = 0
"""For models without vocab"""
LLAMA_VOCAB_TYPE_SPM = 1
"""LLaMA tokenizer based on byte-level BPE with byte fallback"""
LLAMA_VOCAB_TYPE_BPE = 2
"""GPT-2 tokenizer based on byte-level BPE"""
LLAMA_VOCAB_TYPE_WPM = 3
"""BERT tokenizer based on WordPiece"""
LLAMA_VOCAB_TYPE_UGM = 4
"""T5 tokenizer based on Unigram"""
LLAMA_VOCAB_TYPE_RWKV = 5
"""RWKV tokenizer based on greedy tokenization"""
LLAMA_VOCAB_TYPE_PLAMO2 = 6
"""PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming"""
# NOTE: Deprecated and will be removed in the future. (already gone in llama.cpp)
# https://github.com/ggml-org/llama.cpp/blob/master/src/llama-vocab.h#L10
# // pre-tokenization types
# enum llama_vocab_pre_type {
# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0,
# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1,
# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2,
# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,
# LLAMA_VOCAB_PRE_TYPE_FALCON = 4,
# LLAMA_VOCAB_PRE_TYPE_MPT = 5,
# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6,
# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7,
# LLAMA_VOCAB_PRE_TYPE_REFACT = 8,
# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9,
# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10,
# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11,
# LLAMA_VOCAB_PRE_TYPE_OLMO = 12,
# LLAMA_VOCAB_PRE_TYPE_DBRX = 13,
# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14,
# LLAMA_VOCAB_PRE_TYPE_PORO = 15,
# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16,
# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17,
# LLAMA_VOCAB_PRE_TYPE_VIKING = 18,
# LLAMA_VOCAB_PRE_TYPE_JAIS = 19,
# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20,
# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,
# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,
# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,
# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,
# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25,
# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26,
# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27,
# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28,
# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29,
# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30,
# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31,
# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32,
# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33,
# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34,
# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35,
# LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36,
# LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37,
# LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38,
# LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39,
# LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40,
# LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41,
# LLAMA_VOCAB_PRE_TYPE_AFMOE = 42,
# LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43,
# LLAMA_VOCAB_PRE_TYPE_YOUTU = 44,
# LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45,
# };
LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0
LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1
LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2
LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3
LLAMA_VOCAB_PRE_TYPE_FALCON = 4
LLAMA_VOCAB_PRE_TYPE_MPT = 5
LLAMA_VOCAB_PRE_TYPE_STARCODER = 6
LLAMA_VOCAB_PRE_TYPE_GPT2 = 7
LLAMA_VOCAB_PRE_TYPE_REFACT = 8
LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9
LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10
LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11
LLAMA_VOCAB_PRE_TYPE_OLMO = 12
LLAMA_VOCAB_PRE_TYPE_DBRX = 13
LLAMA_VOCAB_PRE_TYPE_SMAUG = 14
LLAMA_VOCAB_PRE_TYPE_PORO = 15
LLAMA_VOCAV_PRE_TYPE_CHATGLM3 = 16
LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17
LLAMA_VOCAB_PRE_TYPE_VIKING = 18
LLAMA_VOCAB_PRE_TYPE_JAIS = 19
LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20
LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21
LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22
LLAMA_VOCAB_PRE_TYPE_BLOOM = 23
LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24
LLAMA_VOCAB_PRE_TYPE_EXAONE = 25
LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26
LLAMA_VOCAB_PRE_TYPE_MINERVA = 27
LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28
LLAMA_VOCAB_PRE_TYPE_GPT4O = 29
LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30
LLAMA_VOCAB_PRE_TYPE_TRILLION = 31
LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32
LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33
LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34
LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35
LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36
LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37
LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38
LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39
LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40
LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41
LLAMA_VOCAB_PRE_TYPE_AFMOE = 42
LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43
LLAMA_VOCAB_PRE_TYPE_YOUTU = 44
LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45
# // note: these values should be synchronized with ggml_rope
# // TODO: maybe move this enum to ggml.h (ggml_rope_type)
# enum llama_rope_type {
# LLAMA_ROPE_TYPE_NONE = -1,
# LLAMA_ROPE_TYPE_NORM = 0,
# LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,
# LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE,
# LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE,
# LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,
# };
LLAMA_ROPE_TYPE_NONE = -1
LLAMA_ROPE_TYPE_NORM = 0
LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2
LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE = 8
LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE = 40
LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION = 24
# enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file
# LLAMA_TOKEN_TYPE_UNDEFINED = 0,
# LLAMA_TOKEN_TYPE_NORMAL = 1,
# LLAMA_TOKEN_TYPE_UNKNOWN = 2,
# LLAMA_TOKEN_TYPE_CONTROL = 3,
# LLAMA_TOKEN_TYPE_USER_DEFINED = 4,
# LLAMA_TOKEN_TYPE_UNUSED = 5,
# LLAMA_TOKEN_TYPE_BYTE = 6,
# };
LLAMA_TOKEN_TYPE_UNDEFINED = 0
LLAMA_TOKEN_TYPE_NORMAL = 1
LLAMA_TOKEN_TYPE_UNKNOWN = 2
LLAMA_TOKEN_TYPE_CONTROL = 3
LLAMA_TOKEN_TYPE_USER_DEFINED = 4
LLAMA_TOKEN_TYPE_UNUSED = 5
LLAMA_TOKEN_TYPE_BYTE = 6
# enum llama_token_attr {
# LLAMA_TOKEN_ATTR_UNDEFINED = 0,
# LLAMA_TOKEN_ATTR_UNKNOWN = 1 bool:
...
# LLAMA_API bool llama_supports_mlock (void);
@ctypes_function("llama_supports_mlock", [], ctypes.c_bool)
def llama_supports_mlock() -> bool:
...
# LLAMA_API bool llama_supports_gpu_offload(void);
@ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool)
def llama_supports_gpu_offload() -> bool:
...
# LLAMA_API bool llama_supports_rpc (void);
@ctypes_function("llama_supports_rpc", [], ctypes.c_bool)
def llama_supports_rpc() -> bool:
...
# // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions
# // In some cases the requested values via llama_context_params may differ from the actual values used by the context
# // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732
# LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);
@ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_ctx(ctx: llama_context_p, /) -> int:
...
# LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx);
@ctypes_function("llama_n_ctx_seq", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_ctx_seq(ctx: llama_context_p, /) -> int:
...
# LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);
@ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_batch(ctx: llama_context_p, /) -> int:
...
# LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx);
@ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_ubatch(ctx: llama_context_p, /) -> int:
...
# LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx);
@ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_seq_max(ctx: llama_context_p, /) -> int:
...
# DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead");
@ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_ctx_train(model: llama_model_p, /) -> int:
...
# DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead");
@ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_embd(model: llama_model_p, /) -> int:
...
# DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead");
@ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_layer(model: llama_model_p, /) -> int:
...
# DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead");
@ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_head(model: llama_model_p, /) -> int:
...
# DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead");
@ctypes_function("llama_n_vocab", [llama_vocab_p_ctypes], ctypes.c_int32)
def llama_n_vocab(model: llama_vocab_p, /) -> int:
...
# LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx);
@ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes)
def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]:
...
# LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx);
@ctypes_function("llama_get_memory", [llama_context_p_ctypes], llama_memory_i_p_ctypes)
def llama_get_memory(ctx: llama_context_p, /) -> Optional[llama_memory_i_p]:
...
# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type
@ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int)
def llama_pooling_type(ctx: llama_context_p, /) -> int:
...
# LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);
@ctypes_function("llama_model_get_vocab", [llama_model_p_ctypes], llama_vocab_p_ctypes)
def llama_model_get_vocab(model: llama_model_p, /) -> Optional[llama_vocab_p]:
...
# LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model);
@ctypes_function("llama_model_rope_type", [llama_model_p_ctypes], ctypes.c_int)
def llama_model_rope_type(model: llama_model_p, /) -> int:
...
# LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model);
@ctypes_function("llama_model_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_ctx_train(model: llama_model_p, /) -> int:
...
# LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model);
@ctypes_function("llama_model_n_embd", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_embd(model: llama_model_p, /) -> int:
...
# LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model);
@ctypes_function("llama_model_n_embd_inp", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_embd_inp(model: llama_model_p, /) -> int:
...
# LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model);
@ctypes_function("llama_model_n_embd_out", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_embd_out(model: llama_model_p, /) -> int:
...
# LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model);
@ctypes_function("llama_model_n_layer", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_layer(model: llama_model_p, /) -> int:
...
# LLAMA_API int32_t llama_model_n_head (const struct llama_model * model);
@ctypes_function("llama_model_n_head", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_head(model: llama_model_p, /) -> int:
...
# LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model);
@ctypes_function("llama_model_n_head_kv", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_head_kv(model: llama_model_p, /) -> int:
...
# LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model);
@ctypes_function("llama_model_n_swa", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_swa(model: llama_model_p, /) -> int:
...
# // Get the model's RoPE frequency scaling factor
# LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model);
@ctypes_function("llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float)
def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float:
"""
Get the model's RoPE frequency scaling factor
"""
...
# // Returns the number of classifier outputs (only valid for classifier models)
# // Undefined behavior for non-classifier models
# LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model);
@ctypes_function("llama_model_n_cls_out", [llama_model_p_ctypes], ctypes.c_uint32)
def llama_model_n_cls_out(model: llama_model_p, /) -> int:
"""
Returns the number of classifier outputs (only valid for classifier models)
Undefined behavior for non-classifier models
"""
...
# // Returns label of classifier output by index ( ctypes.c_char_p:
"""
Returns label of classifier output by index ( int:
"""Apply a loaded control vector to a llama_context, or if data is NULL, clear
the currently loaded vector.
n_embd should be the size of a single layer's control, and data should point
to an n_embd x n_layers buffer starting from layer 1.
il_start and il_end are the layer range the vector should apply to (both inclusive)
See llama_control_vector_load in common to load a control vector."""
...
# //
# // Memory
# //
# // Clear the memory contents
# // If data == true, the data buffers will also be cleared together with the metadata
# LLAMA_API void llama_memory_clear(
# llama_memory_t mem,
# bool data);
@ctypes_function(
"llama_memory_clear", [
llama_memory_i_p_ctypes,
ctypes.c_bool
],
None,
)
def llama_memory_clear(
mem: llama_memory_i_p,
data: bool
):
...
# // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
# // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails
# // seq_id < 0 : match any sequence
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API bool llama_memory_seq_rm(
# llama_memory_t mem,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1);
@ctypes_function(
"llama_memory_seq_rm",
[
llama_memory_i_p_ctypes,
llama_seq_id,
llama_pos,
llama_pos,
],
ctypes.c_bool,
)
def llama_memory_seq_rm(
mem: llama_memory_i_p,
seq_id: Union[llama_seq_id, int],
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
/,
) -> bool:
"""Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails
seq_id < 0 : match any sequence
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
...
# // Copy all tokens that belong to the specified sequence to another sequence
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_memory_seq_cp(
# llama_memory_t mem,
# llama_seq_id seq_id_src,
# llama_seq_id seq_id_dst,
# llama_pos p0,
# llama_pos p1);
@ctypes_function(
"llama_memory_seq_cp",
[
llama_memory_i_p_ctypes,
llama_seq_id,
llama_seq_id,
llama_pos,
llama_pos,
],
None,
)
def llama_memory_seq_cp(
mem: llama_memory_i_p,
seq_id_src: Union[llama_seq_id, int],
seq_id_dst: Union[llama_seq_id, int],
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
/,
):
"""Copy all tokens that belong to the specified sequence to another sequence
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
...
# // Removes all tokens that do not belong to the specified sequence
# LLAMA_API void llama_memory_seq_keep(
# llama_memory_t mem,
# llama_seq_id seq_id);
@ctypes_function(
"llama_memory_seq_keep", [llama_memory_i_p_ctypes, llama_seq_id], None
)
def llama_memory_seq_keep(mem: llama_memory_i_p, seq_id: Union[llama_seq_id, int], /):
"""Removes all tokens that do not belong to the specified sequence"""
...
# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_memory_seq_add(
# llama_memory_t mem,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1,
# llama_pos delta);
@ctypes_function(
"llama_memory_seq_add",
[
llama_memory_i_p_ctypes,
llama_seq_id,
llama_pos,
llama_pos,
llama_pos,
],
None,
)
def llama_memory_seq_add(
mem: llama_memory_i_p,
seq_id: Union[llama_seq_id, int],
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
delta: Union[llama_pos, int],
/,
):
"""Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
...
# // Integer division of the positions by factor of `d > 1`
# // p0 < 0 : [0, p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_memory_seq_div(
# llama_memory_t mem,
# llama_seq_id seq_id,
# llama_pos p0,
# llama_pos p1,
# int d);
@ctypes_function(
"llama_memory_seq_div",
[
llama_memory_i_p_ctypes,
llama_seq_id,
llama_pos,
llama_pos,
ctypes.c_int,
],
None,
)
def llama_memory_seq_div(
mem: llama_memory_i_p,
seq_id: Union[llama_seq_id, int],
p0: Union[llama_pos, int],
p1: Union[llama_pos, int],
d: Union[ctypes.c_int, int],
/,
):
"""Integer division of the positions by factor of `d > 1`
p0 < 0 : [0, p1]
p1 < 0 : [p0, inf)"""
...
# // Returns the smallest position present in the memory for the specified sequence
# // This is typically non-zero only for SWA caches
# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
# // Return -1 if the sequence is empty
# LLAMA_API llama_pos llama_memory_seq_pos_min(
# llama_memory_t mem,
# llama_seq_id seq_id);
@ctypes_function(
"llama_memory_seq_pos_min",
[
llama_memory_i_p_ctypes,
llama_seq_id,
],
ctypes.c_int32,
)
def llama_memory_seq_pos_min(
mem: llama_memory_i_p,
seq_id: Union[llama_seq_id, int]
,/) -> int:
"""Returns the smallest position present in the memory for the specified sequence
This is typically non-zero only for SWA caches
Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
Return -1 if the sequence is empty
"""
...
# // Returns the largest position present in the memory for the specified sequence
# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
# // Return -1 if the sequence is empty
# LLAMA_API llama_pos llama_memory_seq_pos_max(
# llama_memory_t mem,
# llama_seq_id seq_id);
@ctypes_function(
"llama_memory_seq_pos_max",
[
llama_memory_i_p_ctypes,
llama_seq_id,
],
ctypes.c_int32,
)
def llama_memory_seq_pos_max(
mem: llama_memory_i_p,
seq_id: Union[llama_seq_id, int]
,/) -> int:
"""Returns the largest position present in the memory for the specified sequence
This is typically non-zero only for SWA caches
Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
Return -1 if the sequence is empty
"""
...
# // Check if the memory supports shifting
# LLAMA_API bool llama_memory_can_shift(llama_memory_t mem);
@ctypes_function(
"llama_memory_can_shift", [llama_memory_i_p_ctypes], ctypes.c_bool)
def llama_memory_can_shift(mem: llama_memory_i_p, /) -> bool:
...
# //
# // State / sessions
# //
# // Returns the *actual* size in bytes of the state
# // (logits, embedding and memory)
# // Only use when saving the state, not when restoring it, otherwise the size may be too small.
# LLAMA_API size_t llama_state_get_size(struct llama_context * ctx);
@ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t)
def llama_state_get_size(ctx: llama_context_p, /) -> int:
"""Returns the *actual* size in bytes of the state (rng, logits, embedding and memory) - will often be smaller after compacting tokens"""
...
# LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx),
# "use llama_state_get_size instead");
@ctypes_function("llama_get_state_size", [llama_context_p_ctypes], ctypes.c_size_t)
def llama_get_state_size(ctx: llama_context_p, /) -> int:
"""Returns the maximum size in bytes of the state (rng, logits, embedding
and kv_cache) - will often be smaller after compacting tokens"""
...
# // Copies the state to the specified destination address.
# // Destination needs to have allocated enough memory.
# // Returns the number of bytes copied
# LLAMA_API size_t llama_state_get_data(
# struct llama_context * ctx,
# uint8_t * dst,
# size_t size);
@ctypes_function(
"llama_state_get_data",
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
ctypes.c_size_t,
],
ctypes.c_size_t,
)
def llama_state_get_data(
ctx: llama_context_p,
dst: CtypesArray[ctypes.c_uint8],
size: Union[ctypes.c_size_t, int],
/,
) -> int:
"""Copies the state to the specified destination address.
Destination needs to have allocated enough memory.
Returns the number of bytes copied"""
...
# LLAMA_API DEPRECATED(size_t llama_copy_state_data(
# struct llama_context * ctx,
# uint8_t * dst),
# "use llama_state_get_data instead");
@ctypes_function(
"llama_copy_state_data",
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
],
ctypes.c_size_t,
)
def llama_copy_state_data(
ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], /
) -> int:
"""Copies the state to the specified destination address.
Destination needs to have allocated enough memory.
Returns the number of bytes copied"""
...
# // Set the state reading from the specified address
# // Returns the number of bytes read
# LLAMA_API size_t llama_state_set_data(
# struct llama_context * ctx,
# const uint8_t * src,
# size_t size);
@ctypes_function(
"llama_state_set_data",
[llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t],
ctypes.c_size_t,
)
def llama_state_set_data(
ctx: llama_context_p,
src: CtypesArray[ctypes.c_uint8],
size: Union[ctypes.c_size_t, int],
/,
) -> int:
"""Set the state reading from the specified address
Returns the number of bytes read"""
...
# LLAMA_API DEPRECATED(size_t llama_set_state_data(
# struct llama_context * ctx,
# const uint8_t * src),
# "use llama_state_set_data instead");
@ctypes_function(
"llama_set_state_data",
[llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)],
ctypes.c_size_t,
)
def llama_set_state_data(
ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], /
) -> int:
"""Set the state reading from the specified address"""
...
# Save/load session file
# LLAMA_API bool llama_state_load_file(
# struct llama_context * ctx,
# const char * path_session,
# llama_token * tokens_out,
# size_t n_token_capacity,
# size_t * n_token_count_out);
@ctypes_function(
"llama_state_load_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_token_p,
ctypes.c_size_t,
ctypes.POINTER(ctypes.c_size_t),
],
ctypes.c_bool,
)
def llama_state_load_file(
ctx: llama_context_p,
path_session: bytes,
tokens_out: CtypesArray[llama_token],
n_token_capacity: Union[ctypes.c_size_t, int],
n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
/,
) -> bool:
...
# LLAMA_API DEPRECATED(bool llama_load_session_file(
# struct llama_context * ctx,
# const char * path_session,
# llama_token * tokens_out,
# size_t n_token_capacity,
# size_t * n_token_count_out),
# "use llama_state_load_file instead");
@ctypes_function(
"llama_load_session_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_token_p,
ctypes.c_size_t,
ctypes.POINTER(ctypes.c_size_t),
],
ctypes.c_size_t,
)
def llama_load_session_file(
ctx: llama_context_p,
path_session: bytes,
tokens_out: CtypesArray[llama_token],
n_token_capacity: Union[ctypes.c_size_t, int],
n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
/,
) -> int:
...
# LLAMA_API bool llama_state_save_file(
# struct llama_context * ctx,
# const char * path_session,
# const llama_token * tokens,
# size_t n_token_count);
@ctypes_function(
"llama_state_save_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_token_p,
ctypes.c_size_t,
],
ctypes.c_bool,
)
def llama_state_save_file(
ctx: llama_context_p,
path_session: bytes,
tokens: CtypesArray[llama_token],
n_token_count: Union[ctypes.c_size_t, int],
/,
) -> bool:
...
# LLAMA_API DEPRECATED(bool llama_save_session_file(
# struct llama_context * ctx,
# const char * path_session,
# const llama_token * tokens,
# size_t n_token_count),
# "use llama_state_save_file instead");
@ctypes_function(
"llama_save_session_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_token_p,
ctypes.c_size_t,
],
ctypes.c_size_t,
)
def llama_save_session_file(
ctx: llama_context_p,
path_session: bytes,
tokens: CtypesArray[llama_token],
n_token_count: Union[ctypes.c_size_t, int],
/,
) -> int:
...
# // Get the exact size needed to copy the state of a single sequence
# LLAMA_API size_t llama_state_seq_get_size(
# struct llama_context * ctx,
# llama_seq_id seq_id);
@ctypes_function(
"llama_state_seq_get_size",
[llama_context_p_ctypes, llama_seq_id],
llama_seq_id,
)
def llama_state_seq_get_size(ctx: llama_context_p, seq_id: llama_seq_id, /) -> int:
"""Get the exact size needed to copy the state of a single sequence"""
...
# // Copy the state of a single sequence into the specified buffer
# LLAMA_API size_t llama_state_seq_get_data(
# struct llama_context * ctx,
# uint8_t * dst,
# size_t size,
# llama_seq_id seq_id);
@ctypes_function(
"llama_state_seq_get_data",
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
ctypes.c_size_t,
llama_seq_id,
],
ctypes.c_size_t,
)
def llama_state_seq_get_data(
ctx: llama_context_p,
dst: CtypesArray[ctypes.c_uint8],
size: Union[ctypes.c_size_t, int],
seq_id: llama_seq_id,
/,
) -> int:
"""Copy the state of a single sequence into the specified buffer"""
...
# // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence
# // Returns:
# // - Positive: Ok
# // - Zero: Failed to load
# LLAMA_API size_t llama_state_seq_set_data(
# struct llama_context * ctx,
# const uint8_t * src,
# size_t size,
# llama_seq_id dest_seq_id);
@ctypes_function(
"llama_state_seq_set_data",
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
ctypes.c_size_t,
llama_seq_id,
],
ctypes.c_size_t,
)
def llama_state_seq_set_data(
ctx: llama_context_p,
src: CtypesArray[ctypes.c_uint8],
size: Union[ctypes.c_size_t, int],
dest_seq_id: llama_seq_id,
/,
) -> int:
"""Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence"""
...
# LLAMA_API size_t llama_state_seq_save_file(
# struct llama_context * ctx,
# const char * filepath,
# llama_seq_id seq_id,
# const llama_token * tokens,
# size_t n_token_count);
@ctypes_function(
"llama_state_seq_save_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_seq_id,
llama_token_p,
ctypes.c_size_t,
],
ctypes.c_size_t,
)
def llama_state_seq_save_file(
ctx: llama_context_p,
filepath: bytes,
seq_id: llama_seq_id,
tokens: CtypesArray[llama_token],
n_token_count: Union[ctypes.c_size_t, int],
/,
) -> int:
...
# LLAMA_API size_t llama_state_seq_load_file(
# struct llama_context * ctx,
# const char * filepath,
# llama_seq_id dest_seq_id,
# llama_token * tokens_out,
# size_t n_token_capacity,
# size_t * n_token_count_out);
@ctypes_function(
"llama_state_seq_load_file",
[
llama_context_p_ctypes,
ctypes.c_char_p,
llama_seq_id,
llama_token_p,
ctypes.c_size_t,
ctypes.POINTER(ctypes.c_size_t),
],
ctypes.c_size_t,
)
def llama_state_seq_load_file(
ctx: llama_context_p,
filepath: bytes,
dest_seq_id: llama_seq_id,
tokens_out: CtypesArray[llama_token],
n_token_capacity: Union[ctypes.c_size_t, int],
n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
/,
) -> int:
...
# // for backwards-compat
LLAMA_STATE_SEQ_FLAGS_SWA_ONLY = 1
# // work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba)
LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY = 1
llama_state_seq_flags = ctypes.c_uint32
# LLAMA_API size_t llama_state_seq_get_size_ext(
# struct llama_context * ctx,
# llama_seq_id seq_id,
# llama_state_seq_flags flags);
@ctypes_function(
"llama_state_seq_get_size_ext",
[
llama_context_p_ctypes,
llama_seq_id,
llama_state_seq_flags,
],
ctypes.c_size_t,
)
def llama_state_seq_get_size_ext(
ctx: llama_context_p,
seq_id: llama_seq_id,
flags: llama_state_seq_flags,
/,
) -> int:
...
# LLAMA_API size_t llama_state_seq_get_data_ext(
# struct llama_context * ctx,
# uint8_t * dst,
# size_t size,
# llama_seq_id seq_id,
# llama_state_seq_flags flags);
@ctypes_function(
"llama_state_seq_get_data_ext",
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
ctypes.c_size_t,
llama_seq_id,
llama_state_seq_flags,
],
ctypes.c_size_t,
)
def llama_state_seq_get_data_ext(
ctx: llama_context_p,
dst: ctypes.POINTER(ctypes.c_uint8),
size: Union[int, ctypes.c_size_t],
seq_id: llama_seq_id,
flags: llama_state_seq_flags,
/,
) -> int:
...
# LLAMA_API size_t llama_state_seq_set_data_ext(
# struct llama_context * ctx,
# const uint8_t * src,
# size_t size,
# llama_seq_id dest_seq_id,
# llama_state_seq_flags flags);
@ctypes_function(
"llama_state_seq_set_data_ext",
[
llama_context_p_ctypes,
ctypes.POINTER(ctypes.c_uint8),
ctypes.c_size_t,
llama_seq_id,
llama_state_seq_flags,
],
ctypes.c_size_t,
)
def llama_state_seq_set_data_ext(
ctx: llama_context_p,
src: ctypes.POINTER(ctypes.c_uint8),
size: Union[int, ctypes.c_size_t],
dest_seq_id: llama_seq_id,
flags: llama_state_seq_flags,
/,
) -> int:
...
# //
# // Decoding
# //
# // Return batch for single sequence of tokens
# // The sequence ID will be fixed to 0
# // The position of the tokens will be tracked automatically by llama_decode
# //
# // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
# //
# LLAMA_API struct llama_batch llama_batch_get_one(
# llama_token * tokens,
# int32_t n_tokens);
@ctypes_function(
"llama_batch_get_one",
[
llama_token_p,
ctypes.c_int32,
],
llama_batch,
)
def llama_batch_get_one(
tokens: CtypesArray[llama_token],
n_tokens: Union[ctypes.c_int, int],
/,
) -> llama_batch:
"""Return batch for single sequence of tokens starting at pos_0
NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
"""
...
# // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
# // Each token can be assigned up to n_seq_max sequence ids
# // The batch has to be freed with llama_batch_free()
# // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
# // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
# // The rest of the llama_batch members are allocated with size n_tokens
# // All members are left uninitialized
# LLAMA_API struct llama_batch llama_batch_init(
# int32_t n_tokens,
# int32_t embd,
# int32_t n_seq_max);
@ctypes_function(
"llama_batch_init", [ctypes.c_int32, ctypes.c_int32, ctypes.c_int32], llama_batch
)
def llama_batch_init(
n_tokens: ctypes.c_int32,
embd: ctypes.c_int32,
n_seq_max: ctypes.c_int32,
/,
) -> llama_batch:
"""Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
Each token can be assigned up to n_seq_max sequence ids
The batch has to be freed with llama_batch_free()
If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
The rest of the llama_batch members are allocated with size n_tokens
All members are left uninitialized"""
...
# // Frees a batch of tokens allocated with llama_batch_init()
# LLAMA_API void llama_batch_free(struct llama_batch batch);
@ctypes_function("llama_batch_free", [llama_batch], None)
def llama_batch_free(batch: llama_batch, /):
"""Frees a batch of tokens allocated with llama_batch_init()"""
...
# // Process a batch of tokens.
# // In contrast to llama_decode() - this call does not use KV cache.
# // For encode-decoder contexts, processes the batch using the encoder.
# // Can store the encoder output internally for later use by the decoder's cross-attention layers.
# // 0 - success
# // < 0 - error. the memory state is restored to the state before this call
# LLAMA_API int32_t llama_encode(
# struct llama_context * ctx,
# struct llama_batch batch);
@ctypes_function("llama_encode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32)
def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int:
"""Processes a batch of tokens with the ecoder part of the encoder-decoder model.
Stores the encoder output internally for later use by the decoder cross-attention layers.
0 - success
< 0 - error"""
...
# // Process a batch of tokens.
# // Requires the context to have a memory.
# // For encode-decoder contexts, processes the batch using the decoder.
# // Positive return values does not mean a fatal error, but rather a warning.
# // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context
# // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max()
# // Upon other return values, the memory state is restored to the state before this call
# // 0 - success
# // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
# // 2 - aborted (processed ubatches will remain in the context's memory)
# // -1 - invalid input batch
# // < -1 - fatal error (processed ubatches will remain in the context's memory)
# LLAMA_API int32_t llama_decode(
# struct llama_context * ctx,
# struct llama_batch batch);
@ctypes_function("llama_decode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32)
def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int:
"""Positive return values does not mean a fatal error, but rather a warning.
0 - success
1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
2 - aborted (processed ubatches will remain in the context's memory)
-1 - invalid input batch
< -1 - fatal error (processed ubatches will remain in the context's memory)
"""
...
# // Set the number of threads used for decoding
# // n_threads is the number of threads used for generation (single token)
# // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch);
@ctypes_function(
"llama_set_n_threads",
[
llama_context_p_ctypes,
ctypes.c_int32,
ctypes.c_int32,
],
None,
)
def llama_set_n_threads(
ctx: llama_context_p,
n_threads: Union[ctypes.c_int32, int],
n_threads_batch: Union[ctypes.c_int32, int],
/,
):
"""Set the number of threads used for decoding
n_threads is the number of threads used for generation (single token)
n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
"""
...
# // Get the number of threads used for generation of a single token.
# LLAMA_API int32_t llama_n_threads(struct llama_context * ctx);
@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32)
def llama_n_threads(ctx: llama_context_p, /) -> int:
"""Get the number of threads used for generation of a single token"""
...
# // Get the number of threads used for prompt and batch processing (multiple token).
# LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx);
@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32)
def llama_n_threads_batch(ctx: llama_context_p, /) -> int:
"""Get the number of threads used for prompt and batch processing (multiple token)"""
...
# // Set whether the model is in embeddings mode or not
# // TODO: rename to avoid confusion with llama_get_embeddings()
# LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings);
@ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /):
"""
Set whether the model is in embeddings model or not
"""
...
# // Set whether to use causal attention or not
# // If set to true, the model will only attend to the past tokens
# LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);
@ctypes_function("llama_set_causal_attn", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /):
"""Set whether to use causal attention or not
If set to true, the model will only attend to the past tokens"""
...
# // Set whether the model is in warmup mode or not
# // If true, all model tensors are activated during llama_decode() to load and cache their weights.
# LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup);
@ctypes_function("llama_set_warmup", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_warmup(ctx: llama_context_p, warmup: bool, /):
""" Set whether the model is in warmup mode or not
If true, all model tensors are activated during llama_decode() to load and cache their weights"""
...
# // Set abort callback
# LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);
@ctypes_function(
"llama_set_abort_callback",
[llama_context_p_ctypes, ggml_abort_callback, ctypes.c_void_p],
None,
)
def llama_set_abort_callback(
ctx: llama_context_p,
abort_callback: Callable[[ctypes.c_void_p], None],
abort_callback_data: ctypes.c_void_p,
/,
):
"""Set abort callback"""
...
# // Wait until all computations are finished
# // This is automatically done when using one of the functions below to obtain the computation results
# // and is not necessary to call it explicitly in most cases
# LLAMA_API void llama_synchronize(struct llama_context * ctx);
@ctypes_function("llama_synchronize", [llama_context_p_ctypes], None)
def llama_synchronize(ctx: llama_context_p, /):
"""Wait until all computations are finished
This is automatically done when using one of the functions below to obtain the computation results
and is not necessary to call it explicitly in most cases"""
...
# // Token logits obtained from the last call to llama_decode()
# // The logits for which llama_batch.logits[i] != 0 are stored contiguously
# // in the order they have appeared in the batch.
# // Rows: number of tokens for which llama_batch.logits[i] != 0
# // Cols: n_vocab
# // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
# LLAMA_API float * llama_get_logits(struct llama_context * ctx);
@ctypes_function(
"llama_get_logits", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float)
)
def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]:
"""Token logits obtained from the last call to llama_decode()
The logits for which llama_batch.logits[i] != 0 are stored contiguously
in the order they have appeared in the batch.
Rows: number of tokens for which llama_batch.logits[i] != 0
Cols: n_vocab
Returns:
Pointer to the logits buffer of shape (n_tokens, n_vocab)"""
...
# // Logits for the ith token. For positive indices, Equivalent to:
# // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab
# // Negative indicies can be used to access logits in reverse order, -1 is the last logit.
# // returns NULL for invalid ids.
# LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_logits_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(ctypes.c_float),
)
def llama_get_logits_ith(
ctx: llama_context_p, i: ctypes.c_int32, /
) -> ctypes.POINTER(ctypes.c_float):
"""Logits for the ith token. Equivalent to:
llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab"""
...
# // Get all output token embeddings.
# // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,
# // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously
# // in the order they have appeared in the batch.
# // shape: [n_outputs*n_embd]
# // Otherwise, returns NULL.
# // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
# LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
@ctypes_function(
"llama_get_embeddings", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float)
)
def llama_get_embeddings(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]:
"""Get the embeddings for the input
shape: [n_embd] (1-dimensional)"""
...
# // Get the embeddings for the ith token. For positive indices, Equivalent to:
# // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd
# // Negative indicies can be used to access embeddings in reverse order, -1 is the last embedding.
# // shape: [n_embd] (1-dimensional)
# // returns NULL for invalid ids.
# LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_embeddings_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(ctypes.c_float),
)
def llama_get_embeddings_ith(
ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> CtypesArray[ctypes.c_float]:
"""Get the embeddings for the ith sequence
llama_get_embeddings(ctx) + i*n_embd"""
...
# // Get the embeddings for a sequence id
# // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE
# // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[1] with the rank of the sequence
# // otherwise: float[n_embd] (1-dimensional)
# LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);
@ctypes_function(
"llama_get_embeddings_seq",
[llama_context_p_ctypes, llama_seq_id],
ctypes.POINTER(ctypes.c_float),
)
def llama_get_embeddings_seq(
ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /
) -> CtypesArray[ctypes.c_float]:
"""Get the embeddings for a sequence id
Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE
shape: [n_embd] (1-dimensional)"""
...
# //
# // backend sampling API [EXPERIMENTAL]
# // note: use only if the llama_context was created with at least one llama_sampler_seq_config
# //
# // Get the backend sampled token for the ith token.
# // Returns LLAMA_TOKEN_NULL if no token was sampled.
# LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_token_ith",
[llama_context_p_ctypes, ctypes.c_int32],
llama_token,
)
def llama_get_sampled_token_ith(
ctx: llama_context_p, i: ctypes.c_int32, /
) -> ctypes.c_int32:
"""
Get the backend sampled token for the ith token.
Returns LLAMA_TOKEN_NULL if no token was sampled.
"""
...
# // Get the backend sampled probabilites for the ith token
# // The index matches llama_get_sampled_token_ith().
# // Returns NULL if no probabilites were generated.
# LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_probs_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(ctypes.c_float),
)
def llama_get_sampled_probs_ith(
ctx: llama_context_p, i: ctypes.c_int32, /
) -> CtypesArray[ctypes.c_float]:
"""
Get the backend sampled probabilites for the ith token
The index matches llama_get_sampled_token_ith().
Returns NULL if no probabilites were generated.
"""
...
# LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_probs_count_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.c_uint32,
)
def llama_get_sampled_probs_count_ith(
ctx: llama_context_p, i: ctypes.c_int32, /
) -> ctypes.c_uint32:
"""
Get the backend sampled probabilites count for the ith token
"""
...
# // Get the backend sampled logits for the ith token
# // Returns NULL if no logits were sampled.
# LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_logits_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(ctypes.c_float),
)
def llama_get_sampled_logits_ith(
ctx: llama_context_p, i: ctypes.c_int32, /
) -> CtypesArray[ctypes.c_float]:
"""
Get the backend sampled logits for the ith token
Returns NULL if no logits were sampled.
"""
...
# LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_logits_count_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.c_uint32,
)
def llama_get_sampled_logits_count_ith(
ctx: llama_context_p, i: ctypes.c_int32, /
) -> ctypes.c_uint32:
"""
Get the backend sampled logits count for the ith token
"""
...
# // Get the backend sampled candidates (token ids) for the ith token
# // These are needed to map probability/logit indices to vocab token ids.
# // Returns NULL if no candidates were sampled.
# LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_candidates_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.POINTER(llama_token),
)
def llama_get_sampled_candidates_ith(
ctx: llama_context_p, i: ctypes.c_int32, /
) -> CtypesArray[llama_token]:
"""
Get the backend sampled candidates (token ids) for the ith token
These are needed to map probability/logit indices to vocab token ids.
Returns NULL if no candidates were sampled.
"""
...
# LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
"llama_get_sampled_candidates_count_ith",
[llama_context_p_ctypes, ctypes.c_int32],
ctypes.c_uint32,
)
def llama_get_sampled_candidates_count_ith(
ctx: llama_context_p, i: ctypes.c_int32, /
) -> ctypes.c_uint32:
"""
Get the backend sampled candidates (token ids) count for the ith token
"""
...
# //
# // Vocab
# //
# LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
"llama_vocab_get_text", [llama_vocab_p_ctypes, llama_token], ctypes.c_char_p
)
def llama_vocab_get_text(
vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bytes:
...
# LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
"llama_vocab_get_score", [llama_vocab_p_ctypes, llama_token], ctypes.c_float
)
def llama_vocab_get_score(
vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> float:
...
# LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
"llama_vocab_get_attr", [llama_vocab_p_ctypes, llama_token], ctypes.c_int
)
def llama_vocab_get_attr(
vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> int:
...
# // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)
# LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
"llama_vocab_is_eog", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool
)
def llama_vocab_is_eog(vocab: llama_vocab_p, token: Union[llama_token, int], /) -> bool:
"""Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)"""
...
# // Identify if Token Id is a control token or a render-able token
# LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
"llama_vocab_is_control", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool
)
def llama_vocab_is_control(
vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bool:
"""Identify if Token Id is a control token or a render-able token"""
...
# // Special tokens
# LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence
@ctypes_function("llama_vocab_bos", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_bos(vocab: llama_vocab_p, /) -> llama_token:
"""beginning-of-sentence"""
...
# LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence
@ctypes_function("llama_vocab_eos", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_eos(vocab: llama_vocab_p, /) -> llama_token:
"""end-of-sentence"""
...
# LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn
@ctypes_function("llama_vocab_eot", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_eot(vocab: llama_vocab_p, /) -> llama_token:
"""end-of-turn"""
...
# LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator
@ctypes_function("llama_vocab_sep", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_sep(vocab: llama_vocab_p, /) -> llama_token:
"""sentence separator"""
...
# LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line
@ctypes_function("llama_vocab_nl", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_nl(vocab: llama_vocab_p, /) -> llama_token:
"""next-line"""
...
# LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding
@ctypes_function("llama_vocab_pad", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_pad(vocab: llama_vocab_p, /) -> llama_token:
"""padding"""
...
# LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask
@ctypes_function("llama_vocab_mask", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_mask(vocab: llama_vocab_p, /) -> llama_token:
"""mask"""
...
# LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_get_add_bos",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
def llama_vocab_get_add_bos(vocab: llama_vocab_p, /) -> bool:
...
# LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_get_add_eos",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool:
...
# LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_get_add_sep",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool:
...
# LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_pre",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_vocab_fim_pre(vocab: llama_vocab_p, /) -> llama_token:
...
# LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_suf",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_vocab_fim_suf(vocab: llama_vocab_p, /) -> llama_token:
...
# LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_mid",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_vocab_fim_mid(vocab: llama_vocab_p, /) -> llama_token:
...
# LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_pad",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_vocab_fim_pad(vocab: llama_vocab_p, /) -> llama_token:
...
# LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_rep",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_vocab_fim_rep(vocab: llama_vocab_p, /) -> llama_token:
...
# LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab);
@ctypes_function(
"llama_vocab_fim_sep",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token:
...
# DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead");
@ctypes_function(
"llama_token_get_text",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_char_p,
)
def llama_token_get_text(
vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bytes:
...
# DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead");
@ctypes_function(
"llama_token_get_score",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_float,
)
def llama_token_get_score(
vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> float:
...
# DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead");
@ctypes_function(
"llama_token_get_attr",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_int,
)
def llama_token_get_attr(
vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> int:
...
# DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead");
@ctypes_function(
"llama_token_is_eog",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_bool,
)
def llama_token_is_eog(
vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bool:
...
# DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead");
@ctypes_function(
"llama_token_is_control",
[llama_vocab_p_ctypes, llama_token],
ctypes.c_bool,
)
def llama_token_is_control(
vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bool:
...
# DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead");
@ctypes_function(
"llama_token_bos",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_bos(vocab: llama_vocab_p, /) -> int:
...
# DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead");
@ctypes_function(
"llama_token_eos",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_eos(vocab: llama_vocab_p, /) -> int:
...
# DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead");
@ctypes_function(
"llama_token_eot",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_eot(vocab: llama_vocab_p, /) -> int:
...
# DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead");
@ctypes_function(
"llama_token_cls",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_cls(vocab: llama_vocab_p, /) -> int:
...
# DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead");
@ctypes_function(
"llama_token_sep",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_sep(vocab: llama_vocab_p, /) -> int:
...
# DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead");
@ctypes_function(
"llama_token_nl",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_nl(vocab: llama_vocab_p, /) -> int:
...
# DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead");
@ctypes_function(
"llama_token_pad",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_pad(vocab: llama_vocab_p, /) -> int:
...
# DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead");
@ctypes_function(
"llama_add_bos_token",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool:
...
# DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead");
@ctypes_function(
"llama_add_eos_token",
[llama_vocab_p_ctypes],
ctypes.c_bool,
)
def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool:
...
# DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead");
@ctypes_function(
"llama_token_fim_pre",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token:
...
# DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead");
@ctypes_function(
"llama_token_fim_suf",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token:
...
# DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead");
@ctypes_function(
"llama_token_fim_mid",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token:
...
# DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead");
@ctypes_function(
"llama_token_fim_pad",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token:
...
# DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead");
@ctypes_function(
"llama_token_fim_rep",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token:
...
# DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead");
@ctypes_function(
"llama_token_fim_sep",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token:
...
# // CLS is equivalent to BOS
# DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification
# "use llama_vocab_bos instead");
@ctypes_function(
"llama_vocab_cls",
[llama_vocab_p_ctypes],
llama_token,
)
def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token:
...
# //
# // Tokenization
# //
# // The API is thread-safe.
# //
# /// @details Convert the provided text into tokens.
# /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.
# /// @return Returns the number of tokens on success, no more than n_tokens_max
# /// @return Returns a negative number on failure - the number of tokens that would have been returned
# /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit)
# /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so.
# /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated
# /// as plaintext. Does not insert a leading space.
# LLAMA_API int32_t llama_tokenize(
# const struct llama_vocab * vocab,
# const char * text,
# int32_t text_len,
# llama_token * tokens,
# int32_t n_tokens_max,
# bool add_special,
# bool parse_special);
@ctypes_function(
"llama_tokenize",
[
llama_vocab_p_ctypes,
ctypes.c_char_p,
ctypes.c_int32,
llama_token_p,
ctypes.c_int32,
ctypes.c_bool,
ctypes.c_bool,
],
ctypes.c_int32,
)
def llama_tokenize(
vocab: llama_vocab_p,
text: bytes,
text_len: Union[ctypes.c_int, int],
tokens: CtypesArray[llama_token],
n_tokens_max: Union[ctypes.c_int, int],
add_special: Union[ctypes.c_bool, bool],
parse_special: Union[ctypes.c_bool, bool],
/,
) -> int:
"""Convert the provided text into tokens.
Args:
vocab: The vocabulary to use for tokenization.
text: The text to tokenize.
text_len: The length of the text.
tokens: The tokens pointer must be large enough to hold the resulting tokens.
n_max_tokens: The maximum number of tokens to return.
add_special: Allow adding special tokenns if the model is configured to do so.
parse_special: Allow parsing special tokens.
Returns:
Returns the number of tokens on success, no more than n_tokens_max
Returns a negative number on failure - the number of tokens that would have been returned
"""
...
# // Token Id -> Piece.
# // Uses the vocabulary in the provided context.
# // Does not write null terminator to the buffer.
# // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix')
# // @param special If true, special tokens are rendered in the output.
# LLAMA_API int32_t llama_token_to_piece(
# const struct llama_vocab * vocab,
# llama_token token,
# char * buf,
# int32_t length,
# int32_t lstrip,
# bool special);
@ctypes_function(
"llama_token_to_piece",
[
llama_vocab_p_ctypes,
llama_token,
ctypes.c_char_p,
ctypes.c_int32,
ctypes.c_int32,
ctypes.c_bool,
],
ctypes.c_int32,
)
def llama_token_to_piece(
vocab: llama_vocab_p,
token: Union[llama_token, int],
buf: Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]],
length: Union[ctypes.c_int, int],
lstrip: Union[ctypes.c_int, int],
special: Union[ctypes.c_bool, bool],
/,
) -> int:
"""Token Id -> Piece.
Uses the vocabulary in the provided context.
Does not write null terminator to the buffer.
User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens.
Args:
vocab: The vocabulary to use for tokenization.
token: The token to convert.
buf: The buffer to write the token to.
length: The length of the buffer.
lstrip: The number of leading spaces to skip.
special: If true, special tokens are rendered in the output."""
...
# /// @details Convert the provided tokens into text (inverse of llama_tokenize()).
# /// @param text The char pointer must be large enough to hold the resulting text.
# /// @return Returns the number of chars/bytes on success, no more than text_len_max.
# /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned.
# /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so.
# /// @param unparse_special If true, special tokens are rendered in the output.
# LLAMA_API int32_t llama_detokenize(
# const struct llama_vocab * vocab,
# const llama_token * tokens,
# int32_t n_tokens,
# char * text,
# int32_t text_len_max,
# bool remove_special,
# bool unparse_special);
@ctypes_function(
"llama_detokenize",
[
llama_vocab_p_ctypes,
llama_token_p,
ctypes.c_int32,
ctypes.c_char_p,
ctypes.c_int32,
ctypes.c_bool,
ctypes.c_bool,
],
ctypes.c_int32,
)
def llama_detokenize(
vocab: llama_vocab_p,
tokens: CtypesArray[llama_token],
n_tokens: Union[ctypes.c_int, int],
text: bytes,
text_len_max: Union[ctypes.c_int, int],
remove_special: Union[ctypes.c_bool, bool],
unparse_special: Union[ctypes.c_bool, bool],
/,
) -> int:
"""Convert the provided tokens into text (inverse of llama_tokenize()).
Args:
vocab: The model vocab to use for tokenization.
tokens: The tokens to convert.
n_tokens: The number of tokens.
text: The buffer to write the text to.
text_len_max: The length of the buffer.
remove_special: Allow to remove BOS and EOS tokens if model is configured to do so.
unparse_special: If true, special tokens are rendered in the output.
"""
...
# //
# // Chat templates
# //
# /// Apply chat template. Inspired by hf apply_chat_template() on python.
# /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model"
# /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template
# /// @param tmpl A Jinja template to use for this chat. If this is nullptr, the models default chat template will be used instead.
# /// @param chat Pointer to a list of multiple llama_chat_message
# /// @param n_msg Number of llama_chat_message in this chat
# /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message.
# /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages)
# /// @param length The size of the allocated buffer
# /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template.
# LLAMA_API int32_t llama_chat_apply_template(
# const char * tmpl,
# const struct llama_chat_message * chat,
# size_t n_msg,
# bool add_ass,
# char * buf,
# int32_t length);
@ctypes_function(
"llama_chat_apply_template",
[
ctypes.c_char_p, # tmpl
ctypes.POINTER(llama_chat_message), # chat
ctypes.c_size_t, # n_msg
ctypes.c_bool, # add_ass (added)
ctypes.c_char_p, # buf
ctypes.c_int32, # length
],
ctypes.c_int32,
)
def llama_chat_apply_template(
tmpl: bytes,
chat: CtypesArray[llama_chat_message],
n_msg: int,
add_ass: bool, # Added parameter
buf: bytes,
length: int,
/,
) -> int:
"""Apply chat template.
Args:
tmpl: Template to use. If None, uses model's default
chat: Array of chat messages
n_msg: Number of messages
add_ass: Whether to end prompt with assistant token
buf: Output buffer
length: Buffer length
Returns:
Number of bytes written, or needed if buffer too small
"""
...
# // Get list of built-in chat templates
# LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len);
@ctypes_function(
"llama_chat_builtin_templates",
[
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_size_t,
],
ctypes.c_int32,
)
def llama_chat_builtin_templates(
output: CtypesArray[bytes],
len: Union[ctypes.c_size_t, int],
/,
) -> int:
"""Get list of built-in chat templates.
Args:
output: Output buffer to store template names.
len: Length of the output buffer.
Returns:
Number of templates available.
Returns a negative number on error.
"""
...
# //
# // Sampling API
# //
# // Sample usage:
# //
# // // prepare the sampling chain at the start
# // auto sparams = llama_sampler_chain_default_params();
# //
# // llama_sampler * smpl = llama_sampler_chain_init(sparams);
# //
# // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50));
# // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1));
# // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8));
# //
# // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat"
# // // this sampler will be responsible to select the actual token
# // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed));
# //
# // ...
# //
# // // decoding loop:
# // while (...) {
# // ...
# //
# // llama_decode(ctx, batch);
# //
# // // sample from the logits of the last token in the batch
# // const llama_token id = llama_sampler_sample(smpl, ctx, -1);
# //
# // ...
# // }
# //
# // llama_sampler_free(smpl);
# //
# typedef void * llama_sampler_context_t;
llama_sampler_context_t = ctypes.c_void_p
# struct llama_sampler_data {
# struct ggml_tensor * logits;
# struct ggml_tensor * probs;
# struct ggml_tensor * sampled;
# struct ggml_tensor * candidates;
# };
class llama_sampler_data(ctypes.Structure):
if TYPE_CHECKING:
logits: ctypes.c_void_p
probs: ctypes.c_void_p
sampled: ctypes.c_void_p
candidates: ctypes.c_void_p
_fields_ = [
("logits", ctypes.c_void_p),
("probs", ctypes.c_void_p),
("sampled", ctypes.c_void_p),
("candidates", ctypes.c_void_p),
]
# // user code can implement the interface below in order to create custom llama_sampler
# struct llama_sampler_i {
# const char * (*name) (const struct llama_sampler * smpl); // can be NULL
# void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL
# void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required
# void (*reset) ( struct llama_sampler * smpl); // can be NULL
# struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL
# void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL
# // [EXPERIMENTAL]
# // backend sampling interface:
# // return true if the backend supports all ops needed by the sampler
# // note: call once per sampler
# bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft);
# // call after .backend_apply()
# void (*backend_accept)(
# struct llama_sampler * smpl,
# struct ggml_context * ctx,
# struct ggml_cgraph * gf,
# struct ggml_tensor * selected_token);
# // call after .backend_init()
# void (*backend_apply)(
# struct llama_sampler * smpl,
# struct ggml_context * ctx,
# struct ggml_cgraph * gf,
# struct llama_sampler_data * data);
# // called before graph execution to set inputs for the current ubatch
# void (*backend_set_input)(struct llama_sampler * smpl);
# };
class llama_sampler_i(ctypes.Structure):
...
# struct llama_sampler {
# struct llama_sampler_i * iface;
# llama_sampler_context_t ctx;
# };
class llama_sampler(ctypes.Structure):
_fields_ = [
("iface", ctypes.POINTER(llama_sampler_i)),
("ctx", llama_sampler_context_t),
]
if TYPE_CHECKING:
llama_sampler_p = CtypesPointer[llama_sampler]
llama_sampler_p_ctypes = ctypes.POINTER(llama_sampler)
llama_sampler_i_name = ctypes.CFUNCTYPE(ctypes.c_char_p, llama_sampler_p_ctypes)
llama_sampler_i_accept = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes, llama_token)
llama_sampler_i_apply = ctypes.CFUNCTYPE(
None, llama_sampler_p_ctypes, llama_token_data_array_p)
llama_sampler_i_reset = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes)
llama_sampler_i_clone = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, llama_sampler_p_ctypes)
llama_sampler_i_free = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes)
llama_sampler_i_backend_init = ctypes.CFUNCTYPE(
ctypes.c_bool, llama_sampler_p_ctypes, ctypes.c_void_p)
llama_sampler_i_backend_accept = ctypes.CFUNCTYPE(
None, llama_sampler_p_ctypes, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p)
llama_sampler_i_backend_apply = ctypes.CFUNCTYPE(
None, llama_sampler_p_ctypes, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p)
llama_sampler_i_backend_set_input = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes)
llama_sampler_i._fields_ = [
("name", llama_sampler_i_name),
("accept", llama_sampler_i_accept),
("apply", llama_sampler_i_apply),
("reset", llama_sampler_i_reset),
("clone", llama_sampler_i_clone),
("free", llama_sampler_i_free),
("backend_init", llama_sampler_i_backend_init),
("backend_accept", llama_sampler_i_backend_accept),
("backend_apply", llama_sampler_i_backend_apply),
("backend_set_input", llama_sampler_i_backend_set_input),
]
# // [EXPERIMENTAL]
# // attach a sampler to the context
# // note: prefer initializing the context with llama_context_params.samplers when possible
# // note: changing the samplers of a context can cause graph reallocations and degraded performance
# LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl);
@ctypes_function(
"llama_set_sampler",
[llama_context_p_ctypes, llama_seq_id, llama_sampler_p_ctypes],
ctypes.c_bool,
)
def llama_set_sampler(
ctx: llama_context_p, seq_id: llama_seq_id, smpl: llama_sampler_p, /
) -> ctypes.c_bool:
"""
attach a sampler to the context
note: prefer initializing the context with llama_context_params.samplers when possible
note: changing the samplers of a context can cause graph reallocations and degraded performance
"""
...
# // mirror of llama_sampler_i:
# LLAMA_API struct llama_sampler * llama_sampler_init ( struct llama_sampler_i * iface, llama_sampler_context_t ctx);
@ctypes_function(
"llama_sampler_init",
[ctypes.POINTER(llama_sampler_i), llama_sampler_context_t],
llama_sampler_p_ctypes,
)
def llama_sampler_init(
iface: ctypes.pointer(llama_sampler_i), ctx: llama_sampler_context_t, /
) -> llama_sampler_p:
...
# LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl);
@ctypes_function(
"llama_sampler_name",
[llama_sampler_p_ctypes],
ctypes.c_char_p,
)
def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes:
...
# LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token);
@ctypes_function(
"llama_sampler_accept",
[llama_sampler_p_ctypes, llama_token],
None,
)
def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /):
...
# LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p);
@ctypes_function(
"llama_sampler_apply",
[llama_sampler_p_ctypes, llama_token_data_array_p],
None,
)
def llama_sampler_apply(
smpl: llama_sampler_p, cur_p: CtypesArray[llama_token_data_array], /
):
...
# LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl);
@ctypes_function(
"llama_sampler_reset",
[llama_sampler_p_ctypes],
None,
)
def llama_sampler_reset(smpl: llama_sampler_p, /):
...
# LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl);
@ctypes_function(
"llama_sampler_clone",
[llama_sampler_p_ctypes],
llama_sampler_p_ctypes,
)
def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p:
...
# // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add)
# LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl);
@ctypes_function(
"llama_sampler_free",
[llama_sampler_p_ctypes],
None,
)
def llama_sampler_free(smpl: llama_sampler_p, /):
...
# // llama_sampler_chain
# // a type of llama_sampler that can chain multiple samplers one after another
#
# LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params);
@ctypes_function(
"llama_sampler_chain_init",
[llama_sampler_chain_params],
llama_sampler_p_ctypes,
)
def llama_sampler_chain_init(params: llama_sampler_chain_params, /) -> llama_sampler_p:
...
# // important: takes ownership of the sampler object and will free it when llama_sampler_free is called
# LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl);
@ctypes_function(
"llama_sampler_chain_add",
[llama_sampler_p_ctypes, llama_sampler_p_ctypes],
None,
)
def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /):
...
# // return NULL if:
# // - the sampler is NULL
# // - the sampler is not a llama_sampler_chain
# // - the index is out of bounds, unless i == -1
# // - if i == -1, returns the chain itself (can be used to check if the sampler is a chain)
# LLAMA_API struct llama_sampler * llama_sampler_chain_get( struct llama_sampler * chain, int32_t i);
@ctypes_function(
"llama_sampler_chain_get",
[llama_sampler_p_ctypes, ctypes.c_int32],
llama_sampler_p_ctypes,
)
def llama_sampler_chain_get(
chain: llama_sampler_p, i: Union[ctypes.c_int32, int], /
) -> llama_sampler_p:
"""
return NULL if:
- the sampler is NULL
- the sampler is not a llama_sampler_chain
- the index is out of bounds, unless i == -1
- if i == -1, returns the chain itself (can be used to check if the sampler is a chain)
"""
...
# LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain);
@ctypes_function(
"llama_sampler_chain_n",
[llama_sampler_p_ctypes],
ctypes.c_int,
)
def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int:
...
# // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed
# LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i);
@ctypes_function(
"llama_sampler_chain_remove",
[llama_sampler_p_ctypes, ctypes.c_int32],
llama_sampler_p_ctypes,
)
def llama_sampler_chain_remove(
chain: llama_sampler_p, i: Union[ctypes.c_int32, int], /
) -> llama_sampler_p:
...
# // available samplers:
#
# LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void);
@ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes)
def llama_sampler_init_greedy() -> llama_sampler_p:
...
# /// seed == LLAMA_DEFAULT_SEED to use a random seed.
# LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed);
@ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes)
def llama_sampler_init_dist(seed: int) -> llama_sampler_p:
...
# /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
# /// Setting k llama_sampler_p:
...
# /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
# LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep);
@ctypes_function(
"llama_sampler_init_top_p",
[ctypes.c_float, ctypes.c_size_t],
llama_sampler_p_ctypes,
)
def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p:
...
# /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
# LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep);
@ctypes_function(
"llama_sampler_init_min_p",
[ctypes.c_float, ctypes.c_size_t],
llama_sampler_p_ctypes,
)
def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p:
...
# /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
# LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep);
@ctypes_function(
"llama_sampler_init_typical",
[ctypes.c_float, ctypes.c_size_t],
llama_sampler_p_ctypes,
)
def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p:
...
# LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t);
@ctypes_function("llama_sampler_init_temp", [ctypes.c_float], llama_sampler_p_ctypes)
def llama_sampler_init_temp(t: float) -> llama_sampler_p:
...
# /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772.
# LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent);
@ctypes_function(
"llama_sampler_init_temp_ext",
[ctypes.c_float, ctypes.c_float, ctypes.c_float],
llama_sampler_p_ctypes,
)
def llama_sampler_init_temp_ext(
t: float, delta: float, exponent: float
) -> llama_sampler_p:
...
# /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335
# LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed);
@ctypes_function(
"llama_sampler_init_xtc",
[ctypes.c_float, ctypes.c_float, ctypes.c_size_t, ctypes.c_uint32],
llama_sampler_p_ctypes,
)
def llama_sampler_init_xtc(
p: float, t: float, min_keep: int, seed: int, /
) -> llama_sampler_p:
...
# /// @details Top n sigma sampling as described in academic paper "Top-n: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641
# LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float n);
@ctypes_function(
"llama_sampler_init_top_n_sigma",
[ctypes.c_float],
llama_sampler_p_ctypes,
)
def llama_sampler_init_top_n_sigma(n: float, /) -> llama_sampler_p:
...
# /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
# /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
# /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.
# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat(
# int32_t n_vocab,
# uint32_t seed,
# float tau,
# float eta,
# int32_t m);
@ctypes_function(
"llama_sampler_init_mirostat",
[ctypes.c_int32, ctypes.c_uint32, ctypes.c_float, ctypes.c_float, ctypes.c_int32],
llama_sampler_p_ctypes,
)
def llama_sampler_init_mirostat(
n_vocab: int, seed: int, tau: float, eta: float, m: int, /
) -> llama_sampler_p:
...
# /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
# /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2(
# uint32_t seed,
# float tau,
# float eta);
@ctypes_function(
"llama_sampler_init_mirostat_v2",
[ctypes.c_uint32, ctypes.c_float, ctypes.c_float],
llama_sampler_p_ctypes,
)
def llama_sampler_init_mirostat_v2(
seed: int, tau: float, eta: float, /
) -> llama_sampler_p:
...
# /// @details Intializes a GBNF grammar, see grammars/README.md for details.
# /// @param vocab The vocabulary that this grammar will be used with.
# /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails.
# /// @param grammar_root The name of the start symbol for the grammar.
# LLAMA_API struct llama_sampler * llama_sampler_init_grammar(
# const struct llama_vocab * vocab,
# const char * grammar_str,
# const char * grammar_root);
@ctypes_function(
"llama_sampler_init_grammar",
[llama_vocab_p_ctypes, ctypes.c_char_p, ctypes.c_char_p],
llama_sampler_p_ctypes,
)
def llama_sampler_init_grammar(
vocab: llama_vocab_p, grammar_str: bytes, grammar_root: bytes, /
) -> llama_sampler_p:
...
# /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639
# /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group.
# /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included.
# LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns(
# const struct llama_vocab * vocab,
# const char * grammar_str,
# const char * grammar_root,
# const char ** trigger_patterns,
# size_t num_trigger_patterns,
# const llama_token * trigger_tokens,
# size_t num_trigger_tokens);
@ctypes_function(
"llama_sampler_init_grammar_lazy_patterns",
[
llama_vocab_p_ctypes,
ctypes.c_char_p,
ctypes.c_char_p,
ctypes.POINTER(ctypes.c_char_p),
ctypes.c_size_t,
llama_token_p,
ctypes.c_size_t,
],
llama_sampler_p_ctypes,
)
def llama_sampler_init_grammar_lazy_patterns(
vocab: llama_vocab_p,
grammar_str: bytes,
grammar_root: bytes,
trigger_patterns: CtypesArray[bytes],
num_trigger_patterns: int,
trigger_tokens: CtypesArray[llama_token],
num_trigger_tokens: int,
/,
) -> llama_sampler_p:
...
# /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.
# LLAMA_API struct llama_sampler * llama_sampler_init_penalties(
# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size)
# float penalty_repeat, // 1.0 = disabled
# float penalty_freq, // 0.0 = disabled
# float penalty_present); // 0.0 = disabled
@ctypes_function(
"llama_sampler_init_penalties",
[ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float],
llama_sampler_p_ctypes,
)
def llama_sampler_init_penalties(
penalty_last_n: int,
penalty_repeat: float,
penalty_freq: float,
penalty_present: float,
/,
) -> llama_sampler_p:
...
# /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982
# LLAMA_API struct llama_sampler * llama_sampler_init_dry(
# const struct llama_vocab * vocab,
# int32_t n_ctx_train,
# float dry_multiplier,
# float dry_base,
# int32_t dry_allowed_length,
# int32_t dry_penalty_last_n,
# const char ** seq_breakers,
# size_t num_breakers);
@ctypes_function(
"llama_sampler_init_dry",
[
llama_vocab_p_ctypes,
ctypes.c_int32,
ctypes.c_float,
ctypes.c_float,
ctypes.c_int32,
ctypes.c_int32,
ctypes.POINTER(ctypes.POINTER(ctypes.c_char)),
ctypes.c_size_t,
],
llama_sampler_p_ctypes,
)
def llama_sampler_init_dry(
vocab: llama_vocab_p,
n_ctx_train: int,
dry_multiplier: float,
dry_base: float,
dry_allowed_length: int,
dry_penalty_last_n: int,
seq_breakers: CtypesArray[bytes],
num_breakers: int,
/,
) -> llama_sampler_p:
...
# /// adaptive-p: select tokens near a configurable target probability over time.
# ///
# /// the adaptive-p sampler transforms the token probability distribution to favor tokens
# /// that fall near a user-configurable probability target.
# ///
# /// internally, the sampler maintains an exponential moving average of the *ORIGINAL*
# /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an
# /// adapted target probability at each sampling step, thus maintaining the desired target
# /// probability over time.
# ///
# /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last
# /// in the sampler chain (like mirostat, dist, greedy).
# ///
# /// only mild truncation before this sampler is recommended. we suggest applying min-p
# /// before adaptive-p as the only other active sampler in the chain.
# ///
# /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled)
# /// @param decay EMA decay for adaptation; history 1/(1-decay) tokens (valid range 0.0 - 0.99)
# /// @param seed RNG seed
# ///
# /// ref: https://github.com/ggml-org/llama.cpp/pull/17927
# ///
# LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p(
# float target,
# float decay,
# uint32_t seed);
@ctypes_function(
"llama_sampler_init_adaptive_p",
[
ctypes.c_float,
ctypes.c_float,
ctypes.c_uint32,
],
llama_sampler_p_ctypes,
)
def llama_sampler_init_adaptive_p(
target: float,
decay: float,
seed: int,
/,
) -> llama_sampler_p:
...
# LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias(
# int32_t n_vocab,
# int32_t n_logit_bias,
# const llama_logit_bias * logit_bias);
@ctypes_function(
"llama_sampler_init_logit_bias",
[ctypes.c_int32, ctypes.c_int32, llama_logit_bias_p],
llama_sampler_p_ctypes,
)
def llama_sampler_init_logit_bias(
n_vocab: int, n_logit_bias: int, logit_bias: CtypesArray[llama_logit_bias], /
) -> llama_sampler_p:
...
# // this sampler is meant to be used for fill-in-the-middle infilling
# // it's supposed to be used after top_k + top_p sampling
# //
# // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG
# // 2. combine probs of tokens that have the same prefix
# //
# // example:
# //
# // - before:
# // "hel": 0.5
# // "hell": 0.2
# // "hello": 0.1
# // "dummy": 0.1
# //
# // - after:
# // "hel": 0.8
# // "dummy": 0.1
# //
# // 3. discard non-EOG tokens with low prob
# // 4. if no tokens are left -> pick EOT
# //
# LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab);
@ctypes_function(
"llama_sampler_init_infill",
[llama_vocab_p_ctypes],
llama_sampler_p_ctypes,
)
def llama_sampler_init_infill(vocab: llama_vocab_p, /) -> llama_sampler_p:
...
# // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise
# LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl);
@ctypes_function(
"llama_sampler_get_seed",
[llama_sampler_p_ctypes],
ctypes.c_uint32,
)
def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int:
...
# /// @details Sample and accept a token from the idx-th output of the last evaluation
# //
# // Shorthand for:
# // const auto * logits = llama_get_logits_ith(ctx, idx);
# // llama_token_data_array cur_p = { ... init from logits ... };
# // llama_sampler_apply(smpl, &cur_p);
# // auto token = cur_p.data[cur_p.selected].id;
# // llama_sampler_accept(smpl, token);
# // return token;
# // Returns the sampled token
# LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx);
@ctypes_function(
"llama_sampler_sample",
[llama_sampler_p_ctypes, llama_context_p_ctypes, ctypes.c_int32],
llama_token,
)
def llama_sampler_sample(
smpl: llama_sampler_p, ctx: llama_context_p, idx: ctypes.c_int32, /
) -> ctypes.c_int32:
...
# //
# // Model split
# //
# /// @details Build a split GGUF final path for this chunk.
# /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf"
# // Returns the split_path length.
# LLAMA_API int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count);
@ctypes_function(
"llama_split_path",
[ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int, ctypes.c_int],
ctypes.c_int,
)
def llama_split_path(
split_path: bytes,
maxlen: Union[ctypes.c_size_t, int],
path_prefix: bytes,
split_no: Union[ctypes.c_int, int],
split_count: Union[ctypes.c_int, int],
/,
) -> int:
"""Build a split GGUF final path for this chunk."""
...
# /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match.
# /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0"
# // Returns the split_prefix length.
# LLAMA_API int llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int split_no, int split_count);
@ctypes_function(
"llama_split_prefix",
[ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int, ctypes.c_int],
ctypes.c_int,
)
def llama_split_prefix(
split_prefix: bytes,
maxlen: Union[ctypes.c_size_t, int],
split_path: bytes,
split_no: Union[ctypes.c_int, int],
split_count: Union[ctypes.c_int, int],
/,
) -> int:
"""Extract the path prefix from the split_path if and only if the split_no and split_count match."""
...
# // Print system information
# LLAMA_API const char * llama_print_system_info(void);
@ctypes_function("llama_print_system_info", [], ctypes.c_char_p)
def llama_print_system_info() -> bytes:
...
# // Set callback for all future logging events.
# // If this is not called, or NULL is supplied, everything is output on stderr.
# // The logger state is global so these functions are NOT thread safe.
# LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data);
@ctypes_function(
"llama_log_get",
[ctypes.POINTER(ggml_log_callback), ctypes.POINTER(ctypes.c_void_p)],
None,
)
def llama_log_get(
log_callback: Optional[ctypes.pointer(ggml_log_callback)],
user_data: ctypes.pointer(ctypes.c_void_p),
/,
):
"""Get callback for all future logging events.
If this is not called, or NULL is supplied, everything is output on stderr."""
...
# LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data);
@ctypes_function(
"llama_log_set",
[ggml_log_callback, ctypes.c_void_p],
None,
)
def llama_log_set(
log_callback: Optional[ggml_log_callback],
user_data: ctypes.c_void_p,
/,
):
"""Set callback for all future logging events.
If this is not called, or NULL is supplied, everything is output on stderr."""
...
# //
# // Performance utils
# //
# // NOTE: Used by llama.cpp examples/tools, avoid using in third-party apps. Instead, do your own performance measurements.
# //
# struct llama_perf_context_data {
# // ms == milliseconds
# double t_start_ms; // absolute start time
# double t_load_ms; // time needed for loading the model
# double t_p_eval_ms; // time needed for processing the prompt
# double t_eval_ms; // time needed for generating tokens
# int32_t n_p_eval; // number of prompt tokens
# int32_t n_eval; // number of generated tokens
# int32_t n_reused; // number of times a ggml compute graph had been reused
# };
class llama_perf_context_data(ctypes.Structure):
_fields_ = [
("t_start_ms", ctypes.c_double),
("t_load_ms", ctypes.c_double),
("t_p_eval_ms", ctypes.c_double),
("t_eval_ms", ctypes.c_double),
("n_p_eval", ctypes.c_int32),
("n_eval", ctypes.c_int32),
("n_reused", ctypes.c_int32),
]
# struct llama_perf_sampler_data {
# double t_sample_ms; // time needed for sampling in ms
# int32_t n_sample; // number of sampled tokens
# };
class llama_perf_sampler_data(ctypes.Structure):
_fields_ = [
("t_sample_ms", ctypes.c_double),
("n_sample", ctypes.c_int32),
]
# LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx);
@ctypes_function(
"llama_perf_context",
[llama_context_p_ctypes],
llama_perf_context_data,
)
def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data:
...
# LLAMA_API void llama_perf_context_print(const struct llama_context * ctx);
@ctypes_function(
"llama_perf_context_print",
[llama_context_p_ctypes],
None,
)
def llama_perf_context_print(ctx: llama_context_p, /):
...
# LLAMA_API void llama_perf_context_reset( struct llama_context * ctx);
@ctypes_function(
"llama_perf_context_reset",
[llama_context_p_ctypes],
None,
)
def llama_perf_context_reset(ctx: llama_context_p, /):
...
# // NOTE: the following work only with samplers constructed via llama_sampler_chain_init
# LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain);
@ctypes_function(
"llama_perf_sampler",
[llama_sampler_p_ctypes],
llama_perf_sampler_data,
)
def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data:
...
# LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain);
@ctypes_function(
"llama_perf_sampler_print",
[llama_sampler_p_ctypes],
None,
)
def llama_perf_sampler_print(chain: llama_sampler_p, /):
...
# LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain);
@ctypes_function(
"llama_perf_sampler_reset",
[llama_sampler_p_ctypes],
None,
)
def llama_perf_sampler_reset(chain: llama_sampler_p, /):
...
# // print a breakdown of per-device memory use via LLAMA_LOG:
# LLAMA_API void llama_memory_breakdown_print(const struct llama_context * ctx);
@ctypes_function(
"llama_memory_breakdown_print",
[llama_context_p_ctypes],
None,
)
def llama_memory_breakdown_print(ctx: llama_context_p, /):
...
# //
# // training
# //
# // function that returns whether or not a given tensor contains trainable parameters
# typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata);
llama_opt_param_filter = ctypes.CFUNCTYPE(
ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p
)
# // always returns true
# LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata);
@ctypes_function("llama_opt_param_filter_all", [ctypes.c_void_p, ctypes.c_void_p], ctypes.c_bool)
def llama_opt_param_filter_all(
tensor: llama_model_p,
userdata: ctypes.c_void_p, /
) -> bool:
...
# struct llama_opt_params {
# uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0
# llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters
# void * param_filter_ud; // userdata for determining which tensors contain trainable parameters
# ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters
# void * get_opt_pars_ud; // userdata for calculating optimizer parameters
# };
class llama_opt_params(ctypes.Structure):
_fields_ = [
("n_ctx_train", ctypes.c_uint32),
("param_filter", llama_opt_param_filter),
("param_filter_ud", ctypes.c_void_p),
("get_opt_pars", ggml_opt_get_optimizer_params),
("get_opt_pars_ud", ctypes.c_void_p),
]
# LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params);
@ctypes_function(
"llama_opt_init",
[llama_context_p_ctypes, llama_model_p_ctypes, llama_opt_params_p_ctypes],
None,
)
def llama_opt_init(
lctx: llama_context_p,
model: llama_model_p,
lopt_params: llama_opt_params_p, /
):
...
# LLAMA_API void llama_opt_epoch(
# struct llama_context * lctx,
# ggml_opt_dataset_t dataset,
# ggml_opt_result_t result_train,
# ggml_opt_result_t result_eval,
# int64_t idata_split,
# ggml_opt_epoch_callback callback_train,
# ggml_opt_epoch_callback callback_eval);
@ctypes_function(
"llama_opt_epoch",[
llama_context_p_ctypes,
ctypes.c_void_p,
ctypes.c_void_p,
ctypes.c_void_p,
ctypes.c_int64,
ctypes.c_void_p,
ctypes.c_void_p
],
None,
)
def llama_opt_epoch(
lctx: llama_context_p,
dataset: ctypes.c_void_p,
result_train: ctypes.c_void_p,
result_eval: ctypes.c_void_p,
idata_split: ctypes.c_int64,
callback_train: ctypes.c_void_p,
callback_eval: ctypes.c_void_p, /
):
...