description="The path to the model to use for generating completions."
)
model_alias: Optional[str] =Field(
default=None,
description="The alias of the model to use for generating completions.",
)
# Model Params
n_gpu_layers: int=Field(
default=0,
ge=-1,
description="The number of layers to put on the GPU. The rest will be on the CPU. Set -1 to move all to GPU.",
)
split_mode: int=Field(
default=llama_cpp.LLAMA_SPLIT_MODE_LAYER,
description="The split mode to use.",
)
main_gpu: int=Field(
default=0,
ge=0,
description="Main GPU to use.",
)
tensor_split: Optional[List[float]] =Field(
default=None,
description="Split layers across multiple GPUs in proportion.",
)
vocab_only: bool=Field(
default=False, description="Whether to only return the vocabulary."
)
use_mmap: bool=Field(
default=llama_cpp.llama_supports_mmap(),
description="Use mmap.",
)
use_mlock: bool=Field(
default=llama_cpp.llama_supports_mlock(),
description="Use mlock.",
)
kv_overrides: Optional[List[str]] =Field(
default=None,
description="List of model kv overrides in the format key=type:value where type is one of (bool, int, float). Valid true values are (true, TRUE, 1), otherwise false.",
)
# Context Params
seed: int=Field(
default=llama_cpp.LLAMA_DEFAULT_SEED, description="Random seed. -1 for random."