std::ostringstream oss;

oss << '[';

for (int i = ggml_n_dims(tensor) - 1; i >= 0; i--) {

oss << tensor->ne[i] << (i > 0 ? ", " : "");

}

oss << ']';

return oss.str();

std::ostringstream oss;

oss << '[';

for (int i = ggml_n_dims(tensor) - 1; i >= 0; i--) {

oss << tensor->nb[i] << (i > 0 ? ", " : "");

}

oss << ']';

return oss.str();

std::vector<char> buf(ggml_nbytes(tensor));

if (tensor->buffer) {

ggml_backend_tensor_get(tensor, buf.data(), 0, buf.size());

} else {

memcpy(buf.data(), tensor->data, buf.size());

}

std::vector<float> float_buf(ggml_nelements(tensor));

switch (tensor->type) {

case GGML_TYPE_F32:

memcpy(float_buf.data(), buf.data(), buf.size());

break;

case GGML_TYPE_F16:

ggml_fp16_to_fp32_row((ggml_fp16_t *)buf.data(), float_buf.data(), ggml_nelements(tensor));

break;

case GGML_TYPE_Q4_0:

dequantize_row_q4_0((block_q4_0 *)buf.data(), float_buf.data(), ggml_nelements(tensor));

break;

case GGML_TYPE_Q4_1:

dequantize_row_q4_1((block_q4_1 *)buf.data(), float_buf.data(), ggml_nelements(tensor));

break;

case GGML_TYPE_Q5_0:

dequantize_row_q5_0((block_q5_0 *)buf.data(), float_buf.data(), ggml_nelements(tensor));

break;

case GGML_TYPE_Q5_1:

dequantize_row_q5_1((block_q5_1 *)buf.data(), float_buf.data(), ggml_nelements(tensor));

break;

case GGML_TYPE_Q8_0:

dequantize_row_q8_0((block_q8_0 *)buf.data(), float_buf.data(), ggml_nelements(tensor));

break;

default:

CHATGLM_THROW << "Unsupported dtype " << tensor->type;

}

std::ostringstream oss;

oss << "ggml_tensor(";

if (with_data) {

const int n_dims = ggml_n_dims(tensor);

if (n_dims > 3)

oss << "[";

for (int i3 = 0; i3 < tensor->ne[3]; i3++) {

if (n_dims > 2)

oss << (i3 > 0 ? ",\n\n[" : "[");

for (int i2 = 0; i2 < tensor->ne[2]; i2++) {

if (n_dims > 1)

oss << (i2 > 0 ? ",\n\n[" : "[");

for (int i1 = 0; i1 < tensor->ne[1]; i1++) {

oss << (i1 > 0 ? ",\n[" : "[");

for (int i0 = 0; i0 < tensor->ne[0]; i0++) {

oss << (i0 > 0 ? ", " : "");

const int i = ((i3 * tensor->ne[2] + i2) * tensor->ne[1] + i1) * tensor->ne[0] + i0;

oss << std::setw(7) << std::fixed << std::setprecision(4) << float_buf[i];

}

oss << "]";

}

if (n_dims > 1)

oss << "]";

}

if (n_dims > 2)

oss << "]";

}

if (n_dims > 3)

oss << "]";

oss << ", ";

}

oss << "shape=" << shape_to_string(tensor) << ", stride=" << strides_to_string(tensor) << ")";

return oss.str();

std::string target_role = ChatMessage::ROLE_USER;

for (size_t i = 0; i < messages.size(); i++) {

if (messages[i].role != ChatMessage::ROLE_USER && messages[i].role != ChatMessage::ROLE_ASSISTANT) {

continue;

}

CHATGLM_CHECK(messages[i].role == target_role)

<< "expect messages[" << i << "].role to be " << target_role << ", but got " << messages[i].role;

target_role = (target_role == ChatMessage::ROLE_USER) ? ChatMessage::ROLE_ASSISTANT : ChatMessage::ROLE_USER;

}

CHATGLM_CHECK(target_role == ChatMessage::ROLE_ASSISTANT)

<< "expect last message role to be " << ChatMessage::ROLE_USER << ", but got " << ChatMessage::ROLE_ASSISTANT;

std::vector<ChatMessage> user_assistant_messages;

user_assistant_messages.reserve(messages.size());

for (const auto &msg : messages) {

if (msg.role == ChatMessage::ROLE_USER || msg.role == ChatMessage::ROLE_ASSISTANT) {

user_assistant_messages.emplace_back(msg);

}

}

return user_assistant_messages;

for (auto &streamer : streamers_) {

streamer->put(output_ids);

}

for (auto &streamer : streamers_) {

streamer->end();

}

if (is_prompt_) {

// skip prompt

is_prompt_ = false;

return;

}

static const std::vector<char> puncts{',', '!', ':', ';', '?'};

token_cache_.insert(token_cache_.end(), output_ids.begin(), output_ids.end());

std::string text = tokenizer_->decode(token_cache_);

if (is_first_line_) {

ltrim(text);

}

if (text.empty()) {

return;

}

std::string printable_text;

if (text.back() == '\n') {

// flush the cache after newline

printable_text = text.substr(print_len_);

is_first_line_ = false;

token_cache_.clear();

print_len_ = 0;

} else if (std::find(puncts.begin(), puncts.end(), text.back()) != puncts.end()) {

// last symbol is a punctuation, hold on

} else if (text.size() >= 3 && text.compare(text.size() - 3, 3, "�") == 0) {

// ends with an incomplete token, hold on

} else {

printable_text = text.substr(print_len_);

print_len_ = text.size();

}

os_ << printable_text << std::flush;

std::string text = tokenizer_->decode(token_cache_);

if (is_first_line_) {

ltrim(text);

}

os_ << text.substr(print_len_) << std::endl;

is_prompt_ = true;

is_first_line_ = true;

token_cache_.clear();

print_len_ = 0;

CHATGLM_CHECK(!output_ids.empty());

if (num_prompt_tokens_ == 0) {

// before prompt eval

start_us_ = ggml_time_us();

num_prompt_tokens_ = output_ids.size();

} else {

if (num_output_tokens_ == 0) {

// first new token

prompt_us_ = ggml_time_us();

}

num_output_tokens_ += output_ids.size();

}

std::ostringstream oss;

oss << "prompt time: " << prompt_total_time_us() / 1000.f << " ms / " << num_prompt_tokens() << " tokens ("

<< prompt_token_time_us() / 1000.f << " ms/token)\n"

<< "output time: " << output_total_time_us() / 1000.f << " ms / " << num_output_tokens() << " tokens ("

<< output_token_time_us() / 1000.f << " ms/token)\n"

<< "total time: " << (prompt_total_time_us() + output_total_time_us()) / 1000.f << " ms";

return oss.str();

int fd = open(path.c_str(), O_RDONLY);

CHATGLM_CHECK(fd > 0) << "cannot open file " << path << ": " << strerror(errno);

struct stat sb;

CHATGLM_CHECK(fstat(fd, &sb) == 0) << strerror(errno);

size = sb.st_size;

data = (char *)mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0);

CHATGLM_CHECK(data != MAP_FAILED) << strerror(errno);

CHATGLM_CHECK(close(fd) == 0) << strerror(errno);

int fd = open(path.c_str(), O_RDONLY);

CHATGLM_CHECK(fd > 0) << "cannot open file " << path << ": " << strerror(errno);

struct _stat64 sb;

CHATGLM_CHECK(_fstat64(fd, &sb) == 0) << strerror(errno);

size = sb.st_size;

HANDLE hFile = (HANDLE)_get_osfhandle(fd);

HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);

CHATGLM_CHECK(hMapping != NULL) << strerror(errno);

data = (char *)MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);

CloseHandle(hMapping);

CHATGLM_CHECK(data != NULL) << strerror(errno);

CHATGLM_CHECK(close(fd) == 0) << strerror(errno);

if (whence == SEEK_SET) {

ptr = data + offset;

} else if (whence == SEEK_CUR) {

ptr += offset;

} else if (whence == SEEK_END) {

ptr = data + size + offset;

} else {

CHATGLM_THROW << "invalid seek mode " << whence;

}

std::string s(ptr, ptr + length);

ptr += length;

return s;

StateDict sd;

sd.ctx = make_unique_ggml_context(GGML_DEFAULT_GRAPH_SIZE * ggml_tensor_overhead(), nullptr, true);

sd.buf = unique_ggml_backend_buffer_t(ggml_backend_cpu_buffer_from_ptr(data, size));

// assume state dict is stored at the back of file

while (tell() < (int64_t)size) {

// tensor name

int name_size = read_basic<int>();

std::string weight_name = read_string(name_size);

// tensor shape

int64_t ne[4]{1, 1, 1, 1};

int ndim = read_basic<int>();

CHATGLM_CHECK(0 < ndim && ndim <= 4);

for (int i = ndim - 1; i >= 0; i--) {

ne[i] = read_basic<int>();

}

// tensor dtype

ggml_type dtype = (ggml_type)read_basic<int>();

// tensor data

ggml_tensor *tensor = ggml_new_tensor(sd.ctx.get(), dtype, ndim, ne);

constexpr int64_t MEM_ALIGNED = 16;

const int64_t data_offset = (tell() + (MEM_ALIGNED - 1)) & ~(MEM_ALIGNED - 1);

ggml_backend_tensor_alloc(sd.buf.get(), tensor, data + data_offset);

// tensor->data = data + data_offset;

seek(data_offset + ggml_nbytes(tensor), SEEK_SET);

// add to state dict

sd.kv.emplace(weight_name, tensor);

}

return sd;

int width, height, channels;

uint8_t *buffer = stbi_load(path.c_str(), &width, &height, &channels, 3);

CHATGLM_CHECK(channels == 3);

Image image(width, height, channels, buffer);

stbi_image_free(buffer);

return image;

Image output(new_width, new_height, channels);

stbir_resize_uint8_srgb(pixels.data(), width, height, 0, output.pixels.data(), new_width, new_height, 0, STBIR_RGB);

return output;

: compute_meta(ggml_tensor_overhead() * (GGML_DEFAULT_GRAPH_SIZE * 4) + ggml_graph_overhead()),

ctx_w(make_unique_ggml_context(ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE, nullptr, true)),

ctx_kv(make_unique_ggml_context(ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE, nullptr, true)),

ctx_b(make_unique_ggml_context(compute_meta.size(), compute_meta.data(), true)),

gf(ggml_new_graph_custom(ctx_b.get(), GGML_DEFAULT_GRAPH_SIZE * 4, false)) {

#if defined(GGML_USE_CUDA)

backend = unique_ggml_backend_t(ggml_backend_cuda_init(0));

#elif defined(GGML_USE_METAL)

backend = unique_ggml_backend_t(ggml_backend_metal_init());

#else

backend = unique_ggml_backend_t(ggml_backend_cpu_init());

#endif

CHATGLM_CHECK(backend) << "failed to initialize ggml backend";

allocr = unique_ggml_gallocr_t(ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend.get())));

int n_threads = 1;

if (ggml_backend_is_cpu(backend)) {

if (num_tokens > 1) {

// context

n_threads = get_num_physical_cores();

} else {

// decode

n_threads = std::min(get_num_physical_cores(), 16);

}

}

if (num_tokens >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) {

// BLAS is enabled

n_threads = std::min(4, n_threads);

}

if (ggml_backend_is_cpu(backend)) {

ggml_backend_cpu_set_n_threads(backend, n_threads);

}

#ifdef GGML_USE_METAL

if (ggml_backend_is_metal(backend)) {

ggml_backend_metal_set_n_cb(backend, n_threads);

}

#endif

static const std::unordered_map<ModelType, std::string> m{{ModelType::CHATGLM, "ChatGLM"},

{ModelType::CHATGLM2, "ChatGLM2"},

{ModelType::CHATGLM3, "ChatGLM3"},

{ModelType::CHATGLM4, "ChatGLM4"},

{ModelType::CHATGLM4V, "ChatGLM4V"}};

return m.at(model_type);

const GenerationConfig &gen_config, int n_past, int n_ctx) {

ggml_tensor *lm_logits = forward_graph_compute(input_ids, image, n_past, n_ctx, true);

CHATGLM_CHECK(ggml_n_dims(lm_logits) == 1);

int vocab_size = lm_logits->ne[0];

std::vector<float> next_token_logits(vocab_size);

ggml_backend_tensor_get(lm_logits, next_token_logits.data(), 0, vocab_size * sizeof(float));

// check nan

for (int i = 0; i < vocab_size; i++) {

CHATGLM_CHECK(std::isfinite(next_token_logits[i])) << "nan/inf encountered at lm_logits[" << i << "]";

}

// logits pre-process

if (gen_config.repetition_penalty != 1.f) {

sampling_repetition_penalty(next_token_logits.data(), next_token_logits.data() + vocab_size, input_ids,

gen_config.repetition_penalty);

}

int next_token_id;

if (gen_config.do_sample) {

// temperature sampling

if (gen_config.temperature > 0) {

sampling_temperature(next_token_logits.data(), next_token_logits.data() + vocab_size,

gen_config.temperature);

}

std::vector<TokenIdScore> token_scores(vocab_size);

for (int i = 0; i < vocab_size; i++) {

token_scores[i] = TokenIdScore(i, next_token_logits[i]);

}

// top_k sampling

if (0 < gen_config.top_k && gen_config.top_k < (int)token_scores.size()) {

sampling_top_k(token_scores.data(), token_scores.data() + gen_config.top_k,

token_scores.data() + token_scores.size());

token_scores.resize(gen_config.top_k);

}

// top_p sampling

if (0.f < gen_config.top_p && gen_config.top_p < 1.f) {

auto pos = sampling_top_p(token_scores.data(), token_scores.data() + token_scores.size(), gen_config.top_p);

token_scores.resize(pos - token_scores.data());

}

// sample next token

sampling_softmax_inplace(token_scores.data(), token_scores.data() + token_scores.size());

for (size_t i = 0; i < token_scores.size(); i++) {

next_token_logits[i] = token_scores[i].score;

}

thread_local std::random_device rd;

thread_local std::mt19937 gen(rd());

std::discrete_distribution<> dist(next_token_logits.data(), next_token_logits.data() + token_scores.size());

next_token_id = token_scores[dist(gen)].id;

} else {

// greedy search

next_token_id =

std::max_element(next_token_logits.begin(), next_token_logits.end()) - next_token_logits.begin();

}

return next_token_id;

float penalty) {

CHATGLM_CHECK(penalty > 0) << "penalty must be a positive float, but got " << penalty;

const float inv_penalty = 1.f / penalty;

const int vocab_size = last - first;

std::vector<bool> occurrence(vocab_size, false);

for (const int id : input_ids) {

if (!occurrence[id]) {

first[id] *= (first[id] > 0) ? inv_penalty : penalty;

}

occurrence[id] = true;

}

const float inv_temp = 1.f / temp;

for (float *it = first; it != last; it++) {

*it *= inv_temp;

}

float max_score = std::max_element(first, last)->score;

float sum = 0.f;

for (TokenIdScore *p = first; p != last; p++) {

float s = std::exp(p->score - max_score);

p->score = s;

sum += s;

}

float inv_sum = 1.f / sum;

for (TokenIdScore *p = first; p != last; p++) {

p->score *= inv_sum;

}

const GenerationConfig &gen_config, BaseStreamer *streamer) {

CHATGLM_CHECK(gen_config.max_length <= config.max_length)

<< "Requested max_length (" << gen_config.max_length << ") exceeds pre-configured model max_length ("

<< config.max_length << ")";

std::vector<int> output_ids;

output_ids.reserve(gen_config.max_length);

output_ids = input_ids;

if (streamer) {

streamer->put(input_ids);

}

int n_past = 0;

const int n_ctx = input_ids.size();

const int max_new_tokens = (gen_config.max_new_tokens > 0) ? gen_config.max_new_tokens : gen_config.max_length;

while ((int)output_ids.size() < std::min(gen_config.max_length, n_ctx + max_new_tokens)) {

int next_token_id = generate_next_token(output_ids, image, gen_config, n_past, n_ctx);

n_past = count_tokens(output_ids, image);

output_ids.emplace_back(next_token_id);

if (streamer) {

streamer->put({next_token_id});

}

if (next_token_id == config.eos_token_id ||

std::find(config.extra_eos_token_ids.begin(), config.extra_eos_token_ids.end(), next_token_id) !=

config.extra_eos_token_ids.end()) {

break;

}

}

if (streamer) {

streamer->end();

}

return output_ids;

const auto status = sp.LoadFromSerializedProto(serialized_model_proto);

CHATGLM_CHECK(status.ok()) << status.ToString();

bos_token_id = sp.PieceToId("<sop>");

eos_token_id = sp.PieceToId("<eop>");

mask_token_id = sp.PieceToId("[MASK]");

gmask_token_id = sp.PieceToId("[gMASK]");

pad_token_id = sp.PieceToId("<pad>");

std::string input = preprocess(text);

std::vector<int> ids;

sp.Encode(input, &ids);

ids.insert(ids.end(), {gmask_token_id, bos_token_id});

if ((int)ids.size() > max_length) {

// sliding window: always take the last max_length tokens

ids.erase(ids.begin(), ids.end() - max_length);

}

return ids;

std::string prompt = apply_chat_template_text(messages);

std::vector<int> input_ids = encode(prompt, max_length);

return input_ids;

check_chat_messages(messages);

std::vector<ChatMessage> user_assistant_messages = filter_user_assistant_messages(messages);

std::ostringstream oss_prompt;

if (user_assistant_messages.size() == 1) {

oss_prompt << user_assistant_messages.front().content;

} else {

for (size_t i = 0; i < user_assistant_messages.size(); i += 2) {