for (size_t i = 0; i < sizeof(unused_tensors) / sizeof(const char*); i++) {

if (starts_with(name, unused_tensors[i])) {

return true;

}

}

return false;

// do we need to support uz?

const uint32_t exponent_bias = 7;

if (f8 == 0xff) {

return ggml_fp32_to_fp16(-NAN);

} else if (f8 == 0x7f) {

return ggml_fp32_to_fp16(NAN);

}

uint32_t sign = f8 & 0x80;

uint32_t exponent = (f8 & 0x78) >> 3;

uint32_t mantissa = f8 & 0x07;

uint32_t result = sign << 24;

if (exponent == 0) {

if (mantissa > 0) {

exponent = 0x7f - exponent_bias;

// yes, 2 times

if ((mantissa & 0x04) == 0) {

mantissa &= 0x03;

mantissa <<= 1;

exponent -= 1;

}

if ((mantissa & 0x04) == 0) {

mantissa &= 0x03;

mantissa <<= 1;

exponent -= 1;

}

result |= (mantissa & 0x03) << 21;

result |= exponent << 23;

}

} else {

result |= mantissa << 20;

exponent += 0x7f - exponent_bias;

result |= exponent << 23;

}

return ggml_fp32_to_fp16(*reinterpret_cast<const float*>(&result));

// support inplace op

for (int64_t i = n - 1; i >= 0; i--) {

dst[i] = f8_e4m3_to_f16(src[i]);

}

// support inplace op

for (int64_t i = n - 1; i >= 0; i--) {

dst[i] = f8_e5m2_to_f16(src[i]);

}

// support inplace op

for (int64_t i = 0; i < n; i++) {

dst[i] = (float)src[i];

}

// support inplace op

for (int64_t i = 0; i < n; i++) {

dst[i] = (int32_t)src[i];

}

ggml_type src_type,

void* dst,

ggml_type dst_type,

int nrows,

int n_per_row,

std::vector<float> imatrix = {}) {

int n = nrows * n_per_row;

if (src_type == dst_type) {

size_t nbytes = n * ggml_type_size(src_type) / ggml_blck_size(src_type);

memcpy(((char*)dst), ((char*)src), nbytes);

} else if (src_type == GGML_TYPE_F32) {

if (dst_type == GGML_TYPE_F16) {

ggml_fp32_to_fp16_row((float*)src, (ggml_fp16_t*)dst, n);

} else {

imatrix.resize(n_per_row, 1.0f);

const float* im = imatrix.data();

ggml_quantize_chunk(dst_type, (float*)src, dst, 0, nrows, n_per_row, im);

}

} else if (dst_type == GGML_TYPE_F32) {

if (src_type == GGML_TYPE_F16) {

ggml_fp16_to_fp32_row((ggml_fp16_t*)src, (float*)dst, n);

} else {

auto qtype = ggml_get_type_traits(src_type);

if (qtype->to_float == nullptr) {

throw std::runtime_error(sd_format("type %s unsupported for integer quantization: no dequantization available",

ggml_type_name(src_type)));

}

qtype->to_float(src, (float*)dst, n);

}

} else {

// src_type == GGML_TYPE_F16 => dst_type is quantized

// src_type is quantized => dst_type == GGML_TYPE_F16 or dst_type is quantized

auto qtype = ggml_get_type_traits(src_type);

if (qtype->to_float == nullptr) {

throw std::runtime_error(sd_format("type %s unsupported for integer quantization: no dequantization available",

ggml_type_name(src_type)));

}

std::vector<char> buf;

buf.resize(sizeof(float) * n);

char* src_data_f32 = buf.data();

qtype->to_float(src, (float*)src_data_f32, n);

if (dst_type == GGML_TYPE_F16) {

ggml_fp32_to_fp16_row((float*)src_data_f32, (ggml_fp16_t*)dst, n);

} else {

imatrix.resize(n_per_row, 1.0f);

const float* im = imatrix.data();

ggml_quantize_chunk(dst_type, (float*)src_data_f32, dst, 0, nrows, n_per_row, im);

}

}

if (model_files_processed) {

file_data.clear();

model_files_processed = false;

}

file_paths_.push_back(file_path);

return file_paths_.size() - 1;

if (is_directory(file_path)) {

LOG_INFO("load %s using diffusers format", file_path.c_str());

return init_from_diffusers_file(file_path, prefix);

} else if (is_gguf_file(file_path)) {

LOG_INFO("load %s using gguf format", file_path.c_str());

return init_from_gguf_file(file_path, prefix);

} else if (ends_with(file_path, ".json")) {

LOG_INFO("load %s using safetensors index format", file_path.c_str());

return init_from_safetensors_index_file(file_path, prefix);

} else if (is_safetensors_file(file_path)) {

LOG_INFO("load %s using safetensors format", file_path.c_str());

return init_from_safetensors_file(file_path, prefix);

} else if (is_torch_zip_file(file_path)) {

LOG_INFO("load %s using torch zip format", file_path.c_str());

return init_from_torch_zip_file(file_path, prefix);

} else if (init_from_torch_legacy_file(file_path, prefix)) {

LOG_INFO("load %s using torch legacy format", file_path.c_str());

return true;

} else {

if (file_exists(file_path)) {

LOG_WARN("unknown format %s", file_path.c_str());

} else {

LOG_WARN("file %s not found", file_path.c_str());

}

return false;

}

SDVersion version = (version_ == VERSION_COUNT) ? get_sd_version() : version_;

String2TensorStorage new_map;

for (auto& [_, tensor_storage] : tensor_storage_map) {

auto new_name = convert_tensor_name(tensor_storage.name, version);

// LOG_DEBUG("%s -> %s", tensor_storage.name.c_str(), new_name.c_str());

tensor_storage.name = new_name;

new_map[new_name] = std::move(tensor_storage);

}

tensor_storage_map.swap(new_map);

if (version_ == VERSION_COUNT && version != VERSION_COUNT) {

version_ = version;

}

if (!init_from_file(file_path, prefix)) {

return false;

}

convert_tensors_name();

return true;

LOG_DEBUG("init from '%s'", file_path.c_str());

std::vector<TensorStorage> tensor_storages;

std::string error;

if (!read_gguf_file(file_path, tensor_storages, &error)) {

LOG_ERROR("%s", error.c_str());

return false;

}

size_t file_index = add_file_path(file_path);

for (auto& tensor_storage : tensor_storages) {

// LOG_DEBUG("%s", tensor_storage.name.c_str());

if (!starts_with(tensor_storage.name, prefix)) {

tensor_storage.name = prefix + tensor_storage.name;

}

tensor_storage.file_index = file_index;

add_tensor_storage(tensor_storage);

}

return true;

LOG_DEBUG("init from '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());

std::vector<TensorStorage> tensor_storages;

std::string error;

if (!read_safetensors_file(file_path, tensor_storages, &error, &metadata_)) {

LOG_ERROR("%s", error.c_str());

return false;

}

size_t file_index = add_file_path(file_path);

for (auto& tensor_storage : tensor_storages) {

if (is_unused_tensor(tensor_storage.name)) {

continue;

}

if (!starts_with(tensor_storage.name, prefix)) {

tensor_storage.name = prefix + tensor_storage.name;

}

tensor_storage.file_index = file_index;

add_tensor_storage(tensor_storage);

// LOG_DEBUG("%s", tensor_storage.to_string().c_str());

}

return true;

LOG_DEBUG("init from safetensors index '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());

std::vector<std::string> shard_paths;

std::string error;

if (!read_safetensors_index_file(file_path, shard_paths, &error)) {

LOG_ERROR("%s", error.c_str());

return false;

}

for (const std::string& shard_path : shard_paths) {

if (!init_from_file(shard_path, prefix)) {

return false;

}

}

return true;

LOG_DEBUG("init from torch legacy '%s'", file_path.c_str());

std::vector<TensorStorage> tensor_storages;

std::string error;

if (!read_torch_legacy_file(file_path, tensor_storages, &error)) {

if ((!error.empty()) && (ends_with(file_path, ".pt") || ends_with(file_path, ".pth"))) {

LOG_WARN("%s", error.c_str());

}

return false;

}

size_t file_index = add_file_path(file_path);

for (auto& tensor_storage : tensor_storages) {

if (is_unused_tensor(tensor_storage.name)) {

continue;

}

if (!starts_with(tensor_storage.name, prefix)) {

tensor_storage.name = prefix + tensor_storage.name;

}

tensor_storage.file_index = file_index;

add_tensor_storage(tensor_storage);

}

return true;

LOG_DEBUG("init from '%s'", file_path.c_str());

std::vector<TensorStorage> tensor_storages;

std::string error;

if (!read_torch_zip_file(file_path, tensor_storages, &error)) {

LOG_ERROR("%s", error.c_str());

return false;

}

size_t file_index = add_file_path(file_path);

for (auto& tensor_storage : tensor_storages) {

if (!starts_with(tensor_storage.name, prefix)) {

tensor_storage.name = prefix + tensor_storage.name;

}

tensor_storage.file_index = file_index;

add_tensor_storage(tensor_storage);

// LOG_DEBUG("%s", tensor_storage.to_string().c_str());

}

return true;

std::string unet_path = path_join(file_path, "unet/diffusion_pytorch_model.safetensors");

std::string vae_path = path_join(file_path, "vae/diffusion_pytorch_model.safetensors");

std::string clip_path = path_join(file_path, "text_encoder/model.safetensors");

std::string clip_g_path = path_join(file_path, "text_encoder_2/model.safetensors");

if (!init_from_safetensors_file(unet_path, "unet.")) {

return false;

}

if (!init_from_safetensors_file(vae_path, "vae.")) {

LOG_WARN("Couldn't find working VAE in %s", file_path.c_str());

// return false;

}

if (!init_from_safetensors_file(clip_path, "te.")) {

LOG_WARN("Couldn't find working text encoder in %s", file_path.c_str());

// return false;

}

if (!init_from_safetensors_file(clip_g_path, "te.1.")) {

LOG_DEBUG("Couldn't find working second text encoder in %s", file_path.c_str());

}

return true;

TensorStorage token_embedding_weight, input_block_weight, context_ebedding_weight;

bool has_multiple_encoders = false;

bool is_unet = false;

bool is_xl = false;

bool is_flux = false;

bool is_flux2 = false;

bool has_single_block_47 = false;

bool is_wan = false;

int64_t patch_embedding_channels = 0;

bool has_img_emb = false;

bool has_middle_block_1 = false;

bool has_output_block_311 = false;

bool has_output_block_71 = false;

bool has_attn_1024 = false;

for (auto& [name, tensor_storage] : tensor_storage_map) {

if (tensor_storage.name.find("model.diffusion_model.double_blocks.") != std::string::npos ||

tensor_storage.name.find("model.diffusion_model.single_transformer_blocks.") != std::string::npos) {

is_flux = true;

}

if (tensor_storage.name.find("model.diffusion_model.net.lq_proj.latent_proj.0.weight") != std::string::npos) {

return VERSION_PID;

}

if (tensor_storage.name.find("embed_image_indicator.weight") != std::string::npos) {

return VERSION_IDEOGRAM4;

}

if (tensor_storage.name.find("model.diffusion_model.txtfusion.projector.weight") != std::string::npos ||

tensor_storage.name.find("model.diffusion_model.text_fusion.projector.weight") != std::string::npos) {

return VERSION_KREA2;

}

if (tensor_storage.name.find("model.diffusion_model.nerf_final_layer_conv.") != std::string::npos) {

return VERSION_CHROMA_RADIANCE;

}

if (tensor_storage.name.find("model.diffusion_model.joint_blocks.") != std::string::npos) {

return VERSION_SD3;

}

if (tensor_storage.name.find("model.x_embedder.proj1.weight") != std::string::npos &&

tensor_storage_map.find("model.language_model.layers.0.self_attn.q_proj.weight") != tensor_storage_map.end()) {

return VERSION_HIDREAM_O1;

}

if (tensor_storage.name.find("model.diffusion_model.transformer_blocks.0.attn.norm_added_q.weight") != std::string::npos &&

tensor_storage_map.find("model.diffusion_model.transformer_blocks.0.img_mlp.w1.weight") != tensor_storage_map.end()) {

return VERSION_LENS;

}

if (tensor_storage.name.find("net.img_embedder.proj1.weight") != std::string::npos) {

return VERSION_MINIT2I;

}

if (tensor_storage.name.find("model.diffusion_model.transformer_blocks.0.img_mod.1.weight") != std::string::npos) {

auto img_in = tensor_storage_map.find("model.diffusion_model.img_in.weight");

if (img_in != tensor_storage_map.end() && img_in->second.ne[0] == 128) {

return VERSION_MAGE_FLOW;

}

if (tensor_storage_map.find("model.diffusion_model.time_text_embed.addition_t_embedding.weight") != tensor_storage_map.end()) {

return VERSION_QWEN_IMAGE_LAYERED;

}

return VERSION_QWEN_IMAGE;

}

if (tensor_storage.name.find("model.diffusion_model.txt_in.individual_token_refiner.blocks.0.adaLN_modulation.1.weight") != std::string::npos) {

return VERSION_HUNYUAN_VIDEO;

}

if (tensor_storage.name.find("llm_adapter.blocks.0.cross_attn.q_proj.weight") != std::string::npos) {

return VERSION_ANIMA;

}

if (tensor_storage.name.find("model.diffusion_model.double_stream_modulation_img.lin.weight") != std::string::npos) {

is_flux2 = true;

}

if (tensor_storage.name.find("dual_time_embed.semantic_embedder.linear_1.weight") != std::string::npos) {

return VERSION_SEFI_IMAGE;

}

if (tensor_storage.name.find("single_blocks.47.linear1.weight") != std::string::npos) {

has_single_block_47 = true;

}

if (tensor_storage.name.find("model.diffusion_model.double_blocks.0.img_mlp.gate_proj.weight") != std::string::npos) {

return VERSION_OVIS_IMAGE;

}

if (tensor_storage.name.find("model.diffusion_model.cap_embedder.0.weight") != std::string::npos) {

return VERSION_Z_IMAGE;

}

if (tensor_storage.name.find("double_stream_layers.0.img_instruct_attn.processor.img_to_q.weight") != std::string::npos) {

return VERSION_BOOGU_IMAGE;

}

if (tensor_storage.name.find("model.diffusion_model.layers.0.adaLN_sa_ln.weight") != std::string::npos) {

return VERSION_ERNIE_IMAGE;

}

if (tensor_storage.name.find("model.diffusion_model.adaln_single.emb.timestep_embedder.linear_1.bias") != std::string::npos) {

return VERSION_LTXAV;

}

if (tensor_storage.name.find("model.diffusion_model.video_patch_proj.weight") != std::string::npos &&

tensor_storage_map.find("model.diffusion_model.audio_patch_proj.weight") != tensor_storage_map.end()) {

return VERSION_MINIMAX_H3;

}

if (tensor_storage.name.find("model.diffusion_model.blocks.0.cross_attn.norm_k.weight") != std::string::npos) {

is_wan = true;

}

if (tensor_storage.name.find("model.diffusion_model.patch_embedder.weight") != std::string::npos) {

return VERSION_LINGBOT_VIDEO;

}

if (tensor_storage.name.find("model.diffusion_model.patch_embedding.weight") != std::string::npos) {

patch_embedding_channels = tensor_storage.ne[3];

}

if (tensor_storage.name.find("model.diffusion_model.img_emb") != std::string::npos) {

has_img_emb = true;

}

if (tensor_storage.name.find("model.diffusion_model.input_blocks.") != std::string::npos ||

tensor_storage.name.find("unet.down_blocks.") != std::string::npos) {

is_unet = true;

if (has_multiple_encoders) {

is_xl = true;

}

}

if (tensor_storage.name.find("conditioner.embedders.1") != std::string::npos ||

tensor_storage.name.find("cond_stage_model.1") != std::string::npos ||

tensor_storage.name.find("te.1") != std::string::npos) {

has_multiple_encoders = true;

if (is_unet) {

is_xl = true;

}

}

if (tensor_storage.name.find("model.diffusion_model.input_blocks.8.0.time_mixer.mix_factor") != std::string::npos) {

return VERSION_SVD;

}

if (tensor_storage.name.find("model.diffusion_model.middle_block.1.") != std::string::npos ||

tensor_storage.name.find("unet.mid_block.resnets.1.") != std::string::npos) {

has_middle_block_1 = true;

}

if (tensor_storage.name.find("model.diffusion_model.output_blocks.3.1.transformer_blocks.1") != std::string::npos ||

tensor_storage.name.find("unet.up_blocks.1.attentions.0.transformer_blocks.1") != std::string::npos) {

has_output_block_311 = true;

}

if (tensor_storage.name.find("model.diffusion_model.output_blocks.7.1") != std::string::npos ||

tensor_storage.name.find("unet.up_blocks.2.attentions.1") != std::string::npos) {

has_output_block_71 = true;

if (tensor_storage.name.find("model.diffusion_model.output_blocks.7.1.transformer_blocks.0.attn1.to_k.weight") != std::string::npos) {

if (tensor_storage.ne[0] == 1024)

has_attn_1024 = true;

}

}

if (tensor_storage.name == "cond_stage_model.transformer.text_model.embeddings.token_embedding.weight" ||

tensor_storage.name == "cond_stage_model.model.token_embedding.weight" ||

tensor_storage.name == "text_model.embeddings.token_embedding.weight" ||

tensor_storage.name == "te.text_model.embeddings.token_embedding.weight" ||

tensor_storage.name == "conditioner.embedders.0.model.token_embedding.weight" ||

tensor_storage.name == "conditioner.embedders.0.transformer.text_model.embeddings.token_embedding.weight") {

token_embedding_weight = tensor_storage;

// break;

}

if (tensor_storage.name == "model.diffusion_model.input_blocks.0.0.weight" ||

tensor_storage.name == "model.diffusion_model.img_in.weight" ||

tensor_storage.name == "unet.conv_in.weight") {

input_block_weight = tensor_storage;

}

if (tensor_storage.name == "model.diffusion_model.txt_in.weight" || tensor_storage.name == "model.diffusion_model.context_embedder.weight") {

context_ebedding_weight = tensor_storage;

}

}

if (is_wan) {

LOG_DEBUG("patch_embedding_channels %d", patch_embedding_channels);

if (patch_embedding_channels == 184320 && !has_img_emb) {

return VERSION_WAN2_2_I2V;

}

if (patch_embedding_channels == 147456 && !has_img_emb) {

return VERSION_WAN2_2_TI2V;

}

return VERSION_WAN2;

}

bool is_inpaint = input_block_weight.ne[2] == 9;

bool is_ip2p = input_block_weight.ne[2] == 8;

if (is_xl) {

if (is_inpaint) {

return VERSION_SDXL_INPAINT;

}

if (is_ip2p) {

return VERSION_SDXL_PIX2PIX;

}

if (!has_middle_block_1) {

if (!has_output_block_311) {

return VERSION_SDXL_VEGA;

}

return VERSION_SDXL_SSD1B;

}

return VERSION_SDXL;

}

if (is_flux && !is_flux2) {

if (context_ebedding_weight.ne[0] == 3584) {

return VERSION_LONGCAT;

} else {

if (input_block_weight.ne[0] == 384) {

return VERSION_FLUX_FILL;

}

if (input_block_weight.ne[0] == 128) {

return VERSION_FLUX_CONTROLS;

}

if (input_block_weight.ne[0] == 196) {

return VERSION_FLEX_2;

}

return VERSION_FLUX;

}

}

if (is_flux2) {

if (has_single_block_47) {

return VERSION_FLUX2;

}

return VERSION_FLUX2_KLEIN;

}

if (token_embedding_weight.ne[0] == 768) {

if (is_inpaint) {

return VERSION_SD1_INPAINT;

}

if (is_ip2p) {

return VERSION_SD1_PIX2PIX;

}

if (!has_middle_block_1) {

if (!has_output_block_71) {

return VERSION_SDXS_512_DS;

}

return VERSION_SD1_TINY_UNET;

}

return VERSION_SD1;

} else if (token_embedding_weight.ne[0] == 1024) {

if (is_inpaint) {

return VERSION_SD2_INPAINT;

}

if (!has_middle_block_1) {

return has_attn_1024 ? VERSION_SDXS_09 : VERSION_SD2_TINY_UNET;

}

return VERSION_SD2;

}

return VERSION_COUNT;

std::map<ggml_type, uint32_t> wtype_stat;

for (auto& [name, tensor_storage] : tensor_storage_map) {

if (is_unused_tensor(tensor_storage.name)) {

continue;

}

auto iter = wtype_stat.find(tensor_storage.type);

if (iter != wtype_stat.end()) {

iter->second++;

} else {

wtype_stat[tensor_storage.type] = 1;

}

}

return wtype_stat;

std::map<ggml_type, uint32_t> wtype_stat;

for (auto& [name, tensor_storage] : tensor_storage_map) {

if (is_unused_tensor(tensor_storage.name)) {

continue;

}

if ((tensor_storage.name.find("text_encoders") == std::string::npos &&

tensor_storage.name.find("cond_stage_model") == std::string::npos &&

tensor_storage.name.find("te.text_model.") == std::string::npos &&

tensor_storage.name.find("conditioner") == std::string::npos)) {

continue;

}

auto iter = wtype_stat.find(tensor_storage.type);

if (iter != wtype_stat.end()) {

iter->second++;

} else {

wtype_stat[tensor_storage.type] = 1;

}

}

return wtype_stat;

std::map<ggml_type, uint32_t> wtype_stat;

for (auto& [name, tensor_storage] : tensor_storage_map) {

if (is_unused_tensor(tensor_storage.name)) {

continue;

}

if (tensor_storage.name.find("model.diffusion_model.") == std::string::npos && tensor_storage.name.find("unet.") == std::string::npos) {

continue;

}

auto iter = wtype_stat.find(tensor_storage.type);

if (iter != wtype_stat.end()) {

iter->second++;

} else {

wtype_stat[tensor_storage.type] = 1;

}

}

return wtype_stat;

std::map<ggml_type, uint32_t> wtype_stat;

for (auto& [name, tensor_storage] : tensor_storage_map) {

if (is_unused_tensor(tensor_storage.name)) {

continue;

}

if (tensor_storage.name.find("vae.") == std::string::npos &&

tensor_storage.name.find("first_stage_model") == std::string::npos) {

continue;

}

auto iter = wtype_stat.find(tensor_storage.type);

if (iter != wtype_stat.end()) {

iter->second++;

} else {

wtype_stat[tensor_storage.type] = 1;

}

}

return wtype_stat;

TensorTypeRules result;

for (const auto& item : split_string(tensor_type_rules, ',')) {

if (item.size() == 0)

continue;

std::string::size_type pos = item.find('=');

if (pos == std::string::npos) {

LOG_WARN("ignoring invalid quant override \"%s\"", item.c_str());

continue;

}

std::string tensor_pattern = item.substr(0, pos);

std::string type_name = item.substr(pos + 1);

ggml_type tensor_type = GGML_TYPE_COUNT;

if (type_name == "f32") {

tensor_type = GGML_TYPE_F32;

} else {

for (size_t i = 0; i < GGML_TYPE_COUNT; i++) {

auto trait = ggml_get_type_traits((ggml_type)i);

if (trait->to_float && trait->type_size && type_name == trait->type_name) {

tensor_type = (ggml_type)i;

}

}

}

if (tensor_type != GGML_TYPE_COUNT) {

result.emplace_back(tensor_pattern, tensor_type);

} else {

LOG_WARN("ignoring invalid quant override \"%s\"", item.c_str());

}

}

return result;

auto map_rules = parse_tensor_type_rules(tensor_type_rules);

for (auto& [name, tensor_storage] : tensor_storage_map) {

ggml_type dst_type = wtype;

for (const auto& tensor_type_rule : map_rules) {

std::regex pattern(tensor_type_rule.first);

if (std::regex_search(name, pattern)) {

dst_type = tensor_type_rule.second;

break;

}

}

if (dst_type == GGML_TYPE_COUNT) {

continue;

}

if (!tensor_should_be_converted(tensor_storage, dst_type)) {

continue;

}

tensor_storage.expected_type = dst_type;

}

if (model_files_processed) {

return;

}

std::vector<TensorStorage> processed_tensor_storages;

for (const auto& [name, tensor_storage] : tensor_storage_map) {

if (is_unused_tensor(tensor_storage.name)) {

continue;

}

processed_tensor_storages.push_back(tensor_storage);

}

for (size_t file_index = 0; file_index < file_paths_.size(); file_index++) {

std::string file_path = file_paths_[file_index];

std::vector<TensorStorage> file_tensors;

for (const auto& ts : processed_tensor_storages) {

if (ts.file_index == file_index) {

file_tensors.push_back(ts);

}

}

if (file_tensors.empty()) {

continue;

}

bool is_zip = false;

for (auto const& ts : file_tensors) {

if (ts.index_in_zip >= 0) {

is_zip = true;

break;

}

}

ModelFileData fdata = {};

fdata.path = file_path;

fdata.is_zip = is_zip;

fdata.tensors = std::move(file_tensors);

if (enable_mmap && !is_zip) {

LOG_DEBUG("using mmap for I/O");

std::unique_ptr<MmapWrapper> mmapped = MmapWrapper::create(file_path, writable_mmap);

if (mmapped) {

uint8_t* mmap_data = static_cast<uint8_t*>(mmapped->writable_data());

ggml_backend_buffer_t buf_mmap = ggml_backend_cpu_buffer_from_ptr(mmap_data, mmapped->size());

if (buf_mmap) {

LOG_INFO("using mmap for '%s'", file_path.c_str());

fdata.mmbuffer = std::shared_ptr<struct ggml_backend_buffer>(buf_mmap, ggml_backend_buffer_free);

} else {

LOG_WARN("mmap: failed to create backend buffer for file %s", fdata.path.c_str());

}

fdata.mmapped = std::shared_ptr<MmapWrapper>(std::move(mmapped));

} else {

LOG_WARN("failed to memory-map '%s' (falling back to read())", file_path.c_str());

}

}

file_data.push_back(std::move(fdata));

}

model_files_processed = true;

std::set<std::string> ignore_tensors,

bool writable_mmap) {

process_model_files(true, writable_mmap);

std::vector<MmapTensorStore> result;

uint64_t mapped_bytes = 0;

size_t mapped_tensors = 0;

LOG_DEBUG("memory-mapping tensors...");

int64_t t_start = ggml_time_ms();

for (auto& fdata : file_data) {

if (!fdata.mmbuffer)

continue;

const std::vector<TensorStorage>& file_tensors = fdata.tensors;

size_t file_mapped_bytes = 0;

size_t file_mapped_tensors = 0;

for (const auto& tensor_storage : file_tensors) {

const std::string& name = tensor_storage.name;

bool is_ignored = false;

for (const auto& ignore_prefix : ignore_tensors) {

if (starts_with(name, ignore_prefix)) {

is_ignored = true;

break;

}

}

if (is_ignored)

continue;

auto it = tensors.find(name);

if (it == tensors.end())

continue;

ggml_tensor* dst_tensor = it->second;

if (dst_tensor == nullptr)

continue;

if (tensor_storage.is_f8_e4m3 ||

tensor_storage.is_f8_e5m2 ||

tensor_storage.is_f64 ||

tensor_storage.is_i64 ||

tensor_storage.type != dst_tensor->type) {

continue;

}

size_t tensor_size = tensor_storage.nbytes();

size_t tensor_offset = tensor_storage.offset;

if (tensor_storage.ne[0] != dst_tensor->ne[0] ||

tensor_storage.ne[1] != dst_tensor->ne[1] ||

tensor_storage.ne[2] != dst_tensor->ne[2] ||

tensor_storage.ne[3] != dst_tensor->ne[3] ||

tensor_size != ggml_nbytes(dst_tensor)) {

// let load_tensors worry about this

continue;

}

ggml_backend_buffer_t buf_mmap = fdata.mmbuffer.get();

uint8_t* mmap_data = static_cast<uint8_t*>(ggml_backend_buffer_get_base(buf_mmap));

dst_tensor->buffer = buf_mmap;

dst_tensor->data = mmap_data + tensor_offset;

file_mapped_bytes += tensor_size;

file_mapped_tensors++;

}

if (file_mapped_bytes > 0) {

mapped_tensors += file_mapped_tensors;

mapped_bytes += file_mapped_bytes;

result.push_back({fdata.mmapped, fdata.mmbuffer});

}

}

int64_t t_end = ggml_time_ms();

int64_t duration_ms = t_end - t_start;

LOG_INFO("memory-mapped %zu tensors in %zu files (%.2f MB), taking %.2fs",

mapped_tensors,

result.size(),

mapped_bytes / (1024.0 * 1024.0),

duration_ms / 1000.0);

return result;

bool enable_mmap,

const std::set<std::string>* target_tensor_names,

bool log_progress) {

process_model_files(enable_mmap, false);

std::atomic<int64_t> read_time_ms(0);

std::atomic<int64_t> memcpy_time_ms(0);

std::atomic<int64_t> copy_to_backend_time_ms(0);

std::atomic<int64_t> convert_time_ms(0);

std::atomic<uint64_t> bytes_processed(0);

int num_threads_to_use = n_threads_;

int64_t start_time = ggml_time_ms();

size_t total_tensors_to_process = 0;

std::vector<size_t> file_tensors_to_process_counts;

file_tensors_to_process_counts.reserve(file_data.size());

for (const auto& fdata : file_data) {

size_t file_tensors_to_process = 0;