#include "ggml-alloc.h"
#include "../ggml/src/ggml-backend-impl.h"
#include "ggml-cpp.h"
#include "../ggml/src/ggml-impl.h"
#include "ggml.h"
#include
#include
#include
#include
//
// dummy backend with configurable max_buffer_size, tracks allocations
uint8_t * const alloc_base = (uint8_t *) 16;
struct dummy_backend_context {
size_t max_buffer_size = 64;
size_t alignment = 8;
ggml_backend_buffer_i buffer_interface;
ggml_backend_device device;
ggml_backend backend;
std::vector buffers;
size_t allocated_total() const {
size_t n = 0;
for (ggml_backend_buffer_t buf : buffers) {
n += ggml_backend_buffer_get_size(buf);
}
return n;
}
};
// ggml_backend_buffer_type interface
static const char * dummy_backend_buffer_type_get_name(ggml_backend_buffer_type_t) {
return "dummy_buffer_type";
}
static ggml_backend_buffer_t dummy_backend_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
dummy_backend_context * ctx = (dummy_backend_context *) buft->context;
ggml_backend_buffer_t & buffer = ctx->buffers.emplace_back();
buffer = ggml_backend_buffer_init(buft, ctx->buffer_interface, ctx, size);
return buffer;
}
static size_t dummy_backend_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
dummy_backend_context * ctx = (dummy_backend_context *) buft->context;
return ctx->alignment;
}
static size_t dummy_backend_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
dummy_backend_context * ctx = (dummy_backend_context *) buft->context;
return ctx->max_buffer_size;
}
static bool dummy_backend_buffer_type_is_host(ggml_backend_buffer_type_t) {
return true;
}
// ggml_backend_buffer interface
static void dummy_backend_buffer_free_buffer(ggml_backend_buffer_t buffer) {
dummy_backend_context * ctx = (dummy_backend_context *) buffer->context;
auto i = std::find(ctx->buffers.begin(), ctx->buffers.end(), buffer);
GGML_ASSERT(i != ctx->buffers.end());
ctx->buffers.erase(i);
}
static void * dummy_backend_buffer_get_base(ggml_backend_buffer_t) {
return alloc_base;
}
static ggml_status dummy_backend_buffer_init_tensor(ggml_backend_buffer_t, ggml_tensor *) {
return GGML_STATUS_SUCCESS;
}
static void dummy_backend_buffer_memset_tensor(ggml_backend_buffer_t, ggml_tensor *, uint8_t, size_t, size_t) {}
static void dummy_backend_buffer_set_tensor(ggml_backend_buffer_t, ggml_tensor *, const void *, size_t, size_t) {}
static void dummy_backend_buffer_get_tensor(ggml_backend_buffer_t, const ggml_tensor *, void *, size_t, size_t) {}
static void dummy_backend_buffer_clear(ggml_backend_buffer_t, uint8_t) {}
// ggml_backend_device interface
static enum ggml_backend_dev_type dummy_backend_device_get_type(ggml_backend_dev_t) {
return GGML_BACKEND_DEVICE_TYPE_CPU;
}
static bool dummy_backend_device_supports_op(ggml_backend_dev_t, const ggml_tensor *) {
return true;
}
static bool dummy_backend_device_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) {
return device->context == buft->context;
}
// ggml_backend interface
static const char * dummy_backend_get_name(ggml_backend_t) {
return "dummy_backend";
}
// dummy_backend
struct dummy_backend {
std::unique_ptr context;
ggml_backend_buffer_type buffer_type;
};
static dummy_backend dummy_backend_init(size_t max_buffer_size, size_t alignment = 8) {
dummy_backend b{};
b.context = std::make_unique();
b.context->alignment = alignment;
b.context->max_buffer_size = max_buffer_size;
b.context->buffer_interface.free_buffer = dummy_backend_buffer_free_buffer;
b.context->buffer_interface.get_base = dummy_backend_buffer_get_base;
b.context->buffer_interface.init_tensor = dummy_backend_buffer_init_tensor;
b.context->buffer_interface.memset_tensor = dummy_backend_buffer_memset_tensor;
b.context->buffer_interface.set_tensor = dummy_backend_buffer_set_tensor;
b.context->buffer_interface.get_tensor = dummy_backend_buffer_get_tensor;
b.context->buffer_interface.clear = dummy_backend_buffer_clear;
b.context->device.context = b.context.get();
b.context->device.iface.get_type = dummy_backend_device_get_type;
b.context->device.iface.supports_op = dummy_backend_device_supports_op;
b.context->device.iface.supports_buft = dummy_backend_device_supports_buft;
b.context->backend.context = b.context.get();
b.context->backend.device = &b.context->device;
b.context->backend.iface.get_name = dummy_backend_get_name;
b.buffer_type.device = &b.context->device;
b.buffer_type.context = b.context.get();
b.buffer_type.iface.get_name = dummy_backend_buffer_type_get_name;
b.buffer_type.iface.alloc_buffer = dummy_backend_buffer_type_alloc_buffer;
b.buffer_type.iface.get_alignment = dummy_backend_buffer_type_get_alignment;
b.buffer_type.iface.get_max_size = dummy_backend_buffer_type_get_max_size;
b.buffer_type.iface.is_host = dummy_backend_buffer_type_is_host;
return b;
}
//
// test utilities
struct test_context_with_graph {
ggml_context * ctx;
ggml_cgraph * graph;
ggml_context_ptr ctx_ptr;
};
static test_context_with_graph make_context() {
ggml_init_params params{};
params.mem_size = 48 * ggml_tensor_overhead() + ggml_graph_overhead();
params.no_alloc = true;
ggml_context * ctx = ggml_init(params);
ggml_context_ptr ctx_ptr = ggml_context_ptr(ctx);
ggml_cgraph * graph = ggml_new_graph(ctx);
return { ctx, graph, std::move(ctx_ptr) };
}
static ggml_tensor * make_input_1d(ggml_context * ctx, int64_t n_elements) {
ggml_tensor * t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
ggml_set_input(t);
return t;
}
static ggml_tensor * make_input_with_size(ggml_context * ctx, size_t size_bytes) {
GGML_ASSERT(size_bytes % 4 == 0);
return make_input_1d(ctx, size_bytes / 4);
}
static void assign_names(ggml_context * ctx, const char * prefix = "x") {
int i = 0;
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t; t = ggml_get_next_tensor(ctx, t)) {
ggml_format_name(t, "%s%d", prefix, i++);
}
}
static int get_leaf_id(ggml_cgraph * graph, const char * tensor_name) {
for (int i = 0; i < graph->n_leafs; ++i) {
if (strncmp(graph->leafs[i]->name, tensor_name, GGML_MAX_NAME) == 0) {
return i;
}
}
fprintf(stderr, "leaf not found: %s\n", tensor_name);
return -1;
}
static int get_node_id(ggml_cgraph * graph, const char * tensor_name) {
for (int i = 0; i < graph->n_nodes; ++i) {
if (strncmp(graph->nodes[i]->name, tensor_name, GGML_MAX_NAME) == 0) {
return i;
}
}
fprintf(stderr, "node not found: %s", tensor_name);
return -1;
}
static ggml_gallocr_ptr allocate_graph(ggml_cgraph * graph, ggml_tensor * out, ggml_backend_buffer_type_t buft) {
ggml_set_output(out);
ggml_build_forward_expand(graph, out);
ggml_gallocr_ptr galloc = ggml_gallocr_ptr(ggml_gallocr_new(buft));
bool result = ggml_gallocr_alloc_graph(galloc.get(), graph);
GGML_ASSERT(result);
return galloc;
}
//
// correctness checks for result allocations
static void check_all_allocated(ggml_cgraph * graph) {
for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
ggml_tensor * t = ggml_graph_node(graph, i);
GGML_ASSERT(t->buffer != nullptr);
GGML_ASSERT(t->data != nullptr);
}
}
static void check_max_size(ggml_context * ctx) {
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t; t = ggml_get_next_tensor(ctx, t)) {
auto buft = ggml_backend_buffer_get_type(t->buffer);
size_t max_size = ggml_backend_buft_get_max_size(buft);
size_t offset = (char *) t->data - (char *) ggml_backend_buffer_get_base(t->buffer);
GGML_ASSERT(t->data >= ggml_backend_buffer_get_base(t->buffer));
GGML_ASSERT((size_t) offset + ggml_nbytes(t) flags & GGML_TENSOR_FLAG_OUTPUT) {
return false;
}
// Check if `other` is still "alive", ie. an input to any node after the `current` op
for (int i = current_i; i < ggml_graph_n_nodes(graph); ++i) {
ggml_tensor * t = ggml_graph_node(graph, i);
for (int s = 0; s < GGML_MAX_SRC; s++) {
if (t == current && ggml_op_can_inplace(t->op)) {
continue;
}
if (t->src[s] == other) {
return false;
}
if (t->src[s] && t->src[s]->view_src == other) {
return false;
}
}
}
return true;
}
static bool memory_overlap(ggml_tensor * a, ggml_tensor * b) {
if (a->buffer != b->buffer) {
return false;
}
int64_t a0 = (int64_t) a->data;
int64_t a1 = a0 + ggml_nbytes(a);
int64_t b0 = (int64_t) b->data;
int64_t b1 = b0 + ggml_nbytes(b);
return a1 > b0 && b1 > a0;
}
static ggml_tensor * get_view_source(ggml_tensor * t) {
while (t->view_src) {
t = t->view_src;
}
return t;
}
static void check_no_overlap(ggml_cgraph * graph) {
for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
for (int j = 0; j < i; ++j) {
ggml_tensor * t = ggml_graph_node(graph, i);
ggml_tensor * o = ggml_graph_node(graph, j);
GGML_ASSERT(t != o);
if (get_view_source(t) == get_view_source(o)) {
continue;
}
if (memory_overlap(t, o)) {
GGML_ASSERT(can_reuse_memory(graph, i, t, o));
}
}
}
}
//
// test cases
// Scenario where the first backend buffer is completely exhausted and there are further
// tensors which require a second buffer
static void test_max_size_too_many_tensors() {
dummy_backend backend = dummy_backend_init(16);
auto [ctx, graph, ctx_ptr] = make_context();
ggml_tensor * x[7];
x[0] = make_input_with_size(ctx, 8);
x[1] = make_input_with_size(ctx, 8);
x[2] = make_input_with_size(ctx, 8);
x[3] = ggml_mul(ctx, x[0], x[1]);
x[4] = ggml_add(ctx, x[1], x[2]);
x[5] = ggml_add(ctx, x[3], x[0]);
x[6] = ggml_add(ctx, x[4], x[5]);
assign_names(ctx);
ggml_gallocr_ptr galloc = allocate_graph(graph, x[6], &backend.buffer_type);
check_all_allocated(graph);
check_no_overlap(graph);
check_max_size(ctx);
GGML_ASSERT(backend.context->allocated_total() allocated_total() allocated_total() == 24);
}
// This test assumes a max of 16 buffer chunks, and tries to allocate tensors that would
// require more. Expectation is that the last buffer should grow to fit everything,
// leaving it to the backend to error out if it can't allocate that much.
static void test_not_enough_chunks() {
const int max_chunks = 16;
const int max_size = 8;
dummy_backend backend = dummy_backend_init(max_size);
auto [ctx, graph, ctx_ptr] = make_context();
ggml_tensor * x[max_chunks + 1];
for (int i = 0; i < max_chunks + 1; ++i) {
x[i] = make_input_with_size(ctx, max_size);
}
ggml_tensor * acc = x[0];
for (int i = 0; i < max_chunks; ++i) {
acc = ggml_add(ctx, acc, x[i + 1]);
}
assign_names(ctx);
ggml_gallocr_ptr galloc = allocate_graph(graph, acc, &backend.buffer_type);
check_all_allocated(graph);
check_no_overlap(graph);
GGML_ASSERT(backend.context->allocated_total() > max_chunks * max_size);
}
// Fill up leftover unallocated space of a chunk after allocating a large tensor that
// requires a new chunk.
static void test_fill_leftover_space() {
dummy_backend backend = dummy_backend_init(16);
auto [ctx, graph, ctx_ptr] = make_context();
ggml_tensor * x[4];
x[0] = make_input_with_size(ctx, 8);
x[1] = ggml_pad(ctx, x[0], 2, 0, 0, 0);
x[3] = ggml_mean(ctx, x[1]);
assign_names(ctx);
ggml_gallocr_ptr galloc = allocate_graph(graph, x[3], &backend.buffer_type);
check_all_allocated(graph);
check_no_overlap(graph);
check_max_size(ctx);
GGML_ASSERT(backend.context->allocated_total()