llama.cpp/tools/server/server.cpp at master · rioscode/llama.cpp · GitHub

History

562 lines (487 loc) · 23 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558

559

560

561

562

#include "server-context.h"

#include "server-http.h"

#include "server-models.h"

#include "server-cors-proxy.h"

#include "server-stream.h"

#include "server-tools.h"

#include "arg.h"

#include "build-info.h"

#include "common.h"

#include "fit.h"

#include "llama.h"

#include "log.h"

#include <atomic>

#include <clocale>

#include <exception>

#include <signal.h>

#include <thread> // for std::thread::hardware_concurrency

#if defined(_WIN32)

#include <windows.h>

#endif

static std::function<void(int)> shutdown_handler;

static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;

static inline void signal_handler(int signal) {

if (is_terminating.test_and_set()) {

// in case it hangs, we can force terminate the server by hitting Ctrl+C twice

// this is for better developer experience, we can remove when the server is stable enough

fprintf(stderr, "Received second interrupt, terminating immediately.\n");

exit(1);

}

shutdown_handler(signal);

}

// satisfies -Wmissing-declarations (used by llama command)

int llama_server(int argc, char ** argv);

// to be used via CLI (argc / argv are used by router mode only)

int llama_server(common_params & params, int argc, char ** argv);

void llama_server_terminate();

void llama_server_terminate() {

if (shutdown_handler) {

shutdown_handler(0);

}

// wrapper function that handles exceptions and logs errors

// this is to make sure handler_t never throws exceptions; instead, it returns an error response

static server_http_context::handler_t ex_wrapper(server_http_context::handler_t func) {

return [func = std::move(func)](const server_http_req & req) -> server_http_res_ptr {

std::string message;

error_type error;

try {

return func(req);

} catch (const std::invalid_argument & e) {

// treat invalid_argument as invalid request (400)

error = ERROR_TYPE_INVALID_REQUEST;

message = e.what();

} catch (const std::exception & e) {

// treat other exceptions as server error (500)

error = ERROR_TYPE_SERVER;

message = e.what();

} catch (...) {

error = ERROR_TYPE_SERVER;

message = "unknown error";

}

auto res = std::make_unique<server_http_res>();

res->status = 500;

try {

json error_data = format_error_response(message, error);

res->status = json_value(error_data, "code", 500);

res->data = safe_json_to_str({{ "error", error_data }});

SRV_WRN("got exception: %s\n", res->data.c_str());

} catch (const std::exception & e) {

SRV_ERR("got another exception: %s | while handling exception: %s\n", e.what(), message.c_str());

res->data = "Internal Server Error";

}

return res;

};

}

int llama_server(int argc, char ** argv) {

std::setlocale(LC_NUMERIC, "C");

#ifndef _WIN32

// Ignore SIGPIPE so the server does not crash if a child (MCP server, tools runtime) exits while we are writing to its stdin

signal(SIGPIPE, SIG_IGN);

#endif

// own arguments required by this example

common_params params;

common_init();

// start the stream session manager GC right after common init, before any HTTP route can

// touch it. lifecycle is symmetric, stop_gc() runs in clean_up() before backend free

server_stream_session_manager_start();

if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {

return 1;

}

llama_backend_init();

llama_numa_init(params.numa);

return llama_server(params, argc, argv);

}

int llama_server(common_params & params, int argc, char ** argv) {

bool is_run_by_cli = (argv == nullptr);

common_models_handler models_handler;

// note: router mode also accepts -hf remote-preset, so we need to check that first

if (!is_run_by_cli && !params.model.hf_repo.empty()) {

try {

models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);

if (common_models_handler_is_preset_repo(models_handler)) {

// apply the preset and start the server in router mode

common_models_handler_apply(models_handler, params);

}

} catch (const std::exception & e) {

SRV_ERR("failed to fetch model metadata: %s\n", e.what());

return 1;

}

// router server never loads a model and must not touch the GPU

const bool is_router_server = params.model.path.empty()

&& params.model.hf_repo.empty()

&& params.model.docker_repo.empty();

// skip device enumeration so the CUDA primary context stays uncreated

common_params_print_info(params, !is_router_server);

if (!is_router_server) {

// validate batch size for embeddings

// embeddings require all tokens to be processed in a single ubatch

// see https://github.com/ggml-org/llama.cpp/issues/12836

if (params.embedding && params.n_batch > params.n_ubatch) {

SRV_WRN("embeddings enabled with n_batch (%d) > n_ubatch (%d)\n", params.n_batch, params.n_ubatch);

SRV_WRN("setting n_batch = n_ubatch = %d to avoid assertion failure\n", params.n_ubatch);

params.n_batch = params.n_ubatch;

}

if (params.n_parallel < 0) {

SRV_TRC("%s", "n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n");

params.n_parallel = 4;

params.kv_unified = true;

}

// size the KV pool from --kv-unified-per-slot, unless the user pinned it with -c

// or with -c 0 for max context

const bool ctx_pool_auto_sized = params.kv_unified_per_slot > 0 &&

params.n_ctx == 0 &&

(uint32_t) params.fit_params_min_ctx != UINT32_MAX;

if (ctx_pool_auto_sized) {

params.n_ctx = params.n_parallel * params.kv_unified_per_slot;

SRV_INF("--kv-unified-per-slot: sizing KV pool to n_parallel * kv_unified_per_slot = %d * %d = %d\n", params.n_parallel,

params.kv_unified_per_slot, params.n_ctx);

}

// for consistency between server router mode and single-model mode, we set the same model name as alias

auto model_name = params.model.get_name();

if (params.model_alias.empty() && !model_name.empty()) {

params.model_alias.insert(model_name);

}

// note: this is guaranteed to out-live ctx_http and tools

server_mcp mcp_mgr;

// struct that contains llama context and inference

server_context ctx_server;

server_http_context ctx_http;

if (!ctx_http.init(params)) {

SRV_ERR("%s", "failed to initialize HTTP server\n");

return 1;

}

// Router

// register API routes

server_child child; // only used in non-router mode

server_routes routes(params, ctx_server);

server_tools tools;

std::optional<server_models_routes> models_routes{};

if (is_router_server) {

// setup server instances manager

try {

models_routes.emplace(params, argc, argv);

} catch (const std::exception & e) {

SRV_ERR("failed to initialize router models: %s\n", e.what());

return 1;

}

// proxy handlers

// note: routes.get_health stays the same

routes.get_metrics = models_routes->proxy_get;

routes.post_props = models_routes->proxy_post;

routes.post_completions = models_routes->proxy_post;

routes.post_completions_oai = models_routes->proxy_post;

routes.post_chat_completions = models_routes->proxy_post;

routes.post_control = models_routes->proxy_post;

routes.post_responses_oai = models_routes->proxy_post;

routes.post_transcriptions_oai = models_routes->proxy_post;

routes.post_anthropic_messages = models_routes->proxy_post;

routes.post_anthropic_count_tokens = models_routes->proxy_post;

routes.post_infill = models_routes->proxy_post;

routes.post_embeddings = models_routes->proxy_post;

routes.post_embeddings_oai = models_routes->proxy_post;

routes.post_rerank = models_routes->proxy_post;

routes.post_tokenize = models_routes->proxy_post;

routes.post_detokenize = models_routes->proxy_post;

routes.post_apply_template = models_routes->proxy_post;

routes.post_chat_completions_tok = models_routes->proxy_post;

routes.post_responses_tok_oai = models_routes->proxy_post;

routes.get_lora_adapters = models_routes->proxy_get;

routes.post_lora_adapters = models_routes->proxy_post;

routes.get_slots = models_routes->proxy_get;

routes.post_slots = models_routes->proxy_post;

// custom routes for router

routes.get_props = models_routes->get_router_props;

routes.get_models = models_routes->get_router_models;

ctx_http.post("/models", ex_wrapper(models_routes->post_router_models));

ctx_http.post("/models/load", ex_wrapper(models_routes->post_router_models_load));

ctx_http.post("/models/unload", ex_wrapper(models_routes->post_router_models_unload));

ctx_http.get ("/models/sse", ex_wrapper(models_routes->get_router_models_sse));

ctx_http.del ("/models", ex_wrapper(models_routes->del_router_models));

}

ctx_http.get ("/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check)

ctx_http.get ("/v1/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check)

ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics));

ctx_http.get ("/props", ex_wrapper(routes.get_props));

ctx_http.post("/props", ex_wrapper(routes.post_props));

ctx_http.get ("/models", ex_wrapper(routes.get_models));

ctx_http.get ("/v1/models", ex_wrapper(routes.get_models));

ctx_http.post("/completion", ex_wrapper(routes.post_completions)); // legacy

ctx_http.post("/completions", ex_wrapper(routes.post_completions));

ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai));

ctx_http.post("/chat/completions", ex_wrapper(routes.post_chat_completions));

ctx_http.post("/v1/chat/completions", ex_wrapper(routes.post_chat_completions));

ctx_http.post("/v1/chat/completions/control", ex_wrapper(routes.post_control));

ctx_http.post("/v1/responses", ex_wrapper(routes.post_responses_oai));

ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai));

ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));

ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));

ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API

ctx_http.post("/infill", ex_wrapper(routes.post_infill));

ctx_http.post("/embedding", ex_wrapper(routes.post_embeddings)); // legacy

ctx_http.post("/embeddings", ex_wrapper(routes.post_embeddings));

ctx_http.post("/v1/embeddings", ex_wrapper(routes.post_embeddings_oai));

ctx_http.post("/rerank", ex_wrapper(routes.post_rerank));

ctx_http.post("/reranking", ex_wrapper(routes.post_rerank));

ctx_http.post("/v1/rerank", ex_wrapper(routes.post_rerank));

ctx_http.post("/v1/reranking", ex_wrapper(routes.post_rerank));

ctx_http.post("/tokenize", ex_wrapper(routes.post_tokenize));

ctx_http.post("/detokenize", ex_wrapper(routes.post_detokenize));

ctx_http.post("/apply-template", ex_wrapper(routes.post_apply_template));

// token counting

ctx_http.post("/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));

ctx_http.post("/v1/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));

ctx_http.post("/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai));

ctx_http.post("/v1/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai));

ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting

// LoRA adapters hotswap

ctx_http.get ("/lora-adapters", ex_wrapper(routes.get_lora_adapters));

ctx_http.post("/lora-adapters", ex_wrapper(routes.post_lora_adapters));

// Save & load slots

ctx_http.get ("/slots", ex_wrapper(routes.get_slots));

ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots));

// resumable streaming: a child binds the local session factories, the router binds

// proxies that resolve the owning child, see server-stream.h

server_http_context::handler_t stream_get_h;

server_http_context::handler_t streams_lookup_h;

server_http_context::handler_t stream_delete_h;

if (is_router_server) {

stream_get_h = models_routes->router_stream_get;

streams_lookup_h = models_routes->router_streams_lookup;

stream_delete_h = models_routes->router_stream_delete;

} else {

stream_get_h = server_stream_make_get_handler();

streams_lookup_h = server_stream_make_lookup_handler();

stream_delete_h = server_stream_make_delete_handler();

}

ctx_http.get ("/v1/stream", ex_wrapper(stream_get_h));

ctx_http.post("/v1/streams/lookup", ex_wrapper(streams_lookup_h));

ctx_http.del ("/v1/stream", ex_wrapper(stream_delete_h));

// Google Cloud Platform (Vertex AI) compat

ctx_http.register_gcp_compat();

// return 403 for disabled features

server_http_context::handler_t res_403 = [](const server_http_req &) {

auto res = std::make_unique<server_http_res>();

res->status = 403;

res->data = safe_json_to_str({

{"error", {

{"message", "this feature is disabled"},

{"type", "feature_disabled"},

}}

});

return res;

};

if (params.cors_origins == "*" && params.api_keys.empty()) {

SRV_WRN("%s", "-----------------\n");

SRV_WRN("%s", "CORS is set to allow all origins ('*') and no API key is set\n");

SRV_WRN("%s", "this can be a security risk (cross-origin attacks)\n");

SRV_WRN("%s", "more info: https://github.com/ggml-org/llama.cpp/pull/25655\n");

SRV_WRN("%s", "-----------------\n");

}

// CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)

std::vector<std::string> warn_names;

if (is_router_server) {

warn_names.push_back("router mode");

}

if (params.ui_mcp_proxy) {

ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));

ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));

warn_names.push_back("MCP proxy (experimental)");

} else {

ctx_http.get ("/cors-proxy", ex_wrapper(res_403));

ctx_http.post("/cors-proxy", ex_wrapper(res_403));

}

try {

mcp_mgr.start(params);

} catch (const std::exception & e) {

SRV_ERR("MCP starting failed: %s\n", e.what());

return 1;

}

if (!params.server_tools.empty() || !mcp_mgr.empty()) {

try {

tools.setup(params.server_tools, mcp_mgr, params.server_tools_runtime);

} catch (const std::exception & e) {

SRV_ERR("tools setup failed: %s\n", e.what());

return 1;

}

ctx_http.get ("/tools", ex_wrapper(tools.handle_get));

ctx_http.post("/tools", ex_wrapper(tools.handle_post));

if (!params.server_tools.empty()) {

warn_names.push_back("server tools (experimental)");

}

if (!params.server_tools_runtime.empty()) {

warn_names.push_back("tools runtime (experimental)");

}

if (!mcp_mgr.empty()) {

warn_names.push_back("MCP servers (experimental)");

}

} else {

ctx_http.get ("/tools", ex_wrapper(res_403));

ctx_http.post("/tools", ex_wrapper(res_403));

}

if (warn_names.size() > 0) {

SRV_WRN("%s", "-----------------\n");

SRV_WRN("%s", "the following feature(s) are enabled:\n");

for (const auto & name : warn_names) {

SRV_WRN(" %s\n", name.c_str());

}

SRV_WRN("%s", "do not expose the server to untrusted environments\n");

SRV_WRN("%s", "-----------------\n");

}

// Handle downloading model

if (child.is_child() && child.get_mode() == SERVER_CHILD_MODE_DOWNLOAD) {

return child.run_download(params);

} else if (!is_router_server && !is_run_by_cli) {

// single-model mode (NOT spawned by router)

// if this is invoked by CLI, model downloading should be already handled

try {

common_models_handler_apply(models_handler, params);

} catch (const std::exception & e) {

SRV_ERR("failed to download model: %s\n", e.what());

return 1;

}

// Start the server

std::function<void()> clean_up;

if (is_router_server) {

SRV_INF("%s", "starting server in router mode. models will be automatically loaded on-demand\n");

clean_up = [&models_routes, &mcp_mgr]() {

SRV_INF("%s: cleaning up before exit...\n", __func__);

// stop the session GC first, it finalizes live sessions and wakes pending readers

server_stream_session_manager_stop();

if (models_routes.has_value()) {

models_routes->stopping.store(true); // maybe redundant, but just to be safe

models_routes->models.unload_all();

}

mcp_mgr.shutdown();

llama_backend_free();

};

if (!ctx_http.start()) {

clean_up();

SRV_ERR("%s", "exiting due to HTTP server error\n");

return 1;

}

ctx_http.is_ready.store(true);

shutdown_handler = [&](int) {

if (models_routes.has_value()) {

// important to disconnect any SSE clients

models_routes->stopping.store(true);

}

mcp_mgr.shutdown();

ctx_http.stop();

};

try {

models_routes->models.load_startup_models();

} catch (const std::exception & e) {

SRV_ERR("failed to load models on startup: %s\n", e.what());

ctx_http.stop();

if (ctx_http.thread.joinable()) {

ctx_http.thread.join();

}

clean_up();

return 1;

}

} else {

// setup clean up function, to be called before exit

clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() {

SRV_INF("%s: cleaning up before exit...\n", __func__);

// stop the session GC first, it finalizes live sessions and wakes pending readers

server_stream_session_manager_stop();

ctx_http.stop();

ctx_server.terminate();

mcp_mgr.shutdown();

llama_backend_free();

};

// start the HTTP server before loading the model to be able to serve /health requests

if (!ctx_http.start()) {

clean_up();

SRV_ERR("%s", "exiting due to HTTP server error\n");

return 1;

}

// setup communication child --> router if necessary

if (child.is_child()) {

ctx_server.set_state_callback([&](server_state state, json payload) {

child.notify_to_router(server_state_to_str(state), payload);

});

}

if (!ctx_server.load_model(params)) {

clean_up();

if (ctx_http.thread.joinable()) {

ctx_http.thread.join();

}

SRV_ERR("%s", "exiting due to model loading error\n");

return 1;

}

routes.update_meta(ctx_server);

ctx_http.is_ready.store(true);

SRV_INF("%s", "model loaded\n");

shutdown_handler = [&](int) {

mcp_mgr.shutdown();

// this will unblock start_loop()

ctx_server.terminate();

};

}

// register signal handler if not running by CLI

if (!is_run_by_cli) {

#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))

struct sigaction sigint_action;

sigint_action.sa_handler = signal_handler;

sigemptyset (&sigint_action.sa_mask);

sigint_action.sa_flags = 0;

sigaction(SIGINT, &sigint_action, NULL);

sigaction(SIGTERM, &sigint_action, NULL);

#elif defined (_WIN32)

auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {

return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;

};

SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);

#endif

}

SRV_INF("listening on %s\n", ctx_http.listening_address.c_str());

// TODO: remove this in the future

// check the string to also handle the .sock case

if (string_ends_with(ctx_http.listening_address, ":8080")) {

SRV_WRN("%s", "NOTICE: server default port will be changed to :9931 in a future release\n");

SRV_WRN("%s", " ref: https://github.com/ggml-org/llama.cpp/pull/26508\n");

}

if (is_router_server) {

if (!params.models_preset_hf.empty()) {

SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());

SRV_WRN("%s", " please only use presets that you can trust! Unknown presets may be unsafe\n");

}

if (ctx_http.thread.joinable()) {

ctx_http.thread.join(); // keep the main thread alive

}

// when the HTTP server stops, clean up and exit

clean_up();

} else {

// optionally, notify router server that this instance is ready

std::thread monitor_thread;

if (child.is_child()) {

monitor_thread = child.setup(shutdown_handler);

child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());

}

// this call blocks the main thread until queue_tasks.terminate() is called

ctx_server.start_loop();

clean_up();

if (ctx_http.thread.joinable()) {

ctx_http.thread.join();

}

if (monitor_thread.joinable()) {

monitor_thread.join();

}

auto * ll_ctx = ctx_server.get_llama_context();

if (ll_ctx != nullptr) {

common_memory_breakdown_print(ll_ctx);

}

return 0;

}

llama.cpp/tools/server/server.cpp at master · rioscode/llama.cpp · GitHub

Expand file tree

Breadcrumbs

server.cpp

Latest commit

History

Breadcrumbs

server.cpp

File metadata and controls