sleepy/llama.cpp
1
0
Fork 0
Code Issues 8 Pull Requests Actions 168 Packages Projects Releases Wiki Activity

server: support load model on startup, support preset-only options (#18206)

Browse Source 
* server: support autoload model, support preset-only options

* add docs

* load-on-startup

* fix

* Update common/arg.cpp

Co-authored-by: Pascal <admin@serveurperso.com>

---------

Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Xuan-Son Nguyen
2025-12-20 09:25:27 +01:00
committed by GitHub
parent 74e05131e9
commit 9e39a1e6a9
7 changed files with 80 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-models.cpp
+20
View File
@@ -226,6 +226,26 @@ void server_models::load_models() {
SRV_INF(" %c %s\n", has_custom ? '*' : ' ', name.c_str());
}
}
// load any autoload models
std::vector<std::string> models_to_load;
for (const auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val)) {
models_to_load.push_back(name);
}
}
if ((int)models_to_load.size() > base_params.models_max) {
throw std::runtime_error(string_format(
"number of models to load on startup (%zu) exceeds models_max (%d)",
models_to_load.size(),
base_params.models_max
));
}
for (const auto & name : models_to_load) {
SRV_INF("(startup) loading model %s\n", name.c_str());
load(name);
}
}
void server_models::update_meta(const std::string & name, const server_model_meta & meta) {