sleepy/llama.cpp
1
0
Fork 0
Code Issues 9 Pull Requests Actions 148 Packages Projects Releases Wiki Activity

server: (router) add stop-timeout option (#18350)

Browse Source 
* server: (router) add stop-timeout option

* also allow stop while loading

* add docs

* unload_lru: also wait for unload to complete
This commit is contained in:
Xuan-Son Nguyen
2025-12-24 23:47:49 +01:00
committed by GitHub
parent 4cbafad4f0
commit f5acfb2ffa
5 changed files with 133 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+6 -6
View File
@@ -3518,15 +3518,15 @@ void common_params_add_preset_options(std::vector<common_arg> & args) {
[](common_params &, const std::string &) { /* unused */ }
).set_env(COMMON_ARG_PRESET_LOAD_ON_STARTUP).set_preset_only());
args.push_back(common_arg(
{"stop-timeout"}, "SECONDS",
"in server router mode, force-kill model instance after this many seconds of graceful shutdown",
[](common_params &, int) { /* unused */ }
).set_env(COMMON_ARG_PRESET_STOP_TIMEOUT).set_preset_only());
// args.push_back(common_arg(
// {"pin"},
// "in server router mode, do not unload this model if models_max is exceeded",
// [](common_params &) { /* unused */ }
// ).set_preset_only());
// args.push_back(common_arg(
// {"unload-idle-seconds"}, "SECONDS",
// "in server router mode, unload models idle for more than this many seconds",
// [](common_params &, int) { /* unused */ }
// ).set_preset_only());
}