From d57955e95a6dfd99606d1cf26c338f524bdc8d5a Mon Sep 17 00:00:00 2001 From: JustVugg Date: Thu, 16 Jul 2026 12:25:53 +0200 Subject: [PATCH] Say why the engine died instead of dying mute (#305) Two silent failures that compound into "[engine terminated]" with no cause. cap_for_ram() floors the expert cache at cap=1. When resident+slack have already blown the budget, avail is negative and capmax would be 0 -- "I do not fit in your budget". Flooring it to 1 and carrying on turns "I do not fit" into "I overshoot", which is precisely the mid-generation OOM-kill the function exists to prevent: it printed "projected peak 25.1 GB" against a 22 GB budget and started anyway. Now it says so, names PIN_GB when that is what inflated the resident set, and refuses to start when the peak also exceeds the memory actually available on the machine (COLI_RAM_OVERCOMMIT=1 overrides). The kernel kills with SIGKILL: no error, no log, stdout just closes. coli read that EOF and printed "[engine terminated]" without ever reaping the child, so an OOM-kill was indistinguishable from a clean exit -- the report in #305 (Debian 12, dies mid-generation, no message). engine_diag() now reports the signal or exit code, names the OOM-killer when it was SIGKILL, and shows the tail of the engine's stderr. Reported-by: Ne00n <#305> Co-Authored-By: Claude Opus 4.8 --- c/coli | 34 ++++++++++++++++++++++++++++++++-- c/glm.c | 23 +++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/c/coli b/c/coli index efc8526..ad2f13e 100755 --- a/c/coli +++ b/c/coli @@ -344,6 +344,34 @@ class Spinner: if self.th: self.th.join(timeout=0.4) if TTY: sys.stdout.write("\r\033[K"); sys.stdout.flush() +def engine_diag(p, errlog=None): + """Lo stdout del motore ha chiuso: il processo e' morto. Dire PERCHE'. + Un SIGKILL dell'OOM-killer del kernel e' altrimenti invisibile — niente errore, + niente exit code, il motore muore muto e sembra un bug nostro (issue #305). + Quasi sempre e' memoria: il picco reale ha sforato la RAM della macchina.""" + try: rc=p.wait(timeout=5) + except Exception: rc=p.poll() + if rc is None: why="its output closed but the process is still alive" + elif rc<0: # POSIX: morte per segnale + try: why=f"killed by {signal.Signals(-rc).name}" + except Exception: why=f"killed by signal {-rc}" + elif rc>0: why=f"exit code {rc}" + else: why="exited cleanly" + print(f"\n {C.yel}[engine terminated: {why}]{C.r}") + if rc is not None and rc<0 and -rc==getattr(signal,"SIGKILL",-1): + print(f" {C.yel}nothing in the engine sends SIGKILL to itself: this is the kernel's\n" + f" OOM-killer. The peak RSS exceeded the machine's free memory.\n" + f" Lower --ram, lower PIN_GB, or shorten the context.{C.r}") + if errlog is not None: + try: + errlog.flush() + for _ in range(20): # il drain thread scrive su errlog quando il child chiude + tail=open(errlog.name).read().strip() + if tail: break + time.sleep(0.05) + if tail: print(f" {C.dgray}" + "\n ".join(tail.splitlines()[-6:]) + f"{C.r}") + except Exception: pass + def stream_turn(p, sentinel, on_bytes): """legge fino alla sentinella; on_bytes riceve i chunk della risposta. Poi legge la riga STAT. Il PRIMO Ctrl-C durante lo stream non chiude la sessione: il motore (handler SIGINT) @@ -478,7 +506,9 @@ def cmd_chat(a): if st is None: try: errlog.write(p.stderr.read().decode("utf-8","replace")) except (OSError, ValueError): pass - errlog.seek(0); print(errlog.read()[-1500:]); sys.exit("the engine exited while loading") + errlog.flush(); errlog.seek(0); print(errlog.read()[-1500:]) + engine_diag(p) # perche' e' morto (OOM-kill compreso); errlog e' gia' stampato sopra + sys.exit("the engine exited while loading") p.stdout.readline() # TIERS line (web-dashboard protocol): emitted once right after STAT; # left unread it leaks into the first answer's text # READY received. Drain the child's stderr into errlog without blocking: @@ -568,7 +598,7 @@ def cmd_chat(a): st=stream_turn(p, END, echo) if not raw: md.close() sp2.stop() - if st is None: print(f"\n {C.yel}[engine terminated]{C.r}"); break + if st is None: engine_diag(p, errlog); break el=time.time()-t0 if st.get("tok"): print(f"\r {C.dgray}└─ {st['tok']} tok · {st['tps']:.2f} tok/s · hit {st['hit']:.0f}% · RSS {st['rss']:.1f} GB · {el:.0f}s{C.r}") diff --git a/c/glm.c b/c/glm.c index 7e05ad9..ae18c55 100644 --- a/c/glm.c +++ b/c/glm.c @@ -5632,7 +5632,30 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ double slack = 1.2e9 + pc_b + ws_b + kv_b + kvb_b; double avail = ram_gb*1e9 - (double)m->resident_bytes - slack; int capmax = (avail>0 && nsp>0) ? (int)(avail/((double)nsp*eb)) : 0; + int floored = capmax<1; /* il budget non regge nemmeno UNO slot per layer */ if(capmax<1) capmax=1; + /* Il floor a 1 e' una bugia comoda: con avail negativo capmax sarebbe 0, cioe' + * "non ci sto nel tuo budget". Alzarlo a 1 e proseguire trasforma "non ci sto" + * in "sforo" -- ed e' esattamente l'OOM-kill a meta' generazione che questa + * funzione esiste per evitare. Il kernel uccide con SIGKILL: nessun errore, + * nessun log, il motore muore muto (issue #305). Dirlo, e fermarsi se il picco + * non entra nemmeno nella RAM realmente disponibile misurata all'avvio. */ + if(floored){ + double peak = (double)m->resident_bytes + (double)capmax*nsp*eb + slack; + fprintf(stderr,"[RAM_GB=%.1f%s] WARNING: cap=1 is the floor, projected peak %.1f GB is " + "%.1f GB OVER the budget (resident %.1f GB + reserve %.1f GB).%s\n", + ram_gb,auto_b?" auto":"",peak/1e9,(peak-ram_gb*1e9)/1e9, + m->resident_bytes/1e9,slack/1e9, + getenv("PIN_GB")?" PIN_GB is inflating the resident set: lower it or drop it.":""); + if(g_mem_avail_boot>0 && peak > g_mem_avail_boot*1e9 && + !(getenv("COLI_RAM_OVERCOMMIT") && atoi(getenv("COLI_RAM_OVERCOMMIT")))){ + fprintf(stderr,"[RAM] refusing to start: that peak also exceeds the %.1f GB actually " + "available on this machine, so the kernel would OOM-kill this run mid-generation.\n" + "[RAM] lower PIN_GB, lower the context, or raise the RAM budget if the box really has it " + "(COLI_RAM_OVERCOMMIT=1 overrides this check).\n", g_mem_avail_boot); + exit(2); + } + } if(capmax < m->ecap){ fprintf(stderr,"[RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB (ws %.1f, KV %dx%d %.1f, kvb %.1f), " "experts %.1f MB x %d layers -> cap lowered %d->%d (projected peak %.1f GB)\n",