Say why the engine died instead of dying mute (#305)

Two silent failures that compound into "[engine terminated]" with no cause.

cap_for_ram() floors the expert cache at cap=1. When resident+slack have
already blown the budget, avail is negative and capmax would be 0 -- "I do
not fit in your budget". Flooring it to 1 and carrying on turns "I do not
fit" into "I overshoot", which is precisely the mid-generation OOM-kill the
function exists to prevent: it printed "projected peak 25.1 GB" against a
22 GB budget and started anyway. Now it says so, names PIN_GB when that is
what inflated the resident set, and refuses to start when the peak also
exceeds the memory actually available on the machine (COLI_RAM_OVERCOMMIT=1
overrides).

The kernel kills with SIGKILL: no error, no log, stdout just closes. coli
read that EOF and printed "[engine terminated]" without ever reaping the
child, so an OOM-kill was indistinguishable from a clean exit -- the report
in #305 (Debian 12, dies mid-generation, no message). engine_diag() now
reports the signal or exit code, names the OOM-killer when it was SIGKILL,
and shows the tail of the engine's stderr.

Reported-by: Ne00n <#305>

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

c/coli

@@ -344,6 +344,34 @@ class Spinner:
         if self.th: self.th.join(timeout=0.4)
         if TTY: sys.stdout.write("\r\033[K"); sys.stdout.flush()
 
+def engine_diag(p, errlog=None):
+    """Lo stdout del motore ha chiuso: il processo e' morto. Dire PERCHE'.
+    Un SIGKILL dell'OOM-killer del kernel e' altrimenti invisibile — niente errore,
+    niente exit code, il motore muore muto e sembra un bug nostro (issue #305).
+    Quasi sempre e' memoria: il picco reale ha sforato la RAM della macchina."""
+    try: rc=p.wait(timeout=5)
+    except Exception: rc=p.poll()
+    if rc is None:                      why="its output closed but the process is still alive"
+    elif rc<0:                                        # POSIX: morte per segnale
+        try: why=f"killed by {signal.Signals(-rc).name}"
+        except Exception: why=f"killed by signal {-rc}"
+    elif rc>0:                          why=f"exit code {rc}"
+    else:                               why="exited cleanly"
+    print(f"\n  {C.yel}[engine terminated: {why}]{C.r}")
+    if rc is not None and rc<0 and -rc==getattr(signal,"SIGKILL",-1):
+        print(f"  {C.yel}nothing in the engine sends SIGKILL to itself: this is the kernel's\n"
+              f"  OOM-killer. The peak RSS exceeded the machine's free memory.\n"
+              f"  Lower --ram, lower PIN_GB, or shorten the context.{C.r}")
+    if errlog is not None:
+        try:
+            errlog.flush()
+            for _ in range(20):         # il drain thread scrive su errlog quando il child chiude
+                tail=open(errlog.name).read().strip()
+                if tail: break
+                time.sleep(0.05)
+            if tail: print(f"  {C.dgray}" + "\n  ".join(tail.splitlines()[-6:]) + f"{C.r}")
+        except Exception: pass
+
 def stream_turn(p, sentinel, on_bytes):
     """legge fino alla sentinella; on_bytes riceve i chunk della risposta. Poi legge la riga STAT.
     Il PRIMO Ctrl-C durante lo stream non chiude la sessione: il motore (handler SIGINT)
@@ -478,7 +506,9 @@ def cmd_chat(a):
     if st is None:
         try: errlog.write(p.stderr.read().decode("utf-8","replace"))
         except (OSError, ValueError): pass
-        errlog.seek(0); print(errlog.read()[-1500:]); sys.exit("the engine exited while loading")
+        errlog.flush(); errlog.seek(0); print(errlog.read()[-1500:])
+        engine_diag(p)        # perche' e' morto (OOM-kill compreso); errlog e' gia' stampato sopra
+        sys.exit("the engine exited while loading")
     p.stdout.readline()   # TIERS line (web-dashboard protocol): emitted once right after STAT;
                           # left unread it leaks into the first answer's text
     # READY received. Drain the child's stderr into errlog without blocking:
@@ -568,7 +598,7 @@ def cmd_chat(a):
             st=stream_turn(p, END, echo)
             if not raw: md.close()
             sp2.stop()
-            if st is None: print(f"\n  {C.yel}[engine terminated]{C.r}"); break
+            if st is None: engine_diag(p, errlog); break
             el=time.time()-t0
             if st.get("tok"):
                 print(f"\r  {C.dgray}└─ {st['tok']} tok · {st['tps']:.2f} tok/s · hit {st['hit']:.0f}% · RSS {st['rss']:.1f} GB · {el:.0f}s{C.r}")

c/glm.c

@@ -5632,7 +5632,30 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){
     double slack = 1.2e9 + pc_b + ws_b + kv_b + kvb_b;
     double avail = ram_gb*1e9 - (double)m->resident_bytes - slack;
     int capmax = (avail>0 && nsp>0) ? (int)(avail/((double)nsp*eb)) : 0;
+    int floored = capmax<1;   /* il budget non regge nemmeno UNO slot per layer */
     if(capmax<1) capmax=1;
+    /* Il floor a 1 e' una bugia comoda: con avail negativo capmax sarebbe 0, cioe'
+     * "non ci sto nel tuo budget". Alzarlo a 1 e proseguire trasforma "non ci sto"
+     * in "sforo" -- ed e' esattamente l'OOM-kill a meta' generazione che questa
+     * funzione esiste per evitare. Il kernel uccide con SIGKILL: nessun errore,
+     * nessun log, il motore muore muto (issue #305). Dirlo, e fermarsi se il picco
+     * non entra nemmeno nella RAM realmente disponibile misurata all'avvio. */
+    if(floored){
+        double peak = (double)m->resident_bytes + (double)capmax*nsp*eb + slack;
+        fprintf(stderr,"[RAM_GB=%.1f%s] WARNING: cap=1 is the floor, projected peak %.1f GB is "
+            "%.1f GB OVER the budget (resident %.1f GB + reserve %.1f GB).%s\n",
+            ram_gb,auto_b?" auto":"",peak/1e9,(peak-ram_gb*1e9)/1e9,
+            m->resident_bytes/1e9,slack/1e9,
+            getenv("PIN_GB")?" PIN_GB is inflating the resident set: lower it or drop it.":"");
+        if(g_mem_avail_boot>0 && peak > g_mem_avail_boot*1e9 &&
+           !(getenv("COLI_RAM_OVERCOMMIT") && atoi(getenv("COLI_RAM_OVERCOMMIT")))){
+            fprintf(stderr,"[RAM] refusing to start: that peak also exceeds the %.1f GB actually "
+                "available on this machine, so the kernel would OOM-kill this run mid-generation.\n"
+                "[RAM] lower PIN_GB, lower the context, or raise the RAM budget if the box really has it "
+                "(COLI_RAM_OVERCOMMIT=1 overrides this check).\n", g_mem_avail_boot);
+            exit(2);
+        }
+    }
     if(capmax < m->ecap){
         fprintf(stderr,"[RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB (ws %.1f, KV %dx%d %.1f, kvb %.1f), "
             "experts %.1f MB x %d layers -> cap lowered %d->%d (projected peak %.1f GB)\n",