eval_glm: stream results line-by-line + incremental CSV (no more lost runs)

The old subprocess.run(capture_output=True) buffered every score result in
memory until the engine exited, then parsed+scored them all at once. If the
engine crashed (or was killed) at request 39/40 you lost everything -- a
multi-hour run with zero information, which is exactly what happened on the
g64 eval against a disk-I/O-bound 400GB model.

Now streams engine stdout line-by-line via Popen:
  - each score result lands in the CSV immediately (flushed), with full
    provenance (task/qi/oi/gold/logprob/greedy + a header with snap/tasks/
    limit/seed/timestamp)
  - a [progress] line every 5 requests: N/total, elapsed, req/s, ETA, last
    request scored
  - the engine's own [score N req] stderr heartbeat streams live too
  - on partial completion (crash/kill at request N): keeps 1..N-1, fills the
    rest with -inf, and scores the partial results -- never a wasted run

New --out <path> writes the incremental CSV; without it, behaves as before
(results to stdout only). Backward compatible: all existing args unchanged.

c/tools/eval_glm.py

@@ -22,7 +22,7 @@ USO:
   # leve di ricerca: passate al motore via env
   TOPP=0.9 python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --ram 15
 """
-import os, sys, subprocess, argparse, random, json, tempfile, time
+import os, sys, subprocess, argparse, random, json, tempfile, time, threading
 
 # mini-set OFFLINE per testare la meccanica (NON misura qualita': domande banali)
 SMOKE = [
@@ -114,6 +114,7 @@ def main():
     ap.add_argument("--seed", type=int, default=1234)
     ap.add_argument("--dry", action="store_true", help="build requests and stop without running the engine")
     ap.add_argument("--selftest", action="store_true", help="verify the scoring calculations")
+    ap.add_argument("--out", default="", help="write incremental results CSV here (one row per request, flushed as it lands)")
     a = ap.parse_args()
 
     if a.selftest:                                   # acc/acc_norm con logprob sintetici
@@ -143,15 +144,62 @@ def main():
     if a.ram: env["RAM_GB"] = str(a.ram)
     cmd = [a.glm, str(a.cap)] + a.bits.split()
     print("running:", " ".join(cmd), file=sys.stderr)
+
+    # Stream results line-by-line so a crash at request N keeps 1..N-1 and shows
+    # exactly where it stopped. The engine prints "<lp> <contlen> <greedy>" per
+    # request to stdout and "[score N req | ...]" progress to stderr; buffering
+    # both until exit (the old subprocess.run) wastes the whole run on a crash.
+    out_f = open(a.out, "a") if a.out else None
+    if out_f:
+        out_f.write(f"# eval_glm snap={a.snap} tasks={a.tasks} limit={a.limit} seed={a.seed} started={time.strftime('%Y-%m-%dT%H:%M:%S')}\n")
+        out_f.write("req_idx,task,qi,oi,contlen,contchars,gold,logprob,greedy\n")
+        out_f.flush()
     t0 = time.time()
-    proc = subprocess.run(cmd, env=env, capture_output=True, text=True)
-    if proc.returncode != 0:
-        print("ENGINE ERROR:\n", proc.stderr[-2000:], file=sys.stderr); sys.exit(1)
-    lines = [l for l in proc.stdout.strip().splitlines() if l and l[0] in "-0123456789"]
-    if len(lines) != len(reqs):
-        print(f"WARNING: {len(lines)} outputs for {len(reqs)} requests", file=sys.stderr)
-    lp = [float(l.split()[0]) for l in lines]
-    print(f"(engine: {time.time()-t0:.0f}s){proc.stderr.strip().splitlines()[-1] if proc.stderr.strip() else ''}", file=sys.stderr)
+    proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                            text=True, bufsize=1)  # line-buffered
+    lp = [None] * len(reqs)
+    n_done = 0
+    # Drain stderr (engine progress lines) to console live on a background thread
+    # so the [score N req] heartbeat is visible while stdout is consumed below.
+    def _drain_stderr():
+        for line in proc.stderr:
+            print(f"  [engine] {line.rstrip()}", file=sys.stderr)
+    threading.Thread(target=_drain_stderr, daemon=True).start()
+    for line in proc.stdout:
+        line = line.strip()
+        if not line or line[0] not in "-0123456789": continue
+        parts = line.split()
+        if n_done >= len(reqs): break
+        try: logprob = float(parts[0])
+        except (ValueError, IndexError): continue
+        lp[n_done] = logprob
+        greedy = parts[2] if len(parts) > 2 else "?"
+        t, qi, oi, clen, cchars, gold = meta[n_done]
+        if out_f:
+            out_f.write(f"{n_done},{t},{qi},{oi},{clen},{cchars},{gold},{logprob:.6f},{greedy}\n")
+            out_f.flush()
+        n_done += 1
+        if n_done % 5 == 0 or n_done == len(reqs):
+            elapsed = time.time() - t0
+            rate = n_done / elapsed if elapsed > 0 else 0
+            eta = (len(reqs) - n_done) / rate if rate > 0 else 0
+            print(f"[progress] {n_done}/{len(reqs)} requests scored | {elapsed:.0f}s elapsed | "
+                  f"{rate:.2f} req/s | ETA {eta:.0f}s | last: {t} q{qi} opt{oi} lp={logprob:.3f}",
+                  file=sys.stderr)
+    proc.wait()
+    elapsed = time.time() - t0
+    if out_f:
+        out_f.write(f"# finished: {n_done}/{len(reqs)} in {elapsed:.0f}s, exit={proc.returncode}\n")
+        out_f.close()
+    if proc.returncode != 0 and n_done == 0:
+        print(f"ENGINE ERROR (exit {proc.returncode})", file=sys.stderr); sys.exit(1)
+    if n_done != len(reqs):
+        print(f"WARNING: only {n_done}/{len(reqs)} requests scored (engine exited {proc.returncode}); "
+              f"scoring partial results.", file=sys.stderr)
+    # Fill any unscored slots with -inf so argmax never picks them
+    for i in range(len(lp)):
+        if lp[i] is None: lp[i] = float("-inf")
+    print(f"(engine: {elapsed:.0f}s, {n_done}/{len(reqs)} scored, exit {proc.returncode})", file=sys.stderr)
     score_accuracy(tasks, meta, perq, lp)
     print("\nNOTE: compare acc_norm with GLM-5.2's PUBLISHED model-card score. A close result"
           "\n      indicates that int4 quantization preserved quality. (Fill REFERENCE in tools/eval_glm.py.)")