diff --git a/c/tools/eval_glm.py b/c/tools/eval_glm.py index 3aa568e..55e12d6 100644 --- a/c/tools/eval_glm.py +++ b/c/tools/eval_glm.py @@ -22,7 +22,7 @@ USO: # leve di ricerca: passate al motore via env TOPP=0.9 python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --ram 15 """ -import os, sys, subprocess, argparse, random, json, tempfile, time +import os, sys, subprocess, argparse, random, json, tempfile, time, threading # mini-set OFFLINE per testare la meccanica (NON misura qualita': domande banali) SMOKE = [ @@ -114,6 +114,7 @@ def main(): ap.add_argument("--seed", type=int, default=1234) ap.add_argument("--dry", action="store_true", help="build requests and stop without running the engine") ap.add_argument("--selftest", action="store_true", help="verify the scoring calculations") + ap.add_argument("--out", default="", help="write incremental results CSV here (one row per request, flushed as it lands)") a = ap.parse_args() if a.selftest: # acc/acc_norm con logprob sintetici @@ -143,15 +144,62 @@ def main(): if a.ram: env["RAM_GB"] = str(a.ram) cmd = [a.glm, str(a.cap)] + a.bits.split() print("running:", " ".join(cmd), file=sys.stderr) + + # Stream results line-by-line so a crash at request N keeps 1..N-1 and shows + # exactly where it stopped. The engine prints " " per + # request to stdout and "[score N req | ...]" progress to stderr; buffering + # both until exit (the old subprocess.run) wastes the whole run on a crash. + out_f = open(a.out, "a") if a.out else None + if out_f: + out_f.write(f"# eval_glm snap={a.snap} tasks={a.tasks} limit={a.limit} seed={a.seed} started={time.strftime('%Y-%m-%dT%H:%M:%S')}\n") + out_f.write("req_idx,task,qi,oi,contlen,contchars,gold,logprob,greedy\n") + out_f.flush() t0 = time.time() - proc = subprocess.run(cmd, env=env, capture_output=True, text=True) - if proc.returncode != 0: - print("ENGINE ERROR:\n", proc.stderr[-2000:], file=sys.stderr); sys.exit(1) - lines = [l for l in proc.stdout.strip().splitlines() if l and l[0] in "-0123456789"] - if len(lines) != len(reqs): - print(f"WARNING: {len(lines)} outputs for {len(reqs)} requests", file=sys.stderr) - lp = [float(l.split()[0]) for l in lines] - print(f"(engine: {time.time()-t0:.0f}s){proc.stderr.strip().splitlines()[-1] if proc.stderr.strip() else ''}", file=sys.stderr) + proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, bufsize=1) # line-buffered + lp = [None] * len(reqs) + n_done = 0 + # Drain stderr (engine progress lines) to console live on a background thread + # so the [score N req] heartbeat is visible while stdout is consumed below. + def _drain_stderr(): + for line in proc.stderr: + print(f" [engine] {line.rstrip()}", file=sys.stderr) + threading.Thread(target=_drain_stderr, daemon=True).start() + for line in proc.stdout: + line = line.strip() + if not line or line[0] not in "-0123456789": continue + parts = line.split() + if n_done >= len(reqs): break + try: logprob = float(parts[0]) + except (ValueError, IndexError): continue + lp[n_done] = logprob + greedy = parts[2] if len(parts) > 2 else "?" + t, qi, oi, clen, cchars, gold = meta[n_done] + if out_f: + out_f.write(f"{n_done},{t},{qi},{oi},{clen},{cchars},{gold},{logprob:.6f},{greedy}\n") + out_f.flush() + n_done += 1 + if n_done % 5 == 0 or n_done == len(reqs): + elapsed = time.time() - t0 + rate = n_done / elapsed if elapsed > 0 else 0 + eta = (len(reqs) - n_done) / rate if rate > 0 else 0 + print(f"[progress] {n_done}/{len(reqs)} requests scored | {elapsed:.0f}s elapsed | " + f"{rate:.2f} req/s | ETA {eta:.0f}s | last: {t} q{qi} opt{oi} lp={logprob:.3f}", + file=sys.stderr) + proc.wait() + elapsed = time.time() - t0 + if out_f: + out_f.write(f"# finished: {n_done}/{len(reqs)} in {elapsed:.0f}s, exit={proc.returncode}\n") + out_f.close() + if proc.returncode != 0 and n_done == 0: + print(f"ENGINE ERROR (exit {proc.returncode})", file=sys.stderr); sys.exit(1) + if n_done != len(reqs): + print(f"WARNING: only {n_done}/{len(reqs)} requests scored (engine exited {proc.returncode}); " + f"scoring partial results.", file=sys.stderr) + # Fill any unscored slots with -inf so argmax never picks them + for i in range(len(lp)): + if lp[i] is None: lp[i] = float("-inf") + print(f"(engine: {elapsed:.0f}s, {n_done}/{len(reqs)} scored, exit {proc.returncode})", file=sys.stderr) score_accuracy(tasks, meta, perq, lp) print("\nNOTE: compare acc_norm with GLM-5.2's PUBLISHED model-card score. A close result" "\n indicates that int4 quantization preserved quality. (Fill REFERENCE in tools/eval_glm.py.)")