From 5230717cf41022ca9c34d790209bde0655bad6a9 Mon Sep 17 00:00:00 2001 From: bokiko Date: Tue, 14 Jul 2026 15:19:48 +0300 Subject: [PATCH] eval: default the [gMASK] prefix ON for GLM snapshots (#108) Scoring raw completions without GLM's training-time prefix runs the model out-of-distribution: scores drop and A/B sensitivity distorts (#108). Detect GLM via config.json model_type and prepend automatically, with a stderr notice. EVAL_PREFIX (including empty) still overrides for research use. Co-Authored-By: Claude Fable 5 --- c/tools/eval_glm.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/c/tools/eval_glm.py b/c/tools/eval_glm.py index b167253..d750921 100644 --- a/c/tools/eval_glm.py +++ b/c/tools/eval_glm.py @@ -48,11 +48,24 @@ def load_docs(task, data_dir, limit, seed): random.Random(seed).shuffle(docs) return docs[:limit] if limit else docs -def build_requests(tk, docs_by_task): +def detect_prefix(snap): + """GLM sees [gMASK] at the start of every training sequence; scoring raw text + without it is out-of-distribution and silently depresses/distorts scores (#108). + Default the prefix ON for GLM snapshots; EVAL_PREFIX (even empty) overrides.""" + if "EVAL_PREFIX" in os.environ: return os.environ["EVAL_PREFIX"] + try: mt = json.load(open(os.path.join(snap, "config.json"))).get("model_type", "") + except Exception: mt = "" + if "glm" in mt.lower(): + print("[prefix] GLM snapshot: prepending [gMASK] to every context " + "(override with EVAL_PREFIX, disable with EVAL_PREFIX=)", file=sys.stderr) + return "[gMASK]" + return "" + +def build_requests(tk, docs_by_task, prefix=""): reqs, meta, perq = [], [], {} for t, docs in docs_by_task.items(): for qi, d in enumerate(docs): - ctx, conts, gold = d["ctx"], d["choices"], int(d["gold"]) + ctx, conts, gold = prefix + d["ctx"], d["choices"], int(d["gold"]) ctx_ids = tk.encode(ctx).ids for oi, cont in enumerate(conts): full = tk.encode(ctx + cont).ids @@ -115,7 +128,7 @@ def main(): docs_by_task = {t: load_docs(t, a.data, a.limit, a.seed) for t in tasks} for t, d in docs_by_task.items(): print(f"[{t}] {len(d)} questions", file=sys.stderr) - reqs, meta, perq = build_requests(tk, docs_by_task) + reqs, meta, perq = build_requests(tk, docs_by_task, detect_prefix(a.snap)) print(f"total requests: {len(reqs)} (answer options)", file=sys.stderr) if a.dry: for r in reqs[:3]: print(" example request:", r[:80], "...", file=sys.stderr)