From 540781528ee1266fcea63233fcc247404d95910c Mon Sep 17 00:00:00 2001 From: Dennis Paul Date: Tue, 14 Jul 2026 14:11:15 +0200 Subject: [PATCH] quant_ablation: condition GLM contexts on [gMASK] for in-distribution scoring (#155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool tokenized every context with add_special_tokens=False, so pointing it at a GLM snapshot scored the model out-of-distribution — the same bug @bokiko found in eval_glm.py (#108). Measured on this box (GLM-5.2 int4, engine SCORE mode, same 1023 target tokens, only the conditioning prefix changed): corpus no prefix +[gMASK] natural prose PPL 29.2 PPL 9.4 markdown + code PPL 131.0 PPL 24.5 It does not just depress scores, it distorts sensitivity to numerical changes: an exact-vs-IDOT kernel A/B measured without the prefix reported a penalty that halved AND flipped sign on one of two corpora once the prefix was restored (#153). A quantization-ablation tool that scores OOD is therefore worse than useless — it produces confident deltas that are artifacts. The GLM tokenizer does not add the prefix itself (add_special_tokens=True is a no-op there), so it must be prepended explicitly. Auto-detected from the vocab rather than opt-in, so it cannot be lost by omission; --prefix overrides. Models with no such prefix are unaffected: OLMoE (the tool's default) has no BOS at all, add_special_tokens True/False give identical ids, so the numbers in #108 measured on OLMoE still stand. --- c/tools/quant_ablation.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/c/tools/quant_ablation.py b/c/tools/quant_ablation.py index dd192ba..7a870ff 100644 --- a/c/tools/quant_ablation.py +++ b/c/tools/quant_ablation.py @@ -170,15 +170,38 @@ def apply_scheme(model, scheme): # Scoring — mirrors tools/eval_glm.py exactly: # acc = argmax over options of sum(logprob of continuation tokens) # acc_norm = argmax over options of sum(logprob) / len(continuation string in CHARACTERS) +# +# THE PREFIX IS NOT OPTIONAL (issue #108, credit @bokiko). GLM-5.2 sees "[gMASK]" at the +# start of every training sequence. Score it without that prefix and the model runs +# out-of-distribution: measured here, perplexity on plain English prose goes 9.4 -> 29.2, and +# on markdown/code 24.5 -> 131.0. It does not merely depress scores, it distorts SENSITIVITY: +# an int4-vs-exact kernel A/B measured without the prefix reported a penalty that halved and +# flipped sign on one corpus once the prefix was restored (#153). Any quantization delta +# measured OOD is therefore suspect. +# +# The GLM tokenizer does NOT add it for you — add_special_tokens=True is a no-op there — so it +# has to be prepended explicitly. Auto-detected from the vocab so it cannot be lost by +# omission; models with no such prefix (e.g. OLMoE, which has no BOS at all) are unaffected. # -------------------------------------------------------------------------------------- -def load_docs(task, data_dir, limit, seed): +def detect_prefix(tk): + """'[gMASK]' for GLM snapshots, '' otherwise. --prefix overrides.""" + vocab = tk.get_vocab() + if "[gMASK]" in vocab and "" in vocab: + return "[gMASK]" + return "" + + +def load_docs(task, data_dir, limit, seed, prefix=""): path = f"{data_dir}/{task}.jsonl" try: docs = [json.loads(l) for l in open(path) if l.strip()] except FileNotFoundError: raise SystemExit(f"missing {path} — run: python tools/fetch_benchmarks.py --out {data_dir} --tasks {task}") random.Random(seed).shuffle(docs) # same seed/shuffle convention as eval_glm.py - return docs[:limit] if limit else docs + docs = docs[:limit] if limit else docs + if prefix: # condition every context in-distribution + docs = [dict(d, ctx=prefix + d["ctx"]) for d in docs] + return docs @torch.no_grad() @@ -221,6 +244,9 @@ def main(): ap.add_argument("--min-coverage", type=float, default=95.0, help="fail if a scheme quantized less than this %% of params (catches the " "3D-fused-expert trap, where a ndim==2 filter skips every expert)") + ap.add_argument("--prefix", default=None, + help="context prefix (default: auto — '[gMASK]' for GLM, '' otherwise). " + "Scoring GLM without it runs the model out-of-distribution: see #108.") a = ap.parse_args() tasks = a.tasks.split(",") @@ -229,7 +255,11 @@ def main(): parse_scheme(s) # fail fast on a typo tk = AutoTokenizer.from_pretrained(a.model, trust_remote_code=True) - docs = {t: load_docs(t, a.data, a.limit, a.seed) for t in tasks} + prefix = detect_prefix(tk) if a.prefix is None else a.prefix + print(f"[prefix] {prefix!r}" + (" (auto-detected: GLM snapshot)" if prefix and a.prefix is None + else " (no prefix for this model)" if not prefix else " (--prefix)"), + flush=True) + docs = {t: load_docs(t, a.data, a.limit, a.seed, prefix) for t in tasks} means, rows = {}, {} for scheme in schemes: