diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index 888e324..eac51f9 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -286,6 +286,17 @@ def main(): # testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre # e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward. a.ebits = 8 if (a.mtp or a.indexer) else 4 + if a.mtp and a.ebits < 8 and a.group_size <= 0: + # Non solo lossy: eh_proj ha ~20-30x di asimmetria di scala fra le due meta' di + # colonna, quindi l'int4 per-riga (UNA scala per riga) arrotonda a ZERO l'intera + # meta' embedding -> il draft non vede il token -> acceptance ~0% (issue #8). + # EN: not merely lossy: eh_proj has ~20-30x column-scale asymmetry, so per-row + # EN: int4 rounds its ENTIRE embedding half to exact zeros -> the draft cannot + # EN: see the input token -> ~0% acceptance (issue #8). A container converted + # EN: this way is repairable in place with tools/repair_mtp_int8.py. + print(f"WARNING: --mtp with --ebits {a.ebits} and per-row scales ZEROES eh_proj's " + "embedding half -> MTP acceptance ~0% (issue #8). Use the default --ebits 8, " + "or add --group-size 128 for group-scaled int4.") if a.xbits is None: a.xbits = a.ebits # Build per-type bits map. If a type-specific arg is set, use it; otherwise the diff --git a/c/tools/repair_mtp_int8.py b/c/tools/repair_mtp_int8.py new file mode 100644 index 0000000..695e99f --- /dev/null +++ b/c/tools/repair_mtp_int8.py @@ -0,0 +1,229 @@ +"""Repair an existing colibri int4 container whose MTP head was quantized at int4. + +WHY THIS EXISTS + `model.layers..eh_proj.weight` [D, 2D] multiplies the MTP concat + [embedding_norm ; hidden_norm], and its two column halves differ in scale by + ~20-30x per row (embedding-half absmax ~0.05, hidden-half ~1.5 on GLM-5.2). + Per-row int4 uses ONE scale (= absmax/7) per row, so every embedding-half + weight lands below half a quantization step and np.rint rounds the ENTIRE + embedding half to exact zeros (packed bytes 0x88). The MTP head then drafts + garbage: acceptance ~0% (issue #8 measured 0-4% at int4; 39-59% at int8 — + which is why `convert_fp8_to_int4.py --mtp` defaults to --ebits 8). + + A container converted (or downloaded) with an int4 MTP head does not need a + full re-conversion: this script re-downloads ONLY the affected dense tensors + (~355 MB of HTTP range reads against the FP8 source repo), requantizes them + at int8 with the converter's exact math, and patches the local shards in + place. Originals are kept beside as *.bak-int4. + +WHAT IT TOUCHES + The MTP layer's dense tensors only (eh_proj, q_a/q_b/kv_a/kv_b/o_proj, + shared_experts.*): the ones that stream into RAM once and stay resident. + Routed experts (model.layers..mlp.experts.*) are NOT touched — they are + statistically like the main layers' experts and int4 is acceptable there. + The engine auto-detects int8 vs int4 by blob size (qt_from_disk), so no + engine or config change is needed. Cost: ~+133 MB on disk / resident RAM. + +USAGE + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 # repair + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 --dry-run # inspect only + + --source-repo defaults to zai-org/GLM-5.2-FP8 (the checkpoint the public + int4 containers were converted from). Requires numpy and network access; + no torch, no HF token (public repo, anonymous range reads). +""" +import argparse, glob, json, os, ssl, struct, sys, urllib.request +import numpy as np + +# macOS python.org builds ship no CA bundle: use certifi when available (Linux +# system Pythons generally have working system certs and skip this). +try: + import certifi + _SSL_CTX = ssl.create_default_context(cafile=certifi.where()) +except ImportError: + _SSL_CTX = ssl.create_default_context() + + +# ---------- HTTP range reads against the source repo ---------- +def http_range(url, start, length, tries=5): + req = urllib.request.Request(url, headers={"User-Agent": "colibri-mtp-repair", + "Range": f"bytes={start}-{start+length-1}"}) + for attempt in range(tries): + try: + with urllib.request.urlopen(req, timeout=30, context=_SSL_CTX) as r: + data = r.read() + if len(data) == length: + return data + except KeyboardInterrupt: + raise + except Exception as ex: + if attempt == tries - 1: + raise RuntimeError(f"range read failed for {url}: {ex}") + raise RuntimeError(f"short range read for {url}") + + +class SourceRepo: + def __init__(self, repo, revision="main"): + self.base = f"https://huggingface.co/{repo}/resolve/{revision}/" + with urllib.request.urlopen(self.base + "model.safetensors.index.json", timeout=30, context=_SSL_CTX) as r: + self.wmap = json.loads(r.read())["weight_map"] + self._hdr = {} + + def _shard_header(self, shard): + if shard not in self._hdr: + n = struct.unpack("> 3) & 0xF + mant = (b & 7).astype(np.float64) + v = (sign * np.where(e > 0, (1 + mant / 8) * np.exp2(e.astype(np.float64) - 7), + mant / 8 * np.exp2(-6.0))).reshape(m["shape"]) + sn = name + "_scale_inv" + sshard = self.wmap[sn] + shdr, sbase = self._shard_header(sshard) + sm = shdr[sn] + so0, so1 = sm["data_offsets"] + sc = np.frombuffer(http_range(self.base + sshard, sbase + so0, so1 - so0), + dtype=np.float32).reshape(sm["shape"]) + O, I = m["shape"] + scf = np.repeat(np.repeat(sc, 128, axis=0)[:O], 128, axis=1)[:, :I] + return (v * scf).astype(np.float32) + raise ValueError(f"{name}: unsupported source dtype {m['dtype']}") + + +# ---------- quantization: identical to convert_fp8_to_int4.quant_int8 ---------- +def quant_int8(w): + amax = np.abs(w).max(axis=1, keepdims=True) + s = np.maximum(amax / 127, 1e-8) + q = np.clip(np.rint(w / s), -128, 127).astype(np.int8) + return q.reshape(-1).view(np.uint8).copy(), s[:, 0].astype(np.float32) + + +# ---------- local safetensors IO (no deps; preserves byte-identity of untouched tensors) ---------- +def read_shard(path): + with open(path, "rb") as fh: + n = struct.unpack(" [tensor names to repair] + already_ok, skipped_experts = [], 0 + for f in sorted(glob.glob(os.path.join(a.snap, "*.safetensors"))): + with open(f, "rb") as fh: + n = struct.unpack("