From 308e269f468797211a32884f2a942083ac773aac Mon Sep 17 00:00:00 2001 From: andres-gb10 Date: Sat, 18 Jul 2026 21:11:40 +0100 Subject: [PATCH] convert(fp8->int4): --indir copies the full metadata set and resumes interrupted passes (#383) Two gaps in the local path, both raised in #383: - The main --indir pass copied only config.json while the download path copies four files; without tokenizer.json the converted container can't run chat/serve. Copy the same four, print what was copied, and warn when the source is missing any (tokenizer.json called out explicitly). - Local passes restarted from shard 0 on every rerun. out-NNNNN names count EMITTED shards, not input indexes (shards with no relevant tensors emit nothing), so the download path's exists-check can't be mirrored directly: a sidecar manifest per prefix records input -> output (or empty) plus the conversion parameters, written atomically after each shard. Rerun skips what matches and refuses a parameter mismatch on the same outdir instead of mixing containers (the #355 failure mode). Co-Authored-By: Claude Fable 5 --- c/tools/convert_fp8_to_int4.py | 66 +++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index 9cd6937..bef3f74 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -397,24 +397,72 @@ def main(): # Ora il ramo locale rispecchia il download path: prefisso corretto, # flag passate, shard vuoti saltati. prefix = "out-mtp-" if a.mtp else "out-idx-" if a.indexer else "out-" - n = 0 + # RIPRESA (#383): i nomi out-NNNNN contano gli shard EMESSI, non l'indice di + # input (gli shard senza tensori rilevanti non producono file), quindi "il + # file esiste" non basta per saltare il lavoro gia' fatto. Un manifest + # sidecar ricorda input -> output (o "vuoto") e con quali parametri: la + # ripresa salta solo cio' che combacia, e parametri diversi sulla stessa + # outdir vengono rifiutati invece di mescolare container (il modo #355). + # EN: RESUME (#383): out-NNNNN names count EMITTED shards, not the input + # EN: index (shards with no relevant tensors emit no file), so "the file + # EN: exists" is not enough to skip completed work. A sidecar manifest + # EN: records input -> output (or "empty") plus the conversion parameters: + # EN: resume skips only what matches, and different parameters on the same + # EN: outdir are refused instead of mixing containers (the #355 failure mode). + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map} + prog_path = os.path.join(a.outdir, f".{prefix}progress.json") + prog = {} + if os.path.exists(prog_path): + try: prog = json.loads(open(prog_path).read()) + except (OSError, ValueError): prog = {} + if prog and prog.get("params") != params: + print(f"ERROR: {prog_path} records a conversion with {prog.get('params')};\n" + f" this run uses {params}. Refusing to mix conversions in the same " + f"outdir — use a fresh --outdir (or delete the manifest and the " + f"{prefix}*.safetensors shards to redo).") + return + done = prog.setdefault("shards", {}); prog["params"] = params + n = 0; fresh = 0; skipped = 0 for i, sp in enumerate(shards): + key = os.path.basename(sp) + prev = done.get(key) # None = mai visto; "" = visto, vuoto; nome = emesso + if prev is not None and (prev == "" or os.path.exists(os.path.join(a.outdir, prev))): + if prev: n += 1 + skipped += 1 + continue out = {} convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, keep_mtp=a.mtp, keep_idx=a.indexer, group_size=a.group_size, bits_map=bits_map) if not out: # shard senza MTP/idx: niente file (come il download path) - continue - save_file(out, os.path.join(a.outdir, f"{prefix}{n:05d}.safetensors")) - n += 1 - # config/tokenizer solo per la conversione principale — i passaggi mtp/idx - # vanno nella stessa outdir di un container gia' completo di metadati. + done[key] = "" + else: + name = f"{prefix}{n:05d}.safetensors" + save_file(out, os.path.join(a.outdir, name)) + done[key] = name; n += 1; fresh += 1 + tmp_prog = prog_path + ".tmp" # scrittura atomica: una ripresa non vede mai un manifest mezzo scritto + with open(tmp_prog, "w") as f: json.dump(prog, f, indent=1) # EN: atomic write: a resume never sees a half-written manifest + os.replace(tmp_prog, prog_path) + if skipped: print(f"[RESUME] {skipped} shard(s) already done in {a.outdir}, skipped") + # metadati per la conversione principale: gli stessi quattro file del download + # path — senza tokenizer.json chat/serve non partono. I passaggi mtp/idx vanno + # nella stessa outdir di un container gia' completo di metadati. + # EN: metadata for the main pass: the same four files as the download path — + # EN: chat/serve won't start without tokenizer.json. The mtp/idx passes target + # EN: an outdir whose container already has its metadata. if not a.mtp and not a.indexer: - for fn in ["config.json"]: + copied, missing = [], [] + for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json"]: src = os.path.join(a.indir, fn) - if os.path.exists(src): shutil.copy(src, a.outdir) + if os.path.exists(src): shutil.copy(src, a.outdir); copied.append(fn) + else: missing.append(fn) + print(f"[META] copied from {a.indir}: {', '.join(copied) if copied else 'nothing'}") + if missing: + print(f"[META] WARNING: not found in {a.indir}: {', '.join(missing)}" + + (" — chat/serve need tokenizer.json" if "tokenizer.json" in missing else "")) tag = "MTP" if a.mtp else "indexer" if a.indexer else "main" - print(f"converted {n} {tag} shard(s) -> {a.outdir} ({prefix}NNNNN)") + print(f"converted {fresh} {tag} shard(s), {n} in container -> {a.outdir} ({prefix}NNNNN)") return # reale: scarica shard per shard, converte, cancella