Two operator-surprise fixes: openai_server.py (#382, LordMZTE): a request without max_tokens defaulted to min(256, limit) — so `coli serve --ngen 32768` still cut every answer at 256 tokens for clients that omit the field. The operator's configured budget IS the default now; generation still ends at EOS, so it's a cap, not a target. convert_fp8_to_int4.py (#383, bokiko): the resolved conversion plan (mode, source, ebits/io/x bits, grouped-vs-per-row) prints as a [PLAN] line before any work. The two traps in #383 — --mtp defaulting ebits to 8 with the grouped branch silently gated off, and --mtp appearing to ignore --indir — are already defused by the #355 fix (the --mtp pass emits ONLY head tensors on the local path), but a 3.5-hour job must show its plan at second 1, not in a post-hoc size sanity check. bokiko's exact invocation now prints: [PLAN] mode: MTP head only | source: local ./fp8 | experts 8-bit, ... | PER-ROW (grouped branch needs bits<=4; ebits=8 disables it) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+4
-1
@@ -406,7 +406,10 @@ def generation_options(body, limit):
|
||||
maximum = body.get("max_tokens")
|
||||
maximum_param = "max_tokens"
|
||||
if maximum is None:
|
||||
maximum = min(256, limit)
|
||||
# Client omitted max_tokens: honor the operator's configured budget (--max-tokens /
|
||||
# --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382).
|
||||
# Generation still ends at EOS, so this is a cap, not a target.
|
||||
maximum = limit
|
||||
temperature = body.get("temperature")
|
||||
top_p = body.get("top_p")
|
||||
temperature = 0.7 if temperature is None else temperature
|
||||
|
||||
@@ -299,6 +299,16 @@ def main():
|
||||
if bits_map:
|
||||
print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items())))
|
||||
|
||||
# Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il
|
||||
# default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e'
|
||||
# gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job
|
||||
# da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything.
|
||||
mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model"
|
||||
grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \
|
||||
(f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row")
|
||||
print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | "
|
||||
f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}")
|
||||
|
||||
if a.selftest_nvfp4:
|
||||
import torch
|
||||
# 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi.
|
||||
|
||||
Reference in New Issue
Block a user