diff --git a/c/openai_server.py b/c/openai_server.py index d55fbc8..f56e20a 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -406,7 +406,10 @@ def generation_options(body, limit): maximum = body.get("max_tokens") maximum_param = "max_tokens" if maximum is None: - maximum = min(256, limit) + # Client omitted max_tokens: honor the operator's configured budget (--max-tokens / + # --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382). + # Generation still ends at EOS, so this is a cap, not a target. + maximum = limit temperature = body.get("temperature") top_p = body.get("top_p") temperature = 0.7 if temperature is None else temperature diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index fb2fb73..9cd6937 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -299,6 +299,16 @@ def main(): if bits_map: print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items()))) + # Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il + # default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e' + # gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job + # da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything. + mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model" + grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \ + (f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row") + print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | " + f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}") + if a.selftest_nvfp4: import torch # 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi.