From f8e4dc95b5add6cd55105b25731293d2c17fae44 Mon Sep 17 00:00:00 2001
From: JustVugg <JustVugg@users.noreply.github.com>
Date: Sat, 18 Jul 2026 12:28:27 +0200
Subject: [PATCH] serve: honor --ngen when the client omits max_tokens;
 convert: print the resolved plan (#382, #383)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two operator-surprise fixes:

openai_server.py (#382, LordMZTE): a request without max_tokens defaulted to
min(256, limit) — so `coli serve --ngen 32768` still cut every answer at 256
tokens for clients that omit the field. The operator's configured budget IS the
default now; generation still ends at EOS, so it's a cap, not a target.

convert_fp8_to_int4.py (#383, bokiko): the resolved conversion plan (mode,
source, ebits/io/x bits, grouped-vs-per-row) prints as a [PLAN] line before any
work. The two traps in #383 — --mtp defaulting ebits to 8 with the grouped
branch silently gated off, and --mtp appearing to ignore --indir — are already
defused by the #355 fix (the --mtp pass emits ONLY head tensors on the local
path), but a 3.5-hour job must show its plan at second 1, not in a post-hoc
size sanity check. bokiko's exact invocation now prints:
  [PLAN] mode: MTP head only | source: local ./fp8 | experts 8-bit, ... |
         PER-ROW (grouped branch needs bits<=4; ebits=8 disables it)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 c/openai_server.py             |  5 ++++-
 c/tools/convert_fp8_to_int4.py | 10 ++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/c/openai_server.py b/c/openai_server.py
index d55fbc8..f56e20a 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -406,7 +406,10 @@ def generation_options(body, limit):
         maximum = body.get("max_tokens")
         maximum_param = "max_tokens"
     if maximum is None:
-        maximum = min(256, limit)
+        # Client omitted max_tokens: honor the operator's configured budget (--max-tokens /
+        # --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382).
+        # Generation still ends at EOS, so this is a cap, not a target.
+        maximum = limit
     temperature = body.get("temperature")
     top_p = body.get("top_p")
     temperature = 0.7 if temperature is None else temperature
diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py
index fb2fb73..9cd6937 100644
--- a/c/tools/convert_fp8_to_int4.py
+++ b/c/tools/convert_fp8_to_int4.py
@@ -299,6 +299,16 @@ def main():
     if bits_map:
         print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items())))
 
+    # Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il
+    # default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e'
+    # gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job
+    # da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything.
+    mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model"
+    grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \
+          (f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row")
+    print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | "
+          f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}")
+
     if a.selftest_nvfp4:
         import torch
         # 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi.