Don't trust a converted checkpoint's config for the stop set

The engine armed its stop tokens from config.json's eos_token_id and nothing
else. That trusts metadata written by third-party conversion tooling, which is
a thing we already know goes wrong: the README documents a mirror shipping
int4 MTP heads that silently give 0% draft acceptance. GLM-5.2 declares THREE
eos ids (<|endoftext|>, <|user|>, <|observation|>); a converter that rewrites
config.json with a reduced list leaves the engine stopping on fewer tokens
than the model emits, and the missed ones get detokenized and printed into the
chat as literal text while generation runs past the end of the turn.

Two independent defenses:

  - eos_token_id is now unioned with generation_config.json, which is
    HuggingFace's authority for generation (config.json often carries a
    partial legacy copy). An extra stop is harmless; a missing one is not.

  - every added-token the TOKENIZER marks "special":true is armed as a stop,
    whatever the configs say. Those are control tokens (<|user|>, <|assistant|>,
    <sop>, [gMASK], the image/video/audio markers) and are never legitimate
    content in a reply -- GLM itself lists three of them as official eos.
    <think>/<tool_call>/<arg_key> are "special":false and are deliberately NOT
    swept up: they are real output. tok.h was parsing added_tokens but throwing
    the "special" flag away, so the distinction wasn't available to anyone.

On the real per-row checkpoint this takes the armed set from 3 to 18:
  [stop] 18 stop tokens: 154820 154827 154829 154821 ... (15 from the
  tokenizer's special set)

Honesty about scope: this is hygiene for a class of bug, NOT a fix for the
trailing-junk report on #298 that prompted it. I hypothesised @woolcoxm's g64
checkpoint had lost eos ids in conversion; he checked, and it hadn't -- his
config arms all three correctly. The emit path is also innocent: is_stop() is
checked BEFORE emit() at every one of the four call sites (4215, 4256, 4908,
4987), so a correctly-armed stop cannot be printed. His trailing junk is still
unexplained and is more likely quantization noise. What this commit buys is
that a checkpoint we don't control cannot leak control tokens into a reply,
which was true before and is not now.

tests/test_stops.c covers both defenses: the union, a missing
generation_config.json, BOTH configs mutilated (the tokenizer still stops all
five control tokens while leaving <think> alone), and T=NULL (the validation
path keeps config-only behaviour).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

c/glm.c

@@ -1277,6 +1277,35 @@ static void load_cfg(Cfg *c, const char *snap){
     if(eo){ if(eo->t==J_NUM) c->stop_ids[c->n_stop++]=(int)eo->num;
             else if(eo->t==J_ARR) for(int i=0;i<eo->len && c->n_stop<8;i++)
                 c->stop_ids[c->n_stop++]=(int)eo->kids[i]->num; }
+    /* generation_config.json e' il file AUTOREVOLE per la generazione secondo HuggingFace:
+     * config.json ne porta spesso una copia legacy o parziale. Un tool di conversione che
+     * rigenera un config.json ridotto lascia il motore fermo su MENO stop del dovuto, e i
+     * token di controllo che restano finiscono stampati in chat come testo (woolcoxm, #298:
+     * "the stop token being printed to chat", verificato sui token id). Unione dei due:
+     * uno stop in piu' non fa danno, uno in meno si' -- e chi converte i pesi non siamo noi.
+     * EN: generation_config.json is HF's authority for generation; config.json often carries
+     * a partial legacy copy. Union both -- an extra stop is harmless, a missing one is not. */
+    { char gp[2100]; snprintf(gp,sizeof(gp),"%s/generation_config.json",snap);
+      FILE *gf=fopen(gp,"rb");                  /* assente = nessun problema: e' opzionale */
+      if(gf){
+        fseek(gf,0,SEEK_END); long gn=ftell(gf); fseek(gf,0,SEEK_SET);
+        if(gn>0){
+            char *gb=malloc(gn+1); size_t gg=fread(gb,1,gn,gf); gb[gg]=0;
+            char *ga=NULL; jval *gr=json_parse(gb,&ga);
+            jval *ge=gr?json_get(gr,"eos_token_id"):NULL;
+            if(ge){
+                int add[8], na=0;
+                if(ge->t==J_NUM) add[na++]=(int)ge->num;
+                else if(ge->t==J_ARR) for(int i=0;i<ge->len && na<8;i++) add[na++]=(int)ge->kids[i]->num;
+                for(int i=0;i<na && c->n_stop<8;i++){
+                    int dup=0; for(int j=0;j<c->n_stop;j++) if(c->stop_ids[j]==add[i]) dup=1;
+                    if(!dup) c->stop_ids[c->n_stop++]=add[i];
+                }
+            }
+            free(ga); free(gb);
+        }
+        fclose(gf);
+      } }
     /* DSA lightning indexer: parametri + tipo per-layer (lista esplicita o formula freq/offset) */
     c->index_topk=gi(r,"index_topk"); c->index_nh=gi(r,"index_n_heads"); c->index_hd=gi(r,"index_head_dim");
     { jval *it=json_get(r,"indexer_types");
@@ -4107,18 +4136,34 @@ static int pick_tok(const float *lo, int V, int ban){
 
 /* stop-set attivo (popolato da run_text/run_serve dal config; vuoto in validazione,
  * dove si genera un numero fisso di token da confrontare con l'oracolo) */
-static int g_stop[9], g_nstop=0;
+static int g_stop[64], g_nstop=0;   /* config eos + ogni added-token "special" del tokenizer */
 static void repin_pass_limit(Model *m,int limit);
 static void repin_pass(Model *m){ repin_pass_limit(m,16); }
 static inline int is_stop(int t){ for(int i=0;i<g_nstop;i++) if(t==g_stop[i]) return 1; return 0; }
-static void stops_arm(const Cfg *c, int tok_eos){
+/* T=NULL -> solo gli stop del config (validazione/oracolo, dove il tokenizer non serve). */
+static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){
     g_nstop=0;
-    for(int i=0;i<c->n_stop;i++) g_stop[g_nstop++]=c->stop_ids[i];
-    if(tok_eos>=0 && !is_stop(tok_eos)) g_stop[g_nstop++]=tok_eos;
+    for(int i=0;i<c->n_stop && g_nstop<64;i++) g_stop[g_nstop++]=c->stop_ids[i];
+    if(tok_eos>=0 && !is_stop(tok_eos) && g_nstop<64) g_stop[g_nstop++]=tok_eos;
+    int nsp=0;
+    /* DIFESA IN PROFONDITA' (woolcoxm, #298): il tokenizer marca "special":true i token di
+     * CONTROLLO -- <|user|>, <|assistant|>, <|observation|>, <sop>, [gMASK], i marker
+     * image/video/audio. Nessuno di questi e' contenuto legittimo di una risposta: se il
+     * modello ne emette uno, il turno e' finito (infatti GLM ne elenca tre fra gli eos
+     * ufficiali). Senza questo, uno di quei token non elencato nel config veniva
+     * DETOKENIZZATO E STAMPATO IN CHAT come testo, e la generazione proseguiva oltre la
+     * fine reale -- l'"added stuff on the end" riportato su un checkpoint convertito.
+     * Fidarsi del config di pesi convertiti da terzi e' precisamente cio' che non
+     * possiamo controllare; il flag del tokenizer lo possiamo leggere.
+     * NB: <think>/<tool_call>/<arg_key> hanno "special":false e restano contenuto vero. */
+    if(T) for(int id=0; id<T->n_ids && g_nstop<64; id++)
+        if(T->id_special[id] && !is_stop(id)){ g_stop[g_nstop++]=id; nsp++; }
     fprintf(stderr,"[stop] %d stop tokens:",g_nstop);
     for(int i=0;i<g_nstop;i++) fprintf(stderr," %d",g_stop[i]);
+    if(nsp) fprintf(stderr," (%d from the tokenizer's special set)",nsp);
     fprintf(stderr,"\n");
 }
+static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c,tok_eos,NULL); }
 
 /* decode greedy con SELF-SPECULATION n-gram: LOSSLESS (output identico al greedy puro).
  * Ogni forward verifica fino a g_draft token proposti dal contesto: i token accettati
@@ -4399,7 +4444,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){
     Cfg *c=&m->c; char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
     Tok T; tok_load(&T,tkp);
     int eos=tok_id_of(&T,"<|endoftext|>");
-    stops_arm(&m->c, eos);
+    stops_arm_tok(&m->c, eos, &T);
     grammar_setup(&T);                   /* metodo F: GRAMMAR=file.gbnf (#48) */
     if(g_temp<0) g_temp=0.7f;            /* auto: 0.7, NON l'1.0 ufficiale — la coda della
                                           * distribuzione int4 e' rumore di quantizzazione */
@@ -4869,7 +4914,7 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx,
 
 static void run_serve_mux(Model *m, const char *snap){
     char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
-    Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm(&m->c,eos);
+    Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm_tok(&m->c,eos,&T);
     g_draft=0; /* one scheduler owns every forward; MTP/speculation is not ragged-safe */
     int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096;
     int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1;
@@ -4970,7 +5015,7 @@ static void run_serve(Model *m, const char *snap){
     char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
     Tok T; tok_load(&T,tkp);
     int eos=tok_id_of(&T,"<|endoftext|>");
-    stops_arm(&m->c, eos);
+    stops_arm_tok(&m->c, eos, &T);
     grammar_setup(&T);                   /* metodo F: GRAMMAR=file.gbnf (#48) */
     if(g_temp<0) g_temp=0.7f;            /* auto: 0.7, NON l'1.0 ufficiale — la coda della
                                           * distribuzione int4 e' rumore di quantizzazione */