diff --git a/c/glm.c b/c/glm.c index 00c7076..3237c0a 100644 --- a/c/glm.c +++ b/c/glm.c @@ -4115,6 +4115,19 @@ static void stops_arm(const Cfg *c, int tok_eos){ g_nstop=0; for(int i=0;in_stop;i++) g_stop[g_nstop++]=c->stop_ids[i]; if(tok_eos>=0 && !is_stop(tok_eos)) g_stop[g_nstop++]=tok_eos; + /* In serve mode (API), keep only <|endoftext|> as a stop token. The tokens + * <|user|> and <|observation|> are role markers that the Python server + * handles — keeping them as stop tokens causes the engine to halt + * prematurely when the model tries to emit blocks, because + * int4-quantized logits can be noisy enough that argmax picks a stop-token + * ID instead of the correct tool-call token. (#401) */ + if(getenv("SERVE")){ + int kept=0; + for(int i=0;i