From 70fb5b00f388685a015d5e94a3c75916f40cf3a3 Mon Sep 17 00:00:00 2001 From: Colibri Developer Date: Sun, 19 Jul 2026 13:38:57 -0600 Subject: [PATCH 1/2] fix(api): pass tools and tool_choice as parameters to generation() to prevent duplicate extraction The generation() method was re-extracting tools from the raw request body, bypassing the filtering that render_chat() applies when tool_choice is a function object (forced function call). This caused parse_tool_calls() to receive the unfiltered tool list, producing incorrect or missing tool_calls in the response. - Add tools and tool_choice parameters to generation() signature - Pass them from chat_completion() after render_chat() processing - Add early structural validation of tools in generation_options() for clear HTTP 400 errors on malformed input --- c/openai_server.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/c/openai_server.py b/c/openai_server.py index ba19e43..d8703a8 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -365,6 +365,27 @@ def generation_options(body, limit): if body.get("n", 1) != 1: raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value") # `tools`/`functions` are handled by render_chat (declaration) + parse_tool_calls (output). + # Validate tools/functions structure early so malformed input fails with a clear error. + tools_raw = body.get("tools") or body.get("functions") + if tools_raw is not None: + if not isinstance(tools_raw, list): + raise APIError(400, "`tools` must be a non-empty array.", "tools", "invalid_value") + if not tools_raw: + raise APIError(400, "`tools` must be a non-empty array.", "tools", "invalid_value") + for idx, tool in enumerate(tools_raw): + if not isinstance(tool, dict): + raise APIError(400, f"Each tool must be an object, got {type(tool).__name__} at index {idx}.", + f"tools.{idx}", "invalid_value") + fn = tool.get("function", tool) if isinstance(tool, dict) else {} + if not isinstance(fn, dict): + raise APIError(400, f"Tool function must be an object at index {idx}.", + f"tools.{idx}.function", "invalid_value") + if not fn.get("name"): + raise APIError(400, f"Each tool must have a `name` at index {idx}.", + f"tools.{idx}.function.name", "invalid_value") + if not isinstance(fn["name"], str): + raise APIError(400, f"Tool `name` must be a string at index {idx}.", + f"tools.{idx}.function.name", "invalid_value") choice = body.get("tool_choice") if choice is not None: if isinstance(choice, str): @@ -829,7 +850,7 @@ class APIHandler(BaseHTTPRequestHandler): except OSError: pass - def generation(self, body, prompt, request_id, chat): + def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=None): # COLI_DEBUG tees the engine transaction to stderr: 1 = decoded output stream only, # 2 = both sides (rendered prompt + output). render_chat already folds prior turns and # tool results into `prompt`, so level 2 is the full conversation the engine saw. @@ -841,8 +862,8 @@ class APIHandler(BaseHTTPRequestHandler): sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n") sys.stderr.flush() maximum, temperature, top_p = generation_options(body, self.server.max_tokens) - tools = (body.get("tools") or body.get("functions") or None) if chat else None - if body.get("tool_choice") == "none": + # tools and tool_choice come from chat_completion() already processed/filtered + if chat and tool_choice == "none": tools = None # client forbade tools: never surface tool_calls cache_slot = body.get("cache_slot") if (cache_slot is not None and @@ -1044,9 +1065,10 @@ class APIHandler(BaseHTTPRequestHandler): if not isinstance(enable_thinking, bool): raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking") tools = body.get("tools") or body.get("functions") or None + tool_choice = body.get("tool_choice") prompt = render_chat(body.get("messages"), enable_thinking, reasoning_effort, tools, - body.get("tool_choice")) - self.generation(body, prompt, request_id, True) + tool_choice) + self.generation(body, prompt, request_id, True, tools, tool_choice) def completion(self, body, request_id): prompt = body.get("prompt") From 26bd8b403a8db04e684e4006523885385a2305a2 Mon Sep 17 00:00:00 2001 From: Colibri Developer Date: Sun, 19 Jul 2026 13:39:11 -0600 Subject: [PATCH 2/2] fix(engine): filter non-EOS stop tokens in serve mode to prevent premature halt on tool calls GLM-5.2's config defines three stop tokens: <|endoftext|>, <|user|>, and <|observation|>. In serve mode, when the model generates blocks, int4-quantized logit noise can cause argmax to pick a <|user|> or <|observation|> token ID, immediately stopping generation. The <|user|> and <|observation|> tokens are role markers handled by the Python API server, not the C engine. Filter them out in stops_arm() when SERVE=1, keeping only <|endoftext|> as the stop token. - Add SERVE-mode guard in stops_arm() that retains only the EOS token - Log the number of filtered tokens for diagnostics --- c/glm.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/c/glm.c b/c/glm.c index 00c7076..3237c0a 100644 --- a/c/glm.c +++ b/c/glm.c @@ -4115,6 +4115,19 @@ static void stops_arm(const Cfg *c, int tok_eos){ g_nstop=0; for(int i=0;in_stop;i++) g_stop[g_nstop++]=c->stop_ids[i]; if(tok_eos>=0 && !is_stop(tok_eos)) g_stop[g_nstop++]=tok_eos; + /* In serve mode (API), keep only <|endoftext|> as a stop token. The tokens + * <|user|> and <|observation|> are role markers that the Python server + * handles — keeping them as stop tokens causes the engine to halt + * prematurely when the model tries to emit blocks, because + * int4-quantized logits can be noisy enough that argmax picks a stop-token + * ID instead of the correct tool-call token. (#401) */ + if(getenv("SERVE")){ + int kept=0; + for(int i=0;i