diff --git a/c/openai_server.py b/c/openai_server.py index 1dd987b..ae1032c 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -468,6 +468,9 @@ def generation_options(body, limit): raise APIError(400, "`response_format.type` must be \"text\", \"json_object\", " "\"json_schema\" or \"gbnf\".", "response_format", "unsupported_value") + if grammar is not None and len(grammar.encode("utf-8")) > (1 << 20): + raise APIError(400, "`response_format` grammar/schema exceeds 1 MiB.", + "response_format", "invalid_value") maximum = body.get("max_completion_tokens") maximum_param = "max_completion_tokens" diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py index 7cbbc04..8000b40 100644 --- a/c/tests/test_openai_server.py +++ b/c/tests/test_openai_server.py @@ -99,6 +99,16 @@ class TemplateTest(unittest.TestCase): generation_options({"response_format": {"type": "yaml"}}, 8) with self.assertRaises(APIError): generation_options({"response_format": {"type": "json_schema", "json_schema": {}}}, 8) + with self.assertRaises(APIError): # non-dict response_format + generation_options({"response_format": "json"}, 8) + with self.assertRaises(APIError): # empty gbnf + generation_options({"response_format": {"type": "gbnf", "grammar": " "}}, 8) + with self.assertRaises(APIError): # oversized grammar (> 1 MiB pre-check) + generation_options({"response_format": {"type": "gbnf", "grammar": "x" * ((1 << 20) + 1)}}, 8) + # malformed GBNF passes the gateway by design: the ENGINE fail-softs it + # (draft source only — bad grammar costs the speedup, never the request) + opts = generation_options({"response_format": {"type": "gbnf", "grammar": "not a grammar ::="}}, 8) + self.assertEqual(opts[3], "not a grammar ::=") class ProtocolTest(unittest.TestCase): diff --git a/docs/grammar-draft.md b/docs/grammar-draft.md index 707fe5c..6140cbb 100644 --- a/docs/grammar-draft.md +++ b/docs/grammar-draft.md @@ -83,3 +83,27 @@ this implementation adds: forced spans as a **draft source verified in the targe own forward** (lossless even under a wrong grammar, composes with MTP/n-gram in one union batch), deployed where the win is denominated in expert I/O rather than forward passes. + +## Server usage: `response_format` (OpenAI API) + +The gateway (`openai_server.py`) turns `response_format` into a **per-request** +grammar carried to the engine over the `SUBMIT` protocol (optional 7th header +field, `gbytes`; 6-field headers from older clients remain valid — the field is +additive and back-compatible in both directions): + +```jsonc +{"response_format": {"type": "json_object"}} // generic JSON grammar +{"response_format": {"type": "json_schema", + "json_schema": {"schema": { ... }}}} // compiled by schema_gbnf.h +{"response_format": {"type": "gbnf", "grammar": "root ::= ..."}} // raw GBNF (extension) +``` + +Semantics are identical to `GRAMMAR=`/`SCHEMA=`: a **draft source, never a +sampling constraint**. A schema outside the supported subset, or malformed GBNF, +costs the speedup — never the request, never the output. Drafting engages for +greedy requests (`temperature: 0`); sampled requests run undrafted. Compile +overhead is negligible: ~8 µs/request for a typical schema, ~18 µs at the +32-level nesting cap (measured, M3 Max). Grammar payloads are capped at 1 MiB. +As with MTP (#100), a drafted greedy run may differ from an undrafted one in +near-tie tokens (the verify forward has a different batch shape); each output +is a valid greedy stream of its own forward shapes.