Merge pull request #112 from TheGameVIX/agent/codex-exec-optimizer-main
fix(codex): support exec optimizer backend
This commit is contained in:
Yifan Yang
+1
-1
@@ -394,7 +394,7 @@ python scripts/train.py --config configs/searchqa/default.yaml</code></pre>
|
||||
<tr><td><code>claude_chat</code></td><td>Yes</td><td>Yes</td><td>Runs an installed, authenticated Claude Code CLI via <code>claude -p</code>; not a direct Anthropic API client.</td></tr>
|
||||
<tr><td><code>qwen_chat</code></td><td>Yes</td><td>Yes</td><td>Local or hosted Qwen-compatible server.</td></tr>
|
||||
<tr><td><code>minimax_chat</code></td><td>Yes</td><td>Yes</td><td>MiniMax chat endpoint.</td></tr>
|
||||
<tr><td><code>codex_exec</code></td><td>No</td><td>Supported adapters only</td><td>Executes Codex as a target agent.</td></tr>
|
||||
<tr><td><code>codex_exec</code></td><td>Yes</td><td>Supported adapters only</td><td>Executes Codex for optimizer calls and as a target agent where supported.</td></tr>
|
||||
<tr><td><code>claude_code_exec</code></td><td>No</td><td>Supported adapters only</td><td>Executes Claude Code as a target agent.</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
@@ -190,7 +190,7 @@ not via a base class subclass. Supported values (as of this writing):
|
||||
| `qwen_chat` | ✓ | ✓ |
|
||||
| `minimax_chat` | ✓ | ✓ |
|
||||
| `openai_compatible` | ✓ | ✓ |
|
||||
| `codex_exec` | — | ✓ |
|
||||
| `codex_exec` | ✓ | ✓ |
|
||||
| `claude_code_exec` | — | ✓ |
|
||||
|
||||
See `skillopt/model/backend_config.py` for the live whitelist and
|
||||
|
||||
@@ -16,7 +16,7 @@ selecting the generic OpenAI-compatible backend.
|
||||
| `claude_chat` | ✓ | ✓ |
|
||||
| `qwen_chat` | ✓ | ✓ |
|
||||
| `minimax_chat` | ✓ | ✓ |
|
||||
| `codex_exec` | — | ✓ |
|
||||
| `codex_exec` | ✓ | ✓ |
|
||||
| `claude_code_exec` | — | ✓ |
|
||||
|
||||
MiniMax currently has one shared deployment. `model.minimax_model` is applied
|
||||
@@ -28,7 +28,7 @@ MiniMax optimizer model and a different target model.
|
||||
| `model.backend` | str | `azure_openai` | Backward-compatible high-level run label |
|
||||
| `model.optimizer` | str | `gpt-5.5` | Optimizer deployment/model |
|
||||
| `model.target` | str | `gpt-5.5` | Target deployment/model |
|
||||
| `model.optimizer_backend` | str | `openai_chat` | Optimizer client path; chat backends only |
|
||||
| `model.optimizer_backend` | str | `openai_chat` | Optimizer client path; chat backends plus `codex_exec` |
|
||||
| `model.target_backend` | str | `openai_chat` | Target client path; chat or exec backend |
|
||||
| `model.reasoning_effort` | str | `medium` | Shared reasoning effort |
|
||||
| `model.rewrite_reasoning_effort` | str | empty | Optional full-rewrite effort override |
|
||||
|
||||
@@ -320,8 +320,10 @@ def main() -> None:
|
||||
cfg.setdefault("optimizer_backend", "claude_chat")
|
||||
cfg.setdefault("target_backend", "claude_chat")
|
||||
elif backend in {"codex", "codex_exec"}:
|
||||
cfg.setdefault("optimizer_backend", "openai_chat")
|
||||
cfg.setdefault("target_backend", "codex_exec")
|
||||
if not _has_model_override("model.optimizer_backend", "optimizer_backend"):
|
||||
cfg["optimizer_backend"] = "codex_exec"
|
||||
if not _has_model_override("model.target_backend", "target_backend"):
|
||||
cfg["target_backend"] = "codex_exec"
|
||||
elif backend == "claude_code_exec":
|
||||
cfg.setdefault("optimizer_backend", "openai_chat")
|
||||
cfg.setdefault("target_backend", "claude_code_exec")
|
||||
|
||||
+4
-2
@@ -438,8 +438,10 @@ def load_config(args: argparse.Namespace) -> dict:
|
||||
flat.setdefault("optimizer_backend", "claude_chat")
|
||||
flat.setdefault("target_backend", "claude_chat")
|
||||
elif backend in {"codex", "codex_exec"}:
|
||||
flat.setdefault("optimizer_backend", "openai_chat")
|
||||
flat.setdefault("target_backend", "codex_exec")
|
||||
if not _has_model_override("model.optimizer_backend", "optimizer_backend"):
|
||||
flat["optimizer_backend"] = "codex_exec"
|
||||
if not _has_model_override("model.target_backend", "target_backend"):
|
||||
flat["target_backend"] = "codex_exec"
|
||||
elif backend == "claude_code_exec":
|
||||
flat.setdefault("optimizer_backend", "openai_chat")
|
||||
flat.setdefault("target_backend", "claude_code_exec")
|
||||
|
||||
@@ -674,8 +674,10 @@ class ReflACTTrainer:
|
||||
optimizer_backend = optimizer_backend or "claude_chat"
|
||||
target_backend = target_backend or "claude_chat"
|
||||
elif backend in {"codex", "codex_exec"}:
|
||||
optimizer_backend = optimizer_backend or "openai_chat"
|
||||
target_backend = target_backend or "codex_exec"
|
||||
if optimizer_backend in (None, "", "openai_chat"):
|
||||
optimizer_backend = "codex_exec"
|
||||
if target_backend in (None, "", "openai_chat"):
|
||||
target_backend = "codex_exec"
|
||||
elif backend == "claude_code_exec":
|
||||
optimizer_backend = optimizer_backend or "openai_chat"
|
||||
target_backend = target_backend or "claude_code_exec"
|
||||
|
||||
@@ -50,6 +50,6 @@ evaluation:
|
||||
# ── Model ────────────────────────────────────────
|
||||
# Override only what differs from the inherited defaults.
|
||||
model:
|
||||
optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat
|
||||
target_backend: openai_chat # … plus codex_exec / claude_code_exec for target only
|
||||
optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat | codex_exec
|
||||
target_backend: openai_chat # chat backends plus codex_exec / claude_code_exec
|
||||
reasoning_effort: medium
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
|
||||
from skillopt.model import azure_openai as _openai
|
||||
from skillopt.model import claude_backend as _claude
|
||||
from skillopt.model import codex_backend as _codex
|
||||
from skillopt.model import minimax_backend as _minimax
|
||||
from skillopt.model import openai_compatible_backend as _openai_compat
|
||||
from skillopt.model import qwen_backend as _qwen
|
||||
@@ -41,10 +42,14 @@ def set_backend(name: str | None) -> str:
|
||||
set_target_backend("claude_chat")
|
||||
return "claude_chat"
|
||||
if normalized == "codex":
|
||||
set_optimizer_backend("openai_chat")
|
||||
set_optimizer_backend("codex_exec")
|
||||
set_target_backend("codex_exec")
|
||||
return "codex"
|
||||
if normalized in {"codex_exec", "claude_code_exec"}:
|
||||
if normalized == "codex_exec":
|
||||
set_optimizer_backend("codex_exec")
|
||||
set_target_backend("codex_exec")
|
||||
return normalized
|
||||
if normalized == "claude_code_exec":
|
||||
set_optimizer_backend("openai_chat")
|
||||
set_target_backend(normalized)
|
||||
return normalized
|
||||
@@ -73,7 +78,7 @@ def get_backend_name() -> str:
|
||||
return "qwen_chat"
|
||||
if optimizer == "openai_chat" and target == "openai_chat":
|
||||
return "azure_openai"
|
||||
if optimizer == "openai_chat" and target == "codex_exec":
|
||||
if optimizer == "codex_exec" and target == "codex_exec":
|
||||
return "codex"
|
||||
if optimizer == "openai_chat" and target == "qwen_chat":
|
||||
return "qwen_chat"
|
||||
@@ -132,6 +137,15 @@ def chat_optimizer(
|
||||
reasoning_effort=reasoning_effort,
|
||||
timeout=timeout,
|
||||
)
|
||||
if get_optimizer_backend() == "codex_exec":
|
||||
return _codex.chat_optimizer(
|
||||
system=system,
|
||||
user=user,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
retries=retries,
|
||||
stage=stage,
|
||||
timeout=timeout,
|
||||
)
|
||||
return _openai.chat_optimizer(
|
||||
system=system,
|
||||
user=user,
|
||||
@@ -265,6 +279,17 @@ def chat_optimizer_messages(
|
||||
return_message=return_message,
|
||||
timeout=timeout,
|
||||
)
|
||||
if get_optimizer_backend() == "codex_exec":
|
||||
return _codex.chat_optimizer_messages(
|
||||
messages=messages,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
retries=retries,
|
||||
stage=stage,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
return_message=return_message,
|
||||
timeout=timeout,
|
||||
)
|
||||
return _openai.chat_optimizer_messages(
|
||||
messages=messages,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
@@ -449,6 +474,17 @@ def get_token_summary() -> dict:
|
||||
summary[stage]["prompt_tokens"] += values["prompt_tokens"]
|
||||
summary[stage]["completion_tokens"] += values["completion_tokens"]
|
||||
summary[stage]["total_tokens"] += values["total_tokens"]
|
||||
codex_summary = _codex.get_token_summary()
|
||||
for stage, values in codex_summary.items():
|
||||
if stage == "_total":
|
||||
continue
|
||||
if stage not in summary:
|
||||
summary[stage] = values
|
||||
continue
|
||||
summary[stage]["calls"] += values["calls"]
|
||||
summary[stage]["prompt_tokens"] += values["prompt_tokens"]
|
||||
summary[stage]["completion_tokens"] += values["completion_tokens"]
|
||||
summary[stage]["total_tokens"] += values["total_tokens"]
|
||||
total = {
|
||||
"calls": 0,
|
||||
"prompt_tokens": 0,
|
||||
@@ -472,6 +508,7 @@ def reset_token_tracker() -> None:
|
||||
_qwen.reset_token_tracker()
|
||||
_minimax.reset_token_tracker()
|
||||
_openai_compat.reset_token_tracker()
|
||||
_codex.reset_token_tracker()
|
||||
|
||||
|
||||
def configure_azure_openai(
|
||||
@@ -622,6 +659,7 @@ def set_reasoning_effort(effort: str | None) -> None:
|
||||
_qwen.set_reasoning_effort(effort)
|
||||
_minimax.set_reasoning_effort(effort)
|
||||
_openai_compat.set_reasoning_effort(effort)
|
||||
_codex.set_reasoning_effort(effort)
|
||||
|
||||
|
||||
def set_target_deployment(deployment: str) -> None:
|
||||
@@ -630,6 +668,7 @@ def set_target_deployment(deployment: str) -> None:
|
||||
_qwen.set_target_deployment(deployment)
|
||||
_minimax.set_target_deployment(deployment)
|
||||
_openai_compat.set_target_deployment(deployment)
|
||||
_codex.set_target_deployment(deployment)
|
||||
|
||||
|
||||
def set_optimizer_deployment(deployment: str) -> None:
|
||||
@@ -637,3 +676,4 @@ def set_optimizer_deployment(deployment: str) -> None:
|
||||
_claude.set_optimizer_deployment(deployment)
|
||||
_qwen.set_optimizer_deployment(deployment)
|
||||
_openai_compat.set_optimizer_deployment(deployment)
|
||||
_codex.set_optimizer_deployment(deployment)
|
||||
|
||||
@@ -49,10 +49,18 @@ CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS = max(
|
||||
def set_optimizer_backend(backend: str) -> None:
|
||||
global OPTIMIZER_BACKEND
|
||||
OPTIMIZER_BACKEND = normalize_backend_name(backend or "openai_chat")
|
||||
if OPTIMIZER_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible"}:
|
||||
if OPTIMIZER_BACKEND not in {
|
||||
"openai_chat",
|
||||
"claude_chat",
|
||||
"qwen_chat",
|
||||
"minimax_chat",
|
||||
"openai_compatible",
|
||||
"codex_exec",
|
||||
}:
|
||||
raise ValueError(
|
||||
f"Unsupported optimizer backend: {OPTIMIZER_BACKEND!r}. "
|
||||
"Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', and 'openai_compatible'."
|
||||
"Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', "
|
||||
"'openai_compatible', and 'codex_exec'."
|
||||
)
|
||||
os.environ["OPTIMIZER_BACKEND"] = OPTIMIZER_BACKEND
|
||||
|
||||
@@ -81,7 +89,14 @@ def is_target_exec_backend() -> bool:
|
||||
|
||||
|
||||
def is_optimizer_chat_backend() -> bool:
|
||||
return OPTIMIZER_BACKEND in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible"}
|
||||
return OPTIMIZER_BACKEND in {
|
||||
"openai_chat",
|
||||
"claude_chat",
|
||||
"qwen_chat",
|
||||
"minimax_chat",
|
||||
"openai_compatible",
|
||||
"codex_exec",
|
||||
}
|
||||
|
||||
|
||||
def is_target_chat_backend() -> bool:
|
||||
|
||||
@@ -18,6 +18,7 @@ from skillopt.model.common import (
|
||||
CompatToolFunction,
|
||||
tracker,
|
||||
)
|
||||
from skillopt.model.backend_config import get_codex_exec_config
|
||||
|
||||
|
||||
CODEX_BIN = os.environ.get("CODEX_CLI_BIN", "codex")
|
||||
@@ -286,20 +287,21 @@ def _run_codex_exec(
|
||||
timeout: int | None,
|
||||
) -> tuple[str, dict[str, int]]:
|
||||
with tempfile.TemporaryDirectory(prefix="skillopt_codex_") as temp_dir:
|
||||
config = get_codex_exec_config()
|
||||
output_path = os.path.join(temp_dir, "last_message.txt")
|
||||
image_paths = _materialize_attachments(attachments, temp_dir)
|
||||
profile = str(config.get("profile") or os.environ.get("CODEX_PROFILE", "")).strip()
|
||||
reasoning_effort = str(REASONING_EFFORT or config.get("reasoning_effort") or "").strip()
|
||||
|
||||
command = [
|
||||
CODEX_BIN,
|
||||
str(config.get("path") or CODEX_BIN),
|
||||
"exec",
|
||||
"--json",
|
||||
"--ephemeral",
|
||||
"--profile",
|
||||
CODEX_PROFILE,
|
||||
"-c",
|
||||
"approval_policy=\"never\"",
|
||||
f"approval_policy={json.dumps(str(config.get('approval_policy') or 'never'))}",
|
||||
"--sandbox",
|
||||
CODEX_SANDBOX_MODE,
|
||||
str(config.get("sandbox") or CODEX_SANDBOX_MODE),
|
||||
"--skip-git-repo-check",
|
||||
"--cd",
|
||||
_default_working_directory(),
|
||||
@@ -309,8 +311,11 @@ def _run_codex_exec(
|
||||
output_path,
|
||||
]
|
||||
|
||||
if REASONING_EFFORT:
|
||||
command.extend(["-c", f"model_reasoning_effort={json.dumps(REASONING_EFFORT)}"])
|
||||
if profile:
|
||||
command.extend(["--profile", profile])
|
||||
|
||||
if reasoning_effort and reasoning_effort != "none":
|
||||
command.extend(["-c", f"model_reasoning_effort={json.dumps(reasoning_effort)}"])
|
||||
|
||||
schema_path = None
|
||||
if output_schema is not None:
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class _OpenAIClientStub:
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
|
||||
|
||||
def _install_openai_stub() -> None:
|
||||
if "openai" in sys.modules or importlib.util.find_spec("openai") is not None:
|
||||
return
|
||||
openai_stub = types.ModuleType("openai")
|
||||
openai_stub.AzureOpenAI = _OpenAIClientStub
|
||||
openai_stub.OpenAI = _OpenAIClientStub
|
||||
sys.modules["openai"] = openai_stub
|
||||
|
||||
|
||||
def _import_model_modules() -> tuple[Any, Any, Any, Any]:
|
||||
_install_openai_stub()
|
||||
import skillopt.model as model_module
|
||||
from skillopt.model import azure_openai, backend_config, codex_backend
|
||||
|
||||
return model_module, backend_config, codex_backend, azure_openai
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate_backend_state() -> Iterator[tuple[Any, Any, Any, Any]]:
|
||||
model_module, backend_config, codex_backend, azure_openai = _import_model_modules()
|
||||
optimizer_backend = backend_config.get_optimizer_backend()
|
||||
target_backend = backend_config.get_target_backend()
|
||||
env = {
|
||||
key: os.environ.get(key)
|
||||
for key in (
|
||||
"OPTIMIZER_BACKEND",
|
||||
"TARGET_BACKEND",
|
||||
"OPTIMIZER_DEPLOYMENT",
|
||||
"TARGET_DEPLOYMENT",
|
||||
)
|
||||
}
|
||||
yield model_module, backend_config, codex_backend, azure_openai
|
||||
backend_config.set_optimizer_backend(optimizer_backend)
|
||||
backend_config.set_target_backend(target_backend)
|
||||
for key, value in env.items():
|
||||
if value is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def test_codex_exec_can_be_optimizer_backend(
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
_model_module, backend_config, _codex_backend, _azure_openai = isolate_backend_state
|
||||
|
||||
backend_config.set_optimizer_backend("codex_exec")
|
||||
|
||||
assert backend_config.get_optimizer_backend() == "codex_exec"
|
||||
|
||||
|
||||
def test_set_backend_codex_uses_codex_for_optimizer_and_target(
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, backend_config, _codex_backend, _azure_openai = isolate_backend_state
|
||||
|
||||
assert model_module.set_backend("codex") == "codex"
|
||||
|
||||
assert backend_config.get_optimizer_backend() == "codex_exec"
|
||||
assert backend_config.get_target_backend() == "codex_exec"
|
||||
assert model_module.get_backend_name() == "codex"
|
||||
|
||||
|
||||
def test_chat_optimizer_routes_to_codex_backend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, backend_config, codex_backend, azure_openai = isolate_backend_state
|
||||
codex_calls: list[dict[str, Any]] = []
|
||||
|
||||
def fake_codex_optimizer(**kwargs: Any) -> tuple[str, dict[str, int]]:
|
||||
codex_calls.append(kwargs)
|
||||
return "codex result", {
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 2,
|
||||
"total_tokens": 3,
|
||||
}
|
||||
|
||||
def fail_openai_optimizer(**_kwargs: Any) -> tuple[str, dict[str, int]]:
|
||||
raise AssertionError("openai optimizer should not be called for codex_exec")
|
||||
|
||||
monkeypatch.setattr(codex_backend, "chat_optimizer", fake_codex_optimizer)
|
||||
monkeypatch.setattr(azure_openai, "chat_optimizer", fail_openai_optimizer)
|
||||
backend_config.set_optimizer_backend("codex_exec")
|
||||
|
||||
text, usage = model_module.chat_optimizer("system", "user", retries=1, timeout=5)
|
||||
|
||||
assert text == "codex result"
|
||||
assert usage["total_tokens"] == 3
|
||||
assert codex_calls[0]["system"] == "system"
|
||||
assert codex_calls[0]["user"] == "user"
|
||||
assert codex_calls[0]["timeout"] == 5
|
||||
|
||||
|
||||
def test_openai_compatible_still_allowed_as_optimizer_backend(
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
_model_module, backend_config, _codex_backend, _azure_openai = isolate_backend_state
|
||||
|
||||
backend_config.set_optimizer_backend("openai_compatible")
|
||||
|
||||
assert backend_config.get_optimizer_backend() == "openai_compatible"
|
||||
|
||||
|
||||
def test_chat_optimizer_routes_to_openai_compatible_when_selected(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, backend_config, codex_backend, azure_openai = isolate_backend_state
|
||||
from skillopt.model import openai_compatible_backend
|
||||
|
||||
compat_calls: list[dict[str, Any]] = []
|
||||
|
||||
def fake_compat_optimizer(**kwargs: Any) -> tuple[str, dict[str, int]]:
|
||||
compat_calls.append(kwargs)
|
||||
return "compat result", {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
|
||||
|
||||
def fail_codex_optimizer(**_kwargs: Any) -> tuple[str, dict[str, int]]:
|
||||
raise AssertionError("codex optimizer should not be called for openai_compatible")
|
||||
|
||||
def fail_openai_optimizer(**_kwargs: Any) -> tuple[str, dict[str, int]]:
|
||||
raise AssertionError("openai optimizer should not be called for openai_compatible")
|
||||
|
||||
monkeypatch.setattr(openai_compatible_backend, "chat_optimizer", fake_compat_optimizer)
|
||||
monkeypatch.setattr(codex_backend, "chat_optimizer", fail_codex_optimizer)
|
||||
monkeypatch.setattr(azure_openai, "chat_optimizer", fail_openai_optimizer)
|
||||
backend_config.set_optimizer_backend("openai_compatible")
|
||||
|
||||
text, usage = model_module.chat_optimizer("system", "user", retries=1, timeout=7)
|
||||
|
||||
assert text == "compat result"
|
||||
assert usage["total_tokens"] == 2
|
||||
assert compat_calls[0]["timeout"] == 7
|
||||
|
||||
|
||||
def test_chat_optimizer_messages_routes_to_codex_backend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, backend_config, codex_backend, azure_openai = isolate_backend_state
|
||||
codex_calls: list[dict[str, Any]] = []
|
||||
|
||||
def fake_codex_messages(**kwargs: Any) -> tuple[str, dict[str, int]]:
|
||||
codex_calls.append(kwargs)
|
||||
return "codex messages", {"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5}
|
||||
|
||||
def fail_openai_messages(**_kwargs: Any) -> tuple[str, dict[str, int]]:
|
||||
raise AssertionError("openai messages should not be called for codex_exec")
|
||||
|
||||
monkeypatch.setattr(codex_backend, "chat_optimizer_messages", fake_codex_messages)
|
||||
monkeypatch.setattr(azure_openai, "chat_optimizer_messages", fail_openai_messages)
|
||||
backend_config.set_optimizer_backend("codex_exec")
|
||||
|
||||
text, usage = model_module.chat_optimizer_messages(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
retries=1,
|
||||
tools=[{"name": "lookup"}],
|
||||
tool_choice="required",
|
||||
return_message=True,
|
||||
timeout=9,
|
||||
)
|
||||
|
||||
assert text == "codex messages"
|
||||
assert usage["total_tokens"] == 5
|
||||
assert codex_calls[0]["tools"] == [{"name": "lookup"}]
|
||||
assert codex_calls[0]["tool_choice"] == "required"
|
||||
assert codex_calls[0]["return_message"] is True
|
||||
assert codex_calls[0]["timeout"] == 9
|
||||
|
||||
|
||||
def test_codex_optimizer_does_not_change_openai_target_routing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, backend_config, codex_backend, azure_openai = isolate_backend_state
|
||||
|
||||
def fake_openai_target(**_kwargs: Any) -> tuple[str, dict[str, int]]:
|
||||
return "openai target", {"prompt_tokens": 1, "completion_tokens": 0, "total_tokens": 1}
|
||||
|
||||
def fail_codex_target(**_kwargs: Any) -> tuple[str, dict[str, int]]:
|
||||
raise AssertionError("codex target should not be called when target_backend=openai_chat")
|
||||
|
||||
monkeypatch.setattr(azure_openai, "chat_target", fake_openai_target)
|
||||
monkeypatch.setattr(codex_backend, "chat_target", fail_codex_target)
|
||||
backend_config.set_optimizer_backend("codex_exec")
|
||||
backend_config.set_target_backend("openai_chat")
|
||||
|
||||
text, usage = model_module.chat_target("system", "user", retries=1)
|
||||
|
||||
assert text == "openai target"
|
||||
assert usage["total_tokens"] == 1
|
||||
|
||||
|
||||
def test_get_backend_name_keeps_openai_compatible(
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, backend_config, _codex_backend, _azure_openai = isolate_backend_state
|
||||
|
||||
backend_config.set_optimizer_backend("openai_compatible")
|
||||
backend_config.set_target_backend("openai_compatible")
|
||||
|
||||
assert model_module.get_backend_name() == "openai_compatible"
|
||||
|
||||
|
||||
def test_token_summary_merges_codex_once_with_existing_backends(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, _backend_config, codex_backend, azure_openai = isolate_backend_state
|
||||
from skillopt.model import claude_backend, minimax_backend, openai_compatible_backend, qwen_backend
|
||||
|
||||
empty = {"_total": {"calls": 0, "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}}
|
||||
monkeypatch.setattr(azure_openai, "get_token_summary", lambda: empty.copy())
|
||||
monkeypatch.setattr(claude_backend, "get_token_summary", lambda: empty.copy())
|
||||
monkeypatch.setattr(qwen_backend, "get_token_summary", lambda: empty.copy())
|
||||
monkeypatch.setattr(minimax_backend, "get_token_summary", lambda: empty.copy())
|
||||
monkeypatch.setattr(
|
||||
openai_compatible_backend,
|
||||
"get_token_summary",
|
||||
lambda: {"optimizer": {"calls": 1, "prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
codex_backend,
|
||||
"get_token_summary",
|
||||
lambda: {"optimizer": {"calls": 1, "prompt_tokens": 11, "completion_tokens": 13, "total_tokens": 24}},
|
||||
)
|
||||
|
||||
summary = model_module.get_token_summary()
|
||||
|
||||
assert summary["optimizer"]["calls"] == 2
|
||||
assert summary["optimizer"]["prompt_tokens"] == 16
|
||||
assert summary["optimizer"]["completion_tokens"] == 20
|
||||
assert summary["_total"]["total_tokens"] == 36
|
||||
|
||||
|
||||
def test_reset_token_tracker_resets_codex_and_existing_backends(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, _backend_config, codex_backend, azure_openai = isolate_backend_state
|
||||
from skillopt.model import claude_backend, minimax_backend, openai_compatible_backend, qwen_backend
|
||||
|
||||
called: list[str] = []
|
||||
for name, module in [
|
||||
("openai", azure_openai),
|
||||
("claude", claude_backend),
|
||||
("qwen", qwen_backend),
|
||||
("minimax", minimax_backend),
|
||||
("compat", openai_compatible_backend),
|
||||
("codex", codex_backend),
|
||||
]:
|
||||
monkeypatch.setattr(module, "reset_token_tracker", lambda name=name: called.append(name))
|
||||
|
||||
model_module.reset_token_tracker()
|
||||
|
||||
assert called == ["openai", "claude", "qwen", "minimax", "compat", "codex"]
|
||||
|
||||
|
||||
def test_set_reasoning_effort_updates_codex_and_existing_backends(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, _backend_config, codex_backend, azure_openai = isolate_backend_state
|
||||
from skillopt.model import claude_backend, minimax_backend, openai_compatible_backend, qwen_backend
|
||||
|
||||
called: list[tuple[str, str]] = []
|
||||
for name, module in [
|
||||
("openai", azure_openai),
|
||||
("claude", claude_backend),
|
||||
("qwen", qwen_backend),
|
||||
("minimax", minimax_backend),
|
||||
("compat", openai_compatible_backend),
|
||||
("codex", codex_backend),
|
||||
]:
|
||||
monkeypatch.setattr(module, "set_reasoning_effort", lambda effort, name=name: called.append((name, effort)))
|
||||
|
||||
model_module.set_reasoning_effort("high")
|
||||
|
||||
assert called == [
|
||||
("openai", "high"),
|
||||
("claude", "high"),
|
||||
("qwen", "high"),
|
||||
("minimax", "high"),
|
||||
("compat", "high"),
|
||||
("codex", "high"),
|
||||
]
|
||||
|
||||
|
||||
def test_deployment_setters_update_codex_without_dropping_openai_compatible(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolate_backend_state: tuple[Any, Any, Any, Any],
|
||||
) -> None:
|
||||
model_module, _backend_config, codex_backend, _azure_openai = isolate_backend_state
|
||||
from skillopt.model import openai_compatible_backend
|
||||
|
||||
called: list[tuple[str, str, str]] = []
|
||||
monkeypatch.setattr(
|
||||
openai_compatible_backend,
|
||||
"set_target_deployment",
|
||||
lambda deployment: called.append(("compat", "target", deployment)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
openai_compatible_backend,
|
||||
"set_optimizer_deployment",
|
||||
lambda deployment: called.append(("compat", "optimizer", deployment)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
codex_backend,
|
||||
"set_target_deployment",
|
||||
lambda deployment: called.append(("codex", "target", deployment)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
codex_backend,
|
||||
"set_optimizer_deployment",
|
||||
lambda deployment: called.append(("codex", "optimizer", deployment)),
|
||||
)
|
||||
|
||||
model_module.set_target_deployment("target-model")
|
||||
model_module.set_optimizer_deployment("optimizer-model")
|
||||
|
||||
assert ("compat", "target", "target-model") in called
|
||||
assert ("codex", "target", "target-model") in called
|
||||
assert ("compat", "optimizer", "optimizer-model") in called
|
||||
assert ("codex", "optimizer", "optimizer-model") in called
|
||||
Reference in New Issue
Block a user