feat(sleep): real claude + codex backends, gbrain-evals benchmark, rule judges
Upgrade from mock-only to REAL multi-backend validation:
Backends (skillopt/sleep/backend.py):
- CliBackend base: shared attempt/judge/reflect prompts, response cache,
token accounting. Subclasses implement only _call().
- ClaudeCliBackend: drives `claude -p --output-format text`.
- CodexCliBackend: drives the REAL @openai/codex `exec -o <file>` for clean
output; resolve_codex_path() skips the hermes wrapper at ~/.local/bin/codex.
- reflect() now aggregates the exact failing judge criteria into the prompt
(gbrain's lesson: tell the optimizer what the scorer rewards).
Rule judges (skillopt/sleep/judges.py): gbrain-compatible local scorers
(section_present / regex / max_chars / contains / tool_called) — held-out
scoring with no judge-API spend. TaskRecord gains a `judge` field +
reference_kind="rule".
gbrain-evals adapter (experiments/gbrain_bench.py, run_gbrain.py): load
garrytan/gbrain-evals skillopt-v1 deficient skills + train/held-out task
sets and run our consolidate() loop against the SAME suite gbrain scores.
REAL results (docs/sleep/real_api_results.md), brief-writer seed, 1 night:
- Claude (Haiku): held-out 0.00 -> 1.00
- Codex: held-out 0.00 -> 0.67
Both proposed a correct, general format rule into the protected LEARNED block.
CLI: --backend {mock,claude,codex}, --codex-path, --model; experiment +
gbrain runners gain --limit-* cost controls. 17 tests pass.
Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
"""SkillOpt-Sleep — rule-based judges (gbrain-evals compatible).
|
||||
|
||||
Implements the programmatic check operators used by gbrain-evals'
|
||||
skillopt-v1 benchmark so we can score skill outputs locally, with NO judge
|
||||
API call:
|
||||
|
||||
* section_present <name> — a markdown heading containing <name> exists
|
||||
* regex <pattern> — the pattern matches the response
|
||||
* max_chars <n> — response length <= n
|
||||
* min_chars <n> — response length >= n
|
||||
* contains <text> — substring present (case-insensitive)
|
||||
* tool_called <name> — a tool with <name> was invoked (needs a tool loop;
|
||||
in single-shot replay we approximate via an
|
||||
explicit "TOOL_CALL: <name>" marker the agent emits)
|
||||
|
||||
A task whose judge is {"kind": "rule", "checks": [...]} passes (hard=1.0) iff
|
||||
ALL checks pass; soft = fraction of checks passed. This mirrors gbrain's
|
||||
all-checks-must-pass rule scoring and gives the gate a smooth signal.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
|
||||
def _section_present(response: str, name: str) -> bool:
|
||||
# a markdown heading line (#, ##, ...) or bold line that contains `name`
|
||||
pat = re.compile(
|
||||
r"(?im)^\s{0,3}(#{1,6}\s*.*%s|\*\*.*%s.*\*\*\s*:?)\s*$" % (re.escape(name), re.escape(name))
|
||||
)
|
||||
if pat.search(response or ""):
|
||||
return True
|
||||
# also accept "Name:" style label at line start
|
||||
label = re.compile(r"(?im)^\s*%s\s*:" % re.escape(name))
|
||||
return bool(label.search(response or ""))
|
||||
|
||||
|
||||
def _check(op: str, arg: Any, response: str, tools_called: List[str]) -> bool:
|
||||
r = response or ""
|
||||
if op == "section_present":
|
||||
return _section_present(r, str(arg))
|
||||
if op == "regex":
|
||||
try:
|
||||
return bool(re.search(str(arg), r))
|
||||
except re.error:
|
||||
return False
|
||||
if op == "max_chars":
|
||||
return len(r) <= int(arg)
|
||||
if op == "min_chars":
|
||||
return len(r) >= int(arg)
|
||||
if op == "contains":
|
||||
return str(arg).lower() in r.lower()
|
||||
if op == "tool_called":
|
||||
name = str(arg).lower()
|
||||
if any(name == t.lower() for t in tools_called):
|
||||
return True
|
||||
# single-shot approximation: the agent emits an explicit marker
|
||||
return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r))
|
||||
# unknown op: do not block
|
||||
return True
|
||||
|
||||
|
||||
def score_rule_judge(
|
||||
judge: Dict[str, Any],
|
||||
response: str,
|
||||
tools_called: List[str] | None = None,
|
||||
) -> Tuple[float, float, str]:
|
||||
"""Return (hard, soft, rationale) for a gbrain-style rule judge."""
|
||||
checks = (judge or {}).get("checks", []) or []
|
||||
if not checks:
|
||||
return 0.0, 0.0, "no checks"
|
||||
tools_called = tools_called or []
|
||||
passed = 0
|
||||
failed_desc: List[str] = []
|
||||
for c in checks:
|
||||
ok = _check(c.get("op", ""), c.get("arg"), response, tools_called)
|
||||
if ok:
|
||||
passed += 1
|
||||
else:
|
||||
failed_desc.append(f"{c.get('op')}={c.get('arg')}")
|
||||
soft = passed / len(checks)
|
||||
hard = 1.0 if passed == len(checks) else 0.0
|
||||
rationale = "all checks passed" if hard else "failed: " + ", ".join(failed_desc)
|
||||
return hard, soft, rationale
|
||||
Reference in New Issue
Block a user