feat(sleep): real claude + codex backends, gbrain-evals benchmark, rule judges
Upgrade from mock-only to REAL multi-backend validation:
Backends (skillopt/sleep/backend.py):
- CliBackend base: shared attempt/judge/reflect prompts, response cache,
token accounting. Subclasses implement only _call().
- ClaudeCliBackend: drives `claude -p --output-format text`.
- CodexCliBackend: drives the REAL @openai/codex `exec -o <file>` for clean
output; resolve_codex_path() skips the hermes wrapper at ~/.local/bin/codex.
- reflect() now aggregates the exact failing judge criteria into the prompt
(gbrain's lesson: tell the optimizer what the scorer rewards).
Rule judges (skillopt/sleep/judges.py): gbrain-compatible local scorers
(section_present / regex / max_chars / contains / tool_called) — held-out
scoring with no judge-API spend. TaskRecord gains a `judge` field +
reference_kind="rule".
gbrain-evals adapter (experiments/gbrain_bench.py, run_gbrain.py): load
garrytan/gbrain-evals skillopt-v1 deficient skills + train/held-out task
sets and run our consolidate() loop against the SAME suite gbrain scores.
REAL results (docs/sleep/real_api_results.md), brief-writer seed, 1 night:
- Claude (Haiku): held-out 0.00 -> 1.00
- Codex: held-out 0.00 -> 0.67
Both proposed a correct, general format rule into the protected LEARNED block.
CLI: --backend {mock,claude,codex}, --codex-path, --model; experiment +
gbrain runners gain --limit-* cost controls. 17 tests pass.
Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
@@ -133,6 +133,50 @@ class TestConsolidateGate(unittest.TestCase):
|
||||
self.assertEqual(len(r2.applied_edits), 0)
|
||||
|
||||
|
||||
class TestRuleJudge(unittest.TestCase):
|
||||
def test_section_and_regex(self):
|
||||
from skillopt.sleep.judges import score_rule_judge
|
||||
j = {"kind": "rule", "checks": [
|
||||
{"op": "section_present", "arg": "Key Risks"},
|
||||
{"op": "regex", "arg": r"[Cc]onfidence\s*[:=]"},
|
||||
]}
|
||||
ok = "# Brief\n## Key Risks\nstuff\nConfidence: High"
|
||||
self.assertEqual(score_rule_judge(j, ok)[0], 1.0)
|
||||
self.assertEqual(score_rule_judge(j, "just an answer")[0], 0.0)
|
||||
|
||||
def test_max_chars(self):
|
||||
from skillopt.sleep.judges import score_rule_judge
|
||||
j = {"checks": [{"op": "max_chars", "arg": 50}]}
|
||||
self.assertEqual(score_rule_judge(j, "x" * 10)[0], 1.0)
|
||||
self.assertEqual(score_rule_judge(j, "x" * 100)[0], 0.0)
|
||||
|
||||
def test_partial_soft_score(self):
|
||||
from skillopt.sleep.judges import score_rule_judge
|
||||
j = {"checks": [
|
||||
{"op": "contains", "arg": "alpha"},
|
||||
{"op": "contains", "arg": "beta"},
|
||||
]}
|
||||
h, s, _ = score_rule_judge(j, "only alpha here")
|
||||
self.assertEqual(h, 0.0)
|
||||
self.assertAlmostEqual(s, 0.5)
|
||||
|
||||
|
||||
class TestGbrainLoader(unittest.TestCase):
|
||||
def test_loads_when_present(self):
|
||||
from skillopt.sleep.experiments.gbrain_bench import find_data_root, load_seed
|
||||
root = find_data_root()
|
||||
if not root:
|
||||
self.skipTest("gbrain-evals data not present")
|
||||
skill, tasks = load_seed(root, "brief-writer")
|
||||
self.assertTrue(skill)
|
||||
self.assertTrue(any(t.split == "holdout" for t in tasks))
|
||||
self.assertTrue(all(t.reference_kind == "rule" for t in tasks))
|
||||
# the deficient skill must FAIL its own held-out checks (baseline 0)
|
||||
from skillopt.sleep.judges import score_rule_judge
|
||||
ho = [t for t in tasks if t.split == "holdout"][0]
|
||||
self.assertEqual(score_rule_judge(ho.judge, skill)[0], 0.0)
|
||||
|
||||
|
||||
class TestFullCycleAndAdopt(unittest.TestCase):
|
||||
def test_cycle_stage_then_adopt_with_backup(self):
|
||||
with tempfile.TemporaryDirectory() as proj, tempfile.TemporaryDirectory() as home:
|
||||
|
||||
Reference in New Issue
Block a user