url/SkillOpt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #150 from Yif-Yang/test/assert-fractional-degradation-score

Browse Source 
test(sleep): assert fractional candidate score
This commit is contained in:
Yifan Yang
2026-07-19 01:28:47 +08:00
committed by GitHub
parent 67b67acf5f 7933f37bf0
commit b860a5cf88
1 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_sleep_engine.py
+7 -1
View File
@@ -1333,6 +1333,8 @@ class TestVerifierDiscipline(unittest.TestCase):
"""Gate must reject an edit that partially degrades held-out (1.0→0.5), """Gate must reject an edit that partially degrades held-out (1.0→0.5),
not just all-or-nothing collapses. Train improves (0.0→1.0) which makes not just all-or-nothing collapses. Train improves (0.0→1.0) which makes
the regression easy to miss — the gate catches it anyway.""" the regression easy to miss — the gate catches it anyway."""
from skillopt_sleep.replay import aggregate_scores, replay_batch
be = MockFractionalDegradationBackend() be = MockFractionalDegradationBackend()
train = TaskRecord(id="t3", project="/p", intent="train", reference="ABC", train = TaskRecord(id="t3", project="/p", intent="train", reference="ABC",
reference_kind="exact", tags=["rule:__reward_hacking__"], split="train") reference_kind="exact", tags=["rule:__reward_hacking__"], split="train")
@@ -1342,6 +1344,11 @@ class TestVerifierDiscipline(unittest.TestCase):
reference_kind="exact", tags=["rule:real"], split="val") reference_kind="exact", tags=["rule:real"], split="val")
tasks = [train, val1, val2] tasks = [train, val1, val2]
candidate_pairs = replay_batch(be, [val1, val2], be.HACK_TEXT, "")
candidate_hard, _candidate_soft = aggregate_scores(candidate_pairs)
self.assertEqual([result.hard for _task, result in candidate_pairs], [1.0, 0.0])
self.assertEqual(candidate_hard, 0.5)
res = consolidate(be, tasks, "", "", edit_budget=4, gate_metric="hard", night=1) res = consolidate(be, tasks, "", "", edit_budget=4, gate_metric="hard", night=1)
self.assertFalse(res.accepted) self.assertFalse(res.accepted)
@@ -1356,7 +1363,6 @@ class TestVerifierDiscipline(unittest.TestCase):
self.assertIn("definitive answer", res.rejected_edits[0].content) self.assertIn("definitive answer", res.rejected_edits[0].content)
class TestDiagnosticsRedaction(unittest.TestCase): class TestDiagnosticsRedaction(unittest.TestCase):
"""diagnostics.json surfaces backend stderr / optimizer replies / task """diagnostics.json surfaces backend stderr / optimizer replies / task
responses for debugging — but those can carry credentials (e.g. a codex 401 responses for debugging — but those can carry credentials (e.g. a codex 401