feat(sleep): real tool-loop replay for gbrain quick-answerer (tool_called judge)
The 4th gbrain seed (quick-answerer) is judged by tool_called=search: the agent
must ACTUALLY call a search tool. Add an honest tool loop:
- Backend.attempt_with_tools(task, skill, memory, tools) -> (response, tools_called)
- Claude: exposes a real ./search shell shim, runs with --allowedTools Bash in a
clean cwd; detects the call from the shim's log (not a self-reported marker).
- Codex: same shim under `exec --sandbox workspace-write`.
- Mock: deterministic — "calls" a tool iff skill/memory instructs it (for CI).
- replay_one routes tasks with a tool_called check through the tool loop and
feeds detected calls to the rule judge; ReplayResult gains tools_called.
Verified live (Claude haiku): deficient skill -> tools_called=[] hard=0;
learned "must run ./search" rule -> tools_called=['search'] hard=1.0.
20 tests pass.
Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
@@ -213,6 +213,32 @@ class TestLlmMiner(unittest.TestCase):
|
||||
self.assertEqual(make_llm_miner(EmptyBackend())([digest]), [])
|
||||
|
||||
|
||||
class TestToolLoop(unittest.TestCase):
|
||||
def test_tool_called_judge_via_replay(self):
|
||||
from skillopt.sleep.backend import MockBackend
|
||||
from skillopt.sleep.replay import replay_one, _required_tools
|
||||
from skillopt.sleep.memory import set_learned
|
||||
from skillopt.sleep.types import TaskRecord
|
||||
|
||||
task = TaskRecord(
|
||||
id="qa1", project="/p", intent="answer the question",
|
||||
reference_kind="rule",
|
||||
judge={"kind": "rule", "checks": [{"op": "tool_called", "arg": "search"}]},
|
||||
)
|
||||
self.assertEqual(_required_tools(task), ["search"])
|
||||
be = MockBackend()
|
||||
# deficient skill: no instruction to search -> tool not called -> hard 0
|
||||
deficient = "Answer from memory. Do NOT use tools."
|
||||
r0 = replay_one(be, task, deficient, "")
|
||||
self.assertEqual(r0.hard, 0.0)
|
||||
self.assertEqual(r0.tools_called, [])
|
||||
# learned rule to use ./search -> tool called -> hard 1
|
||||
learned = set_learned(deficient, ["Before answering you MUST run ./search first."])
|
||||
r1 = replay_one(be, task, learned, "")
|
||||
self.assertEqual(r1.hard, 1.0)
|
||||
self.assertEqual(r1.tools_called, ["search"])
|
||||
|
||||
|
||||
class TestFullCycleAndAdopt(unittest.TestCase):
|
||||
def test_cycle_stage_then_adopt_with_backup(self):
|
||||
with tempfile.TemporaryDirectory() as proj, tempfile.TemporaryDirectory() as home:
|
||||
|
||||
Reference in New Issue
Block a user