Add Codex Desktop transcript harvesting
This commit is contained in:
Kirill Kostarev
committed by
carpedkm
carpedkm
parent
e8c3e10b30
commit
31715a8b43
+106
-15
@@ -15,11 +15,11 @@ from skillopt_sleep.backend import MockBackend, exact_score, keyword_soft_score
|
||||
from skillopt_sleep.config import load_config
|
||||
from skillopt_sleep.consolidate import consolidate
|
||||
from skillopt_sleep.cycle import run_sleep_cycle
|
||||
from skillopt_sleep.experiments.personas import researcher_persona, programmer_persona
|
||||
from skillopt_sleep.harvest import digest_transcript, _detect_feedback, _is_meta_prompt
|
||||
from skillopt_sleep.experiments.personas import programmer_persona, researcher_persona
|
||||
from skillopt_sleep.harvest import _detect_feedback, _is_meta_prompt, digest_transcript
|
||||
from skillopt_sleep.memory import apply_edits, current_learned_lines, extract_learned, set_learned
|
||||
from skillopt_sleep.mine import assign_splits, heuristic_mine, dedup_tasks
|
||||
from skillopt_sleep.staging import adopt, latest_staging
|
||||
from skillopt_sleep.mine import assign_splits, heuristic_mine
|
||||
from skillopt_sleep.staging import adopt
|
||||
from skillopt_sleep.types import EditRecord, SessionDigest, TaskRecord
|
||||
|
||||
|
||||
@@ -89,6 +89,97 @@ class TestHarvest(unittest.TestCase):
|
||||
self.assertIsInstance(d.session_id, str)
|
||||
self.assertGreaterEqual(d.n_user_turns + d.n_assistant_turns, 0)
|
||||
|
||||
def _write_jsonl(self, path, records):
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
for record in records:
|
||||
f.write(json.dumps(record) + "\n")
|
||||
|
||||
def test_digest_codex_archived_session_sanitizes_and_skips_meta(self):
|
||||
from skillopt_sleep.harvest_codex import digest_codex_archived_session
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "rollout-example.jsonl")
|
||||
self._write_jsonl(path, [
|
||||
{"type": "turn_context", "timestamp": "2026-06-12T10:00:00Z",
|
||||
"payload": {"cwd": "/repo/Yoshi", "type": None}},
|
||||
{"type": "response_item", "timestamp": "2026-06-12T10:00:01Z",
|
||||
"payload": {"type": "message", "role": "developer",
|
||||
"content": [{"type": "text", "text": "do not copy"}]}},
|
||||
{"type": "response_item", "timestamp": "2026-06-12T10:00:02Z",
|
||||
"payload": {"type": "user_message",
|
||||
"message": "# AGENTS.md instructions for /repo/Yoshi\n"
|
||||
"<INSTRUCTIONS>do not keep</INSTRUCTIONS>"}},
|
||||
{"type": "response_item", "timestamp": "2026-06-12T10:00:03Z",
|
||||
"payload": {"type": "user_message",
|
||||
"message": "run deploy with sk-1234567890abcdef and token local-secret"}},
|
||||
{"type": "response_item", "timestamp": "2026-06-12T10:00:04Z",
|
||||
"payload": {"type": "function_call", "name": "exec_command",
|
||||
"arguments": "raw args should not copy"}},
|
||||
{"type": "response_item", "timestamp": "2026-06-12T10:00:05Z",
|
||||
"payload": {"type": "function_call_output",
|
||||
"output": "raw output should not copy"}},
|
||||
{"type": "response_item", "timestamp": "2026-06-12T10:00:06Z",
|
||||
"payload": {"type": "agent_message", "message": "done"}},
|
||||
])
|
||||
|
||||
digest = digest_codex_archived_session(path, project="/repo/Yoshi")
|
||||
|
||||
self.assertIsNotNone(digest)
|
||||
joined = "\n".join(digest.user_prompts + digest.assistant_finals)
|
||||
self.assertEqual(digest.project, "/repo/Yoshi")
|
||||
self.assertIn("[REDACTED_OPENAI_KEY]", joined)
|
||||
self.assertIn("token [REDACTED]", joined)
|
||||
self.assertIn("exec_command", digest.tools_used)
|
||||
self.assertNotIn("AGENTS.md instructions", joined)
|
||||
self.assertNotIn("do not copy", joined)
|
||||
self.assertNotIn("raw args should not copy", joined)
|
||||
self.assertNotIn("raw output should not copy", joined)
|
||||
|
||||
def test_harvest_codex_filters_project_and_cli_source(self):
|
||||
from skillopt_sleep.__main__ import _cfg_from_args
|
||||
from skillopt_sleep.harvest_sources import harvest_for_config
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
codex_home = os.path.join(tmp, ".codex")
|
||||
sessions = os.path.join(codex_home, "archived_sessions")
|
||||
os.makedirs(sessions)
|
||||
self._write_jsonl(os.path.join(sessions, "rollout-yoshi.jsonl"), [
|
||||
{"type": "turn_context", "timestamp": "2026-06-12T10:00:00Z",
|
||||
"payload": {"cwd": "/repo/Yoshi", "type": None}},
|
||||
{"type": "response_item", "timestamp": "2026-06-12T10:00:01Z",
|
||||
"payload": {"type": "user_message", "message": "fix Yoshi"}},
|
||||
{"type": "response_item", "timestamp": "2026-06-12T10:00:02Z",
|
||||
"payload": {"type": "agent_message", "message": "fixed"}},
|
||||
])
|
||||
self._write_jsonl(os.path.join(sessions, "rollout-other.jsonl"), [
|
||||
{"type": "turn_context", "timestamp": "2026-06-12T10:00:00Z",
|
||||
"payload": {"cwd": "/repo/Other", "type": None}},
|
||||
{"type": "response_item", "timestamp": "2026-06-12T10:00:01Z",
|
||||
"payload": {"type": "user_message", "message": "fix Other"}},
|
||||
])
|
||||
|
||||
Args = type("Args", (), {
|
||||
"project": "/repo/Yoshi",
|
||||
"scope": "",
|
||||
"backend": "",
|
||||
"model": "",
|
||||
"codex_path": "",
|
||||
"claude_home": "",
|
||||
"codex_home": codex_home,
|
||||
"source": "codex",
|
||||
"lookback_hours": 0,
|
||||
"edit_budget": 0,
|
||||
"auto_adopt": False,
|
||||
})
|
||||
|
||||
cfg = _cfg_from_args(Args())
|
||||
digests = harvest_for_config(cfg, limit=10)
|
||||
|
||||
self.assertEqual(cfg.get("transcript_source"), "codex")
|
||||
self.assertEqual(len(digests), 1)
|
||||
self.assertEqual(digests[0].session_id, "rollout-yoshi")
|
||||
self.assertEqual(digests[0].user_prompts, ["fix Yoshi"])
|
||||
|
||||
|
||||
class TestMine(unittest.TestCase):
|
||||
def _digest(self, prompts, feedback):
|
||||
@@ -115,7 +206,6 @@ class TestMine(unittest.TestCase):
|
||||
|
||||
def test_dream_never_in_val_or_test(self):
|
||||
# the anti-overfitting guarantee: origin='dream' tasks only ever land in train
|
||||
from skillopt_sleep.types import TaskRecord
|
||||
real = researcher_persona()
|
||||
dream = [TaskRecord(id=f"d{i}", project="/p", intent=f"dream {i}",
|
||||
origin="dream", derived_from="r0") for i in range(5)]
|
||||
@@ -235,7 +325,7 @@ class TestLlmMiner(unittest.TestCase):
|
||||
class TestMultiObjectiveAndPrefs(unittest.TestCase):
|
||||
def test_multi_objective_reward(self):
|
||||
from skillopt_sleep.replay import multi_objective_reward
|
||||
from skillopt_sleep.types import ReplayResult, TaskRecord
|
||||
from skillopt_sleep.types import ReplayResult
|
||||
t = TaskRecord(id="t", project="/p", intent="x")
|
||||
expensive = [(t, ReplayResult(id="t", hard=1.0, tokens=4000, latency_ms=20000))]
|
||||
cheap = [(t, ReplayResult(id="t", hard=1.0, tokens=200, latency_ms=1000))]
|
||||
@@ -249,7 +339,7 @@ class TestMultiObjectiveAndPrefs(unittest.TestCase):
|
||||
|
||||
def test_preferences_injected_into_reflect(self):
|
||||
from skillopt_sleep.backend import CliBackend
|
||||
from skillopt_sleep.types import TaskRecord, ReplayResult
|
||||
from skillopt_sleep.types import ReplayResult
|
||||
captured = {}
|
||||
|
||||
class CapBackend(CliBackend):
|
||||
@@ -269,7 +359,6 @@ class TestMultiObjectiveAndPrefs(unittest.TestCase):
|
||||
def test_replay_records_cost(self):
|
||||
from skillopt_sleep.backend import MockBackend
|
||||
from skillopt_sleep.replay import replay_one
|
||||
from skillopt_sleep.types import TaskRecord
|
||||
t = TaskRecord(id="t", project="/p", intent="hello world",
|
||||
reference_kind="exact", reference="hi")
|
||||
r = replay_one(MockBackend(), t, "some skill text", "")
|
||||
@@ -280,7 +369,7 @@ class TestMultiObjectiveAndPrefs(unittest.TestCase):
|
||||
class TestMultiRolloutAndBudget(unittest.TestCase):
|
||||
def test_rolloutset_stats(self):
|
||||
from skillopt_sleep.rollout import RolloutSet
|
||||
from skillopt_sleep.types import ReplayResult, TaskRecord
|
||||
from skillopt_sleep.types import ReplayResult
|
||||
rs = RolloutSet(task=TaskRecord(id="t", project="/p", intent="x"),
|
||||
attempts=[ReplayResult(id="t", hard=1.0),
|
||||
ReplayResult(id="t", hard=0.0),
|
||||
@@ -305,7 +394,7 @@ class TestMultiRolloutAndBudget(unittest.TestCase):
|
||||
def test_contrastive_reflect_with_stub(self):
|
||||
from skillopt_sleep.backend import Backend
|
||||
from skillopt_sleep.rollout import RolloutSet, contrastive_reflect
|
||||
from skillopt_sleep.types import ReplayResult, TaskRecord
|
||||
from skillopt_sleep.types import ReplayResult
|
||||
|
||||
class StubBackend(Backend):
|
||||
name = "stub"
|
||||
@@ -323,8 +412,11 @@ class TestMultiRolloutAndBudget(unittest.TestCase):
|
||||
class TestSlowUpdate(unittest.TestCase):
|
||||
def test_protected_field_roundtrip(self):
|
||||
from skillopt_sleep.slow_update import (
|
||||
replace_slow_field, extract_slow_field, has_slow_field,
|
||||
SLOW_UPDATE_START, SLOW_UPDATE_END,
|
||||
SLOW_UPDATE_END,
|
||||
SLOW_UPDATE_START,
|
||||
extract_slow_field,
|
||||
has_slow_field,
|
||||
replace_slow_field,
|
||||
)
|
||||
base = "# skill\nkeep me\n"
|
||||
doc = replace_slow_field(base, "durable lesson A")
|
||||
@@ -341,7 +433,7 @@ class TestSlowUpdate(unittest.TestCase):
|
||||
def test_run_slow_update_with_stub_backend(self):
|
||||
from skillopt_sleep.backend import Backend
|
||||
from skillopt_sleep.slow_update import run_slow_update
|
||||
from skillopt_sleep.types import TaskRecord, ReplayResult
|
||||
from skillopt_sleep.types import ReplayResult
|
||||
|
||||
class StubBackend(Backend):
|
||||
name = "stub"
|
||||
@@ -366,9 +458,8 @@ class TestSlowUpdate(unittest.TestCase):
|
||||
class TestToolLoop(unittest.TestCase):
|
||||
def test_tool_called_judge_via_replay(self):
|
||||
from skillopt_sleep.backend import MockBackend
|
||||
from skillopt_sleep.replay import replay_one, _required_tools
|
||||
from skillopt_sleep.memory import set_learned
|
||||
from skillopt_sleep.types import TaskRecord
|
||||
from skillopt_sleep.replay import _required_tools, replay_one
|
||||
|
||||
task = TaskRecord(
|
||||
id="qa1", project="/p", intent="answer the question",
|
||||
|
||||
Reference in New Issue
Block a user