diff --git a/.gitignore b/.gitignore index 4b90712..4a5142a 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ logs/ external/ # SkillOpt-Sleep runtime state (staging proposals, config, diagnostics, cron logs) .skillopt-sleep/ +# SkillOpt-Sleep handoff-backend round data (prompts/answers derived from transcripts) +.skillopt-sleep-handoff/ +.skillopt-sleep-handoff.night*.done/ /BabyVision/ /MMRB/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a07c52..1e42464 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to SkillOpt are documented here. This project adheres to [Semantic Versioning](https://semver.org/) and the format is based on [Keep a Changelog](https://keepachangelog.com/). +## [Unreleased] + +### Added +- **Handoff backend** (`--backend handoff`) for SkillOpt-Sleep — runs the + sleep cycle with no model subprocess or API key: the engine writes each + pending model call to `PROMPTS.md`/`pending.json` (exit code 3) and the + user's own agent session answers into `answers/.md`; re-running the + same command resumes statelessly from the answers (typically 3–6 rounds + per night). Mined tasks are pinned per night so answering sessions cannot + shift the task set. Ships a `/skillopt-sleep-handoff` Claude Code command + that automates the loop with fresh-context subagents to protect the + held-out gate. + ## [0.2.0] — 2026-07-02 The headline of this release is **SkillOpt-Sleep**: a nightly offline diff --git a/plugins/README.md b/plugins/README.md index f822a22..c6fe040 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -22,7 +22,7 @@ sleep** idea (short-term experience → long-term competence). | Platform | Folder | Mechanism | Status | |---|---|---|---| -| **Claude Code** | [`claude-code/`](claude-code) | `.claude-plugin` + `/skillopt-sleep` command + skill + hooks | full, installable | +| **Claude Code** | [`claude-code/`](claude-code) | `.claude-plugin` + `/skillopt-sleep` + `/skillopt-sleep-handoff` commands + skill + hooks | full, installable | | **Codex** | [`codex/`](codex) | user-level `skillopt-sleep` skill + shared runner | full | | **Copilot** | [`copilot/`](copilot) | MCP server (`sleep_*` tools) + `copilot-instructions` | full (MCP) | | **Devin** | [`devin/`](devin) | MCP server (`sleep_*` tools) + Devin ATIF-v1.7 harvest + `.devin/rules` | full (MCP) | @@ -149,6 +149,47 @@ The reward can weight not just correctness but **cost and speed**, so a skill ca learn to be cheaper and faster, not only more accurate. *What it does for you:* "answer directly instead of opening five files" becomes a learned habit. +### `--backend handoff` — session-executed calls (no API subprocess) + +For subscription seats and environments where the engine shouldn't spawn +`claude -p` / API calls itself. The engine still runs every deterministic +stage (harvest → mine → replay scoring → gate → stage), but each model call +(attempt / judge / reflect) is written to a prompt file that **your own agent +session answers between engine runs**: + +```bash +python -m skillopt_sleep run --backend handoff --project "$(pwd)" +# exit 3 => .skillopt-sleep-handoff/PROMPTS.md + pending.json were written +# answer each prompt (each in a FRESH context) into answers/.md +# re-run the same command => it resumes from the answers and either +# finishes (exit 0) or stages the next prompt batch (exit 3) +``` + +A typical night converges in 3–6 rounds: baseline attempts → reflect → +candidate re-scoring per accepted edit. Resume is stateless — replay is +deterministic and answers are cached by prompt hash, so re-running skips +everything already answered. Mined tasks are pinned to +`.skillopt-sleep-handoff/tasks.json` on the first round, so the sessions that +answer the prompts can't shift the task set and invalidate earlier answers. +On a completed real run the handoff directory is archived to +`.skillopt-sleep-handoff.night.done`. + +On Claude Code, `/skillopt-sleep-handoff run` drives the whole loop for you, +answering each prompt in an isolated fresh-context subagent. + +**Integrity rule:** answer every prompt in a fresh context (a subagent with no +conversation history). Answering from a session that has already seen the +mined tasks and their references contaminates the held-out gate and fakes the +improvement score. + +*What it does for you:* the sleep cycle runs entirely on your interactive +session's subscription budget — no API key, no headless subprocess — while the +gate, splits, and staging discipline stay in the engine. + +Limitations: `--rollouts-k > 1` gives no contrastive spread (identical prompt +→ identical answer file), and tool-loop tasks fall back to the single-shot +`TOOL_CALL:` marker convention. + ### `schedule` / `unschedule` — set it and forget it Built-in nightly scheduling (no manual cron): @@ -176,7 +217,7 @@ schedule, if you trust it). | Flag | Default | Meaning | |---|---|---| -| `--backend mock\|claude\|codex\|copilot` | `mock` | who runs/optimizes (mock = free) | +| `--backend mock\|claude\|codex\|copilot\|handoff` | `mock` | who runs/optimizes (mock = free; handoff = your own session answers) | | `--preferences "..."` | – | your house rules, as a prior | | `--gate on\|off` | `on` | strict held-out gate vs. greedy | | `--rollouts-k K` | `1` | multi-rollout contrastive reflection | diff --git a/plugins/claude-code/README.md b/plugins/claude-code/README.md index 137912d..8f9f53b 100644 --- a/plugins/claude-code/README.md +++ b/plugins/claude-code/README.md @@ -60,6 +60,9 @@ they shell out to the CLIs you already have. /skillopt-sleep run # full cycle: stages a reviewed proposal (still no live edits) /skillopt-sleep status # see history + the latest staged proposal /skillopt-sleep adopt # apply the staged proposal to CLAUDE.md / SKILL.md (with backup) + +/skillopt-sleep-handoff run # same cycle, but THIS session answers the model calls + # (no claude -p subprocess, no API key — subscription-friendly) ``` Or call the engine directly (Python ≥ 3.10): @@ -74,6 +77,26 @@ Default backend is **`mock`** — deterministic, no API spend — so you can try plumbing for free. Switch to `--backend claude` or `--backend codex` for genuine improvement on your own budget. +### Handoff mode (session answers the model calls) + +`--backend handoff` runs the cycle without any model subprocess: the engine +executes the deterministic stages and writes every model call it needs to +`.skillopt-sleep-handoff/PROMPTS.md` + `pending.json` (exit code 3). You (or +the `/skillopt-sleep-handoff` command, which automates the loop with isolated +fresh-context subagents) write each raw answer to `answers/.md` and re-run +the same command; it resumes from the answers and either finishes or stages +the next batch. Typically 3–6 rounds per night. + +```bash +python -m skillopt_sleep run --backend handoff --project "$(pwd)" +# ... answer .skillopt-sleep-handoff/PROMPTS.md into answers/.md ... +python -m skillopt_sleep run --backend handoff --project "$(pwd)" # resume +``` + +Answer every prompt in a **fresh context** — a session that has already seen +the mined tasks and their references would contaminate the held-out gate. +Details: [the plugins README](../README.md#--backend-handoff--session-executed-calls-no-api-subprocess). + ## Does it actually improve? (real models, public benchmark) SkillOpt-Sleep is validated against [gbrain-evals](https://github.com/garrytan/gbrain-evals)' diff --git a/plugins/claude-code/commands/skillopt-sleep-handoff.md b/plugins/claude-code/commands/skillopt-sleep-handoff.md new file mode 100644 index 0000000..fffad76 --- /dev/null +++ b/plugins/claude-code/commands/skillopt-sleep-handoff.md @@ -0,0 +1,67 @@ +--- +description: Run the SkillOpt-Sleep cycle with the handoff backend — no API subprocess; this session answers the engine's model calls via prompt/answer files, in isolated fresh-context subagents +argument-hint: "[run | dry-run] [--preferences \"...\"] (default: run)" +allowed-tools: Bash, Read, Write, Task +--- + +# /skillopt-sleep-handoff — session-executed sleep cycle + +You are driving **SkillOpt-Sleep in handoff mode**: the Python engine runs +every deterministic stage (harvest → mine → replay scoring → gate → stage) +and outsources each model call (attempt / judge / reflect) to YOU via +prompt files. No `claude -p` subprocess, no API key — the model work runs +on this session's budget, but each prompt MUST be answered in a fresh, +isolated context so the validation gate stays honest. + +## Requested action: $ARGUMENTS + +(If `$ARGUMENTS` is empty, treat it as `run`.) + +## The loop + +Repeat until the engine exits 0 (done) — at most 8 rounds: + +1. **Run the engine** via the bundled runner: + + ```bash + "${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" --backend handoff --project "$(pwd)" --scope invoked + ``` + + - exit 0 → the night is complete; go to "Finish" below. + - exit 3 → pending model calls; continue with step 2. + - anything else → stop and show the user the error output. + +2. **Read the batch**: `Read` `.skillopt-sleep-handoff/pending.json` in the + project. Each entry has `id`, `prompt`, `max_tokens`, `answer_file`. + +3. **Answer each prompt in ISOLATION** — this is the integrity rule: + - For each entry, launch a subagent (Task tool) whose ENTIRE input is + the `prompt` text verbatim. Add nothing: no summary of this session, + no mention of SkillOpt, no other prompts from the batch. + - Take the subagent's reply and `Write` the raw answer text (no + commentary, no code fences) to the entry's `answer_file`. + - NEVER answer from this session's own context — you have seen the + mined tasks and their references, so inline answers would contaminate + the held-out gate and fake the improvement score. + +4. **Re-run the same engine command** — it resumes from the answers + directory and either finishes or stages the next batch. + +## Finish + +- `Read` the `report.md` in the staging dir the engine printed and show + the user: held-out baseline → candidate score, the gate decision, the + proposed edits, and where the proposal is staged. +- Tell the user nothing live changed; offer `/skillopt-sleep adopt`. +- The engine archives `.skillopt-sleep-handoff/` on a completed real run; + do not delete it yourself. + +## Safety reminders + +- **Never** edit `CLAUDE.md` or `SKILL.md` yourself — only `adopt` does + that, with a backup. +- Mined tasks are pinned to `.skillopt-sleep-handoff/tasks.json` on round + one, so sessions created while answering prompts cannot shift the task + set. Do not edit that file. +- If a batch looks like it contains secrets or content the user would not + want re-processed, stop and ask before answering. diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py index 608487a..01293e1 100644 --- a/skillopt_sleep/__main__.py +++ b/skillopt_sleep/__main__.py @@ -13,7 +13,7 @@ Common flags: --max-tasks N cap mined tasks per run --target-skill-path PATH explicit live SKILL.md to stage/adopt --tasks-file PATH reviewed TaskRecord JSON file to replay instead of harvesting - --backend mock|claude|codex|copilot + --backend mock|claude|codex|copilot|handoff --source claude|codex|auto --model NAME --lookback-hours N @@ -69,7 +69,8 @@ def _report_payload(rep, outcome) -> Dict[str, Any]: def _add_common(p: argparse.ArgumentParser) -> None: p.add_argument("--project", default="") p.add_argument("--scope", default="", choices=["", "all", "invoked"]) - p.add_argument("--backend", default="", choices=["", "mock", "claude", "codex", "copilot"]) + p.add_argument("--backend", default="", + choices=["", "mock", "claude", "codex", "copilot", "handoff"]) p.add_argument("--model", default="") p.add_argument("--codex-path", default="", help="path to the real @openai/codex binary") p.add_argument("--claude-home", default="", help="override ~/.claude (also isolates state)") @@ -156,7 +157,14 @@ def cmd_run(args, dry: bool = False) -> int: file=sys.stderr, ) return 2 + if cfg.get("backend", "mock") == "handoff": + return _run_handoff(cfg, args, seed_tasks=tasks, task_meta=task_meta, dry=dry) outcome = run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=dry) + _print_run_report(outcome, args, task_meta) + return 0 + + +def _print_run_report(outcome, args, task_meta: Dict[str, Any]) -> None: rep = outcome.report if args.json: payload = _report_payload(rep, outcome) @@ -180,6 +188,187 @@ def cmd_run(args, dry: bool = False) -> int: print("[sleep] review it, then: python -m skillopt_sleep adopt") if outcome.adopted: print(f"[sleep] auto-adopted: {', '.join(outcome.adopted_paths)}") + + +def _handoff_dir_for(cfg) -> str: + project = cfg.get("invoked_project") or os.getcwd() + return os.environ.get("SKILLOPT_SLEEP_HANDOFF_DIR", "") or os.path.join( + project, ".skillopt-sleep-handoff" + ) + + +def _redact_deep(obj): + """Redact secret-looking substrings in every string of a JSON-like tree.""" + from skillopt_sleep.staging import redact_secrets + if isinstance(obj, str): + return redact_secrets(obj) + if isinstance(obj, list): + return [_redact_deep(x) for x in obj] + if isinstance(obj, dict): + return {k: _redact_deep(v) for k, v in obj.items()} + return obj + + +def _flush_handoff(backend, args) -> int: + prompts_path = backend.flush_pending() + if args.json: + print(json.dumps({ + "handoff_pending": len(backend.pending), + "prompts": prompts_path, + "answers_dir": backend.answers_dir, + }, ensure_ascii=False, indent=2)) + else: + print(f"[sleep] handoff: {len(backend.pending)} model call(s) need answers") + print(f"[sleep] prompts: {prompts_path}") + print(f"[sleep] write each raw answer to {backend.answers_dir}/.md, " + "then re-run this exact command to resume") + return 3 + + +def _handoff_mine_and_pin(cfg, args, backend, snapshot: str, dry: bool): + """Harvest + mine with the same knobs as run_sleep_cycle (harvest window, + target-skill filter, candidate-limit bump, LLM mining — routed through the + handoff files like every other model call), then pin the result to + ``tasks.json``. Session digests are pinned too, so the sessions created + while answering prompts cannot change what gets mined between rounds. + + Returns ``(exit_code, tasks)``; ``tasks is None`` means exit now. + """ + import time + + from skillopt_sleep.handoff_backend import PendingCalls + from skillopt_sleep.state import SleepState, _now_iso + from skillopt_sleep.types import SessionDigest + + project = cfg.get("invoked_project") or os.getcwd() + state = SleepState.load(cfg.state_path) + started = _now_iso() + + digests_path = os.path.join(backend.handoff_dir, "digests.json") + digests = None + if os.path.exists(digests_path): + try: + with open(digests_path, encoding="utf-8") as f: + raw = json.load(f) + known = set(SessionDigest.__dataclass_fields__) + digests = [SessionDigest(**{k: v for k, v in d.items() if k in known}) + for d in raw] + except Exception: + # Corrupted/truncated pin (e.g. an interrupted earlier round): + # fall back to a fresh harvest instead of crashing the run. + print("[sleep] handoff: digests.json unreadable — re-harvesting", + file=sys.stderr) + digests = None + if digests is None: + since = state.last_harvest_for(project) + lookback_hours = cfg.get("lookback_hours", 72) + if since is None and lookback_hours and lookback_hours > 0: + since = _now_iso(time.time() - lookback_hours * 3600) + max_tasks = cfg.get("max_tasks_per_night", 40) + session_limit = cfg.get("max_sessions_per_night", 0) or max_tasks * 3 + digests = harvest_for_config(cfg, since_iso=since, limit=session_limit) + os.makedirs(backend.handoff_dir, exist_ok=True) + with open(digests_path, "w", encoding="utf-8") as f: + json.dump(_redact_deep([d.to_dict() for d in digests]), f, + ensure_ascii=False, indent=2) + + max_tasks = cfg.get("max_tasks_per_night", 40) + session_limit = cfg.get("max_sessions_per_night", 0) or max_tasks * 3 + target_skill_path = cfg.managed_skill_path() if cfg.get("target_skill_path", "") else "" + target_skill_text = _read_text(target_skill_path) if target_skill_path else "" + candidate_limit = max_tasks + if cfg.get("target_task_filter", True) and target_skill_text: + candidate_limit = max(max_tasks, max_tasks * 3) + llm_miner = None + if cfg.get("llm_mine", True): + try: + from skillopt_sleep.llm_miner import make_llm_miner + llm_miner = make_llm_miner( + backend, max_sessions=session_limit, max_tasks=candidate_limit, + ) + except Exception: + llm_miner = None + try: + tasks = mine( + digests, + max_tasks=max_tasks, + candidate_limit=candidate_limit, + holdout_fraction=cfg.get("holdout_fraction", 0.34), + seed=cfg.get("seed", 42), + llm_miner=llm_miner, + target_skill_text=target_skill_text, + target_skill_path=target_skill_path, + ) + except PendingCalls: + tasks = [] + if backend.pending: + # LLM mining needs answers before the task set can be pinned. + return _flush_handoff(backend, args), None + if not tasks: + print("[sleep] handoff: no tasks mined — nothing to consolidate") + if not dry: + # Advance the harvest window like run_sleep_cycle's no-tasks + # branch, or every later run re-scans the same stale window. + state.set_last_harvest(project, started) + state.save() + return 0, None + payload = make_tasks_payload( + tasks, + project=project, + transcript_source=cfg.get("transcript_source", ""), + n_sessions=len(digests), + target_skill_path=target_skill_path, + ) + # NOT marked reviewed: feeding this snapshot back through --tasks-file + # with a real backend must still hit the human-review gate above. The + # driver itself loads it directly, with the same trust as in-cycle mining. + write_tasks_file(snapshot, _redact_deep(payload)) + print(f"[sleep] handoff: pinned {len(tasks)} tasks -> {snapshot}") + return 0, tasks + + +def _run_handoff(cfg, args, *, seed_tasks, task_meta: Dict[str, Any], dry: bool) -> int: + """Drive the handoff backend: run until model calls are needed, then + write the prompt batch and exit 3; on a fully-answered run, finish + normally. Session digests and mined tasks are pinned under the handoff + dir on the first rounds so wall-clock time between rounds (including + the very sessions that answer the prompts) cannot change the task set + and invalidate earlier answers. + """ + from skillopt_sleep.handoff_backend import HandoffBackend, PendingCalls + + hdir = _handoff_dir_for(cfg) + backend = HandoffBackend(model=cfg.get("model", ""), handoff_dir=hdir) + tasks = seed_tasks + if tasks is None: + snapshot = os.path.join(hdir, "tasks.json") + if os.path.exists(snapshot): + tasks, _meta = load_tasks_file( + snapshot, + holdout_fraction=cfg.get("holdout_fraction", 0.34), + seed=cfg.get("seed", 42), + ) + else: + rc, tasks = _handoff_mine_and_pin(cfg, args, backend, snapshot, dry) + if tasks is None: + return rc + outcome = None + try: + outcome = run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=dry, backend=backend) + except PendingCalls: + pass + if backend.pending: + return _flush_handoff(backend, args) + _print_run_report(outcome, args, task_meta) + # A completed real run ends the night: archive the handoff dir so the + # next night re-harvests instead of replaying the pinned snapshot. + if not dry and outcome.staging_dir and os.path.isdir(hdir): + import time + done = f"{hdir}.night{outcome.report.night}.done" + if os.path.exists(done): + done = f"{done}.{int(time.time())}" + os.rename(hdir, done) + print(f"[sleep] handoff: archived round data -> {done}") return 0 diff --git a/skillopt_sleep/backend.py b/skillopt_sleep/backend.py index ef56343..d7bb040 100644 --- a/skillopt_sleep/backend.py +++ b/skillopt_sleep/backend.py @@ -1415,6 +1415,13 @@ def get_backend( return AzureResponsesBackend(deployment=model, endpoints=eps) if n in {"copilot", "github_copilot", "copilot_cli", "gh_copilot"}: return CopilotCliBackend(model=model) + if n in {"handoff", "session", "file"}: + # Lazy import: handoff_backend imports CliBackend from this module. + from skillopt_sleep.handoff_backend import HandoffBackend + hdir = os.environ.get("SKILLOPT_SLEEP_HANDOFF_DIR", "") or os.path.join( + project_dir or os.getcwd(), ".skillopt-sleep-handoff" + ) + return HandoffBackend(model=model, handoff_dir=hdir) return MockBackend() diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index 6ad0d4f..cb54ca8 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -14,7 +14,7 @@ import sys from dataclasses import dataclass from typing import List, Optional -from skillopt_sleep.backend import get_backend +from skillopt_sleep.backend import Backend, get_backend from skillopt_sleep.config import SleepConfig, load_config from skillopt_sleep.dream import dream_consolidate from skillopt_sleep.harvest_sources import harvest_for_config @@ -94,6 +94,7 @@ def run_sleep_cycle( seed_tasks: Optional[List[TaskRecord]] = None, dry_run: bool = False, clock: Optional[float] = None, + backend: Optional[Backend] = None, ) -> CycleOutcome: """Run one full sleep cycle and return the outcome. @@ -104,6 +105,8 @@ def run_sleep_cycle( inject a known persona instead of harvesting ~/.claude). dry_run : harvest+mine+replay but DO NOT stage/adopt (report only). clock : fixed epoch seconds for deterministic timestamps in tests. + backend : optional pre-built Backend; the handoff driver passes one so + it can inspect the backend's pending calls after the run. """ cfg = cfg or load_config() state = SleepState.load(cfg.state_path) @@ -111,7 +114,7 @@ def run_sleep_cycle( project = _project_paths(cfg) started = _now_iso(clock) - backend = get_backend( + backend = backend or get_backend( cfg.get("backend", "mock"), model=cfg.get("model", ""), codex_path=cfg.get("codex_path", ""), diff --git a/skillopt_sleep/handoff_backend.py b/skillopt_sleep/handoff_backend.py new file mode 100644 index 0000000..0e41ae2 --- /dev/null +++ b/skillopt_sleep/handoff_backend.py @@ -0,0 +1,173 @@ +"""SkillOpt-Sleep — handoff backend (session-executed model calls). + +Runs the sleep cycle WITHOUT spawning any model subprocess or API call. +Every intelligent operation (attempt / judge / reflect) is turned into a +prompt file that an interactive agent session answers between engine runs: + + run 1: the engine executes the deterministic stages; every model call + it needs is recorded as a pending prompt; the run stops and + writes PROMPTS.md + pending.json into the handoff directory. + you: answer each prompt (each in a FRESH context, so the session's + own history cannot contaminate the held-out gate) and write the + raw answer text to answers/.md. + run 2: the engine re-runs; answered prompts resolve from answers/, the + cycle advances to the next model-dependent stage, and either + finishes or writes the next PROMPTS.md batch. + +Resume needs no serialized engine state: harvest -> mine -> replay is +deterministic, so re-running regenerates identical prompts and the answers +directory acts as a persistent, cross-run call cache. A prompt that embeds +a still-unanswered response (detected via the pending sentinel) aborts the +run immediately so placeholder text never propagates into scores, edits, +or staging. A typical night converges in 3-6 rounds: baseline attempts -> +reflect -> candidate re-scoring per accepted edit. + +Limitations (v1): `dream_rollouts > 1` yields no contrastive spread (the +same prompt maps to the same answer file), and tool-loop tasks fall back +to the base single-shot 'TOOL_CALL: ' marker convention. +""" +from __future__ import annotations + +import json +import os +import threading +from typing import Dict + +from skillopt_sleep.backend import CliBackend, skill_hash + +PENDING_SENTINEL_PREFIX = "[[SKILLOPT-SLEEP-PENDING:" +PENDING_SENTINEL_SUFFIX = "]]" + +# reflect() appends this when a reply fails to parse; with a placeholder +# reply the retry is a dependent call, not a genuinely new question. +_REFLECT_RETRY_MARKER = "your previous reply was not valid JSON" + +PROMPTS_FILENAME = "PROMPTS.md" +PENDING_FILENAME = "pending.json" + + +class PendingCalls(RuntimeError): + """The cycle cannot advance until pending prompts are answered.""" + + def __init__(self, pending: Dict[str, Dict[str, object]]): + self.pending = dict(pending) + super().__init__( + f"{len(self.pending)} model call(s) awaiting handoff answers" + ) + + +class HandoffBackend(CliBackend): + """Backend that outsources every model call to prompt/answer files. + + ``_call`` resolves a prompt from ``answers/.md`` when the + answer exists; otherwise it records the prompt as pending and returns a + sentinel placeholder so independent calls in the same phase can still + be collected into one batch. Any call whose prompt was BUILT FROM a + placeholder raises :class:`PendingCalls` — that call depends on answers + the user has not provided yet, so continuing would only mint garbage. + """ + + name = "handoff" + + def __init__(self, model: str = "", handoff_dir: str = "") -> None: + super().__init__(model=model, timeout=0) + self.handoff_dir = os.path.abspath( + handoff_dir or os.path.join(os.getcwd(), ".skillopt-sleep-handoff") + ) + self.answers_dir = os.path.join(self.handoff_dir, "answers") + os.makedirs(self.answers_dir, exist_ok=True) + # key -> {"prompt": str, "max_tokens": int}, insertion-ordered + self.pending: Dict[str, Dict[str, object]] = {} + self._lock = threading.Lock() + + # ── prompt/answer plumbing ──────────────────────────────────────────── + def answer_path(self, key: str) -> str: + return os.path.join(self.answers_dir, f"{key}.md") + + def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: + if PENDING_SENTINEL_PREFIX in prompt: + # Built from a still-pending response — dependent call. + raise PendingCalls(self.pending) + if _REFLECT_RETRY_MARKER in prompt and self.pending: + # Retry of a reflect whose first reply is the placeholder. + raise PendingCalls(self.pending) + key = skill_hash(prompt) + path = self.answer_path(key) + if os.path.exists(path): + with open(path, encoding="utf-8") as f: + return f.read().strip() + with self._lock: + self.pending[key] = {"prompt": prompt, "max_tokens": max_tokens} + return f"{PENDING_SENTINEL_PREFIX}{key}{PENDING_SENTINEL_SUFFIX}" + + # ── handoff file emission ───────────────────────────────────────────── + def flush_pending(self) -> str: + """Write PROMPTS.md (human/agent-readable) + pending.json (machine). + + Prompts can themselves contain markdown fences, so PROMPTS.md + delimits each prompt with BEGIN/END marker lines instead of fences. + Returns the PROMPTS.md path. + """ + from skillopt_sleep.staging import redact_secrets + + os.makedirs(self.handoff_dir, exist_ok=True) + with self._lock: + items = list(self.pending.items()) + payload = { + "format": "skillopt_sleep.handoff.v1", + "answers_dir": self.answers_dir, + "pending": [ + { + "id": key, + "answer_file": self.answer_path(key), + "max_tokens": item["max_tokens"], + "prompt": redact_secrets(str(item["prompt"])), + } + for key, item in items + ], + } + with open(os.path.join(self.handoff_dir, PENDING_FILENAME), "w", + encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + f.write("\n") + + lines = [ + "# SkillOpt-Sleep — pending model calls (handoff)", + "", + f"{len(items)} prompt(s) below need answers before the sleep " + "cycle can continue.", + "", + "For EACH prompt:", + "", + "1. Answer it in a FRESH context (e.g. a subagent with no", + " conversation history). Do NOT let the current session's", + " context, the other prompts in this file, or the optimization", + " run itself leak into the answer — that contaminates the", + " held-out validation gate.", + "2. Write ONLY the raw answer text (no commentary, no code", + " fences) to the prompt's answer file.", + "", + "When every answer file exists, re-run the same engine command", + "(`python -m skillopt_sleep run --backend handoff ...`); it", + "resumes automatically from the answers directory.", + "", + ] + for i, (key, item) in enumerate(items, start=1): + lines += [ + "---", + "", + f"## Prompt {i} of {len(items)}", + "", + f"- id: `{key}`", + f"- answer file: `answers/{key}.md`", + f"- suggested max tokens: {item['max_tokens']}", + "", + f"----- BEGIN PROMPT {key} -----", + redact_secrets(str(item["prompt"])), + f"----- END PROMPT {key} -----", + "", + ] + prompts_path = os.path.join(self.handoff_dir, PROMPTS_FILENAME) + with open(prompts_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + return prompts_path diff --git a/tests/test_handoff_backend.py b/tests/test_handoff_backend.py new file mode 100644 index 0000000..0ecfae1 --- /dev/null +++ b/tests/test_handoff_backend.py @@ -0,0 +1,220 @@ +"""Tests for the handoff backend: session-executed model calls via +prompt/answer files, resumed across engine runs.""" +from __future__ import annotations + +import json +import os +import re +import tempfile +import unittest + +from skillopt_sleep.backend import get_backend +from skillopt_sleep.config import load_config +from skillopt_sleep.cycle import run_sleep_cycle +from skillopt_sleep.handoff_backend import ( + PENDING_SENTINEL_PREFIX, + HandoffBackend, + PendingCalls, +) +from skillopt_sleep.mine import assign_splits +from skillopt_sleep.types import TaskRecord + +# The rule the simulated executor "learns"; once present in the skill the +# executor answers arithmetic tasks correctly, mirroring MockBackend's +# rule-gated model of reality. +RULE = "Always answer with just the number." + + +def _tasks(): + ts = [ + TaskRecord( + id=f"t{i}", project="/p", + intent=f"What is {i} + {i}?", + reference_kind="exact", reference=str(i + i), + ) + for i in range(1, 7) + ] + return assign_splits(ts, holdout_fraction=0.34, seed=42) + + +def _answer_pending(backend: HandoffBackend) -> None: + """Deterministic stand-in for the interactive session answering PROMPTS.md.""" + for key, item in list(backend.pending.items()): + prompt = str(item["prompt"]) + if "You are SkillOpt's optimizer" in prompt: + ans = json.dumps([{ + "op": "add", "content": RULE, + "rationale": "outputs failed exact match; answer bare numbers", + }]) + else: + m = re.search(r"What is (\d+) \+ (\d+)\?", prompt) + if m and RULE in prompt: + ans = str(int(m.group(1)) + int(m.group(2))) + else: + ans = "cannot say" + with open(backend.answer_path(key), "w", encoding="utf-8") as f: + f.write(ans) + + +class TestHandoffBackendUnit(unittest.TestCase): + def test_miss_records_pending_and_returns_sentinel(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + out = be._call("some prompt") + self.assertTrue(out.startswith(PENDING_SENTINEL_PREFIX)) + self.assertEqual(len(be.pending), 1) + + def test_answer_file_resolves_call(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + key = be._call("what is 2+2?").split(":")[-1].rstrip("]") + with open(be.answer_path(key), "w", encoding="utf-8") as f: + f.write("4\n") + fresh = HandoffBackend(handoff_dir=hdir) + self.assertEqual(fresh._call("what is 2+2?"), "4") + self.assertEqual(len(fresh.pending), 0) + + def test_dependent_prompt_raises(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + placeholder = be._call("first question") + with self.assertRaises(PendingCalls) as ctx: + be._call(f"judge this response: {placeholder}") + self.assertEqual(len(ctx.exception.pending), 1) + + def test_reflect_retry_of_pending_reply_raises(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + be._call("reflect on failures") + with self.assertRaises(PendingCalls): + be._call("reflect on failures\n\nIMPORTANT: " + "your previous reply was not valid JSON. Reply with " + "ONLY the JSON array, no prose, no markdown fences.") + + def test_flush_writes_prompts_and_pending_json(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + be._call("prompt A") + be._call("prompt B") + prompts_path = be.flush_pending() + self.assertTrue(os.path.exists(prompts_path)) + with open(os.path.join(hdir, "pending.json"), encoding="utf-8") as f: + payload = json.load(f) + self.assertEqual(payload["format"], "skillopt_sleep.handoff.v1") + self.assertEqual(len(payload["pending"]), 2) + self.assertEqual(payload["pending"][0]["prompt"], "prompt A") + with open(prompts_path, encoding="utf-8") as f: + md = f.read() + self.assertIn("BEGIN PROMPT", md) + self.assertIn("prompt B", md) + + def test_get_backend_registration(self): + with tempfile.TemporaryDirectory() as proj: + be = get_backend("handoff", project_dir=proj) + self.assertIsInstance(be, HandoffBackend) + self.assertTrue(be.handoff_dir.startswith(os.path.realpath(proj)) + or be.handoff_dir.startswith(proj)) + + +class TestHandoffCycle(unittest.TestCase): + def test_cycle_converges_over_handoff_rounds(self): + with tempfile.TemporaryDirectory() as proj, \ + tempfile.TemporaryDirectory() as home: + hdir = os.path.join(proj, ".skillopt-sleep-handoff") + + def cfg(): + return load_config( + invoked_project=proj, projects="invoked", + backend="handoff", + claude_home=os.path.join(home, ".claude"), + ) + + final = None + rounds = 0 + for _ in range(8): + rounds += 1 + backend = HandoffBackend(handoff_dir=hdir) + try: + outcome = run_sleep_cycle( + cfg(), seed_tasks=_tasks(), dry_run=True, backend=backend, + ) + except PendingCalls: + outcome = None + if backend.pending: + backend.flush_pending() + _answer_pending(backend) + continue + final = outcome + break + + self.assertIsNotNone(final, "cycle never completed within 8 rounds") + self.assertGreater(rounds, 1, "expected at least one handoff round") + rep = final.report + self.assertTrue(rep.accepted, f"gate did not accept: {rep.gate_action}") + self.assertGreater(rep.candidate_score, rep.baseline_score) + self.assertTrue(any(RULE in e.content for e in rep.edits)) + + +class TestHandoffCli(unittest.TestCase): + def test_run_exits_3_and_stages_prompts(self): + from skillopt_sleep.__main__ import main + from skillopt_sleep.tasks_file import make_tasks_payload, write_tasks_file + + with tempfile.TemporaryDirectory() as proj, \ + tempfile.TemporaryDirectory() as home: + tasks_path = os.path.join(proj, "tasks.json") + payload = make_tasks_payload(_tasks(), project=proj) + payload["reviewed"] = True + write_tasks_file(tasks_path, payload) + + rc = main([ + "run", "--backend", "handoff", + "--project", proj, + "--claude-home", os.path.join(home, ".claude"), + "--tasks-file", tasks_path, + ]) + self.assertEqual(rc, 3) + hdir = os.path.join(proj, ".skillopt-sleep-handoff") + self.assertTrue(os.path.exists(os.path.join(hdir, "PROMPTS.md"))) + self.assertTrue(os.path.exists(os.path.join(hdir, "pending.json"))) + + def test_corrupt_digests_pin_falls_back_to_reharvest(self): + from skillopt_sleep.__main__ import main + + with tempfile.TemporaryDirectory() as proj, \ + tempfile.TemporaryDirectory() as home: + hdir = os.path.join(proj, ".skillopt-sleep-handoff") + os.makedirs(hdir, exist_ok=True) + with open(os.path.join(hdir, "digests.json"), "w", encoding="utf-8") as f: + f.write("{not valid json") + rc = main([ + "run", "--backend", "handoff", + "--project", proj, + "--claude-home", os.path.join(home, ".claude"), + ]) + # must not crash: corrupt pin -> fresh harvest -> no tasks -> 0 + self.assertEqual(rc, 0) + + def test_run_with_no_tasks_exits_0_and_advances_harvest_window(self): + from skillopt_sleep.__main__ import main + from skillopt_sleep.config import load_config + from skillopt_sleep.state import SleepState + + with tempfile.TemporaryDirectory() as proj, \ + tempfile.TemporaryDirectory() as home: + claude_home = os.path.join(home, ".claude") + rc = main([ + "run", "--backend", "handoff", + "--project", proj, + "--claude-home", claude_home, + ]) + self.assertEqual(rc, 0) + # the no-tasks branch must persist last-harvest, otherwise every + # later run re-scans the same stale window forever + cfg = load_config(invoked_project=proj, claude_home=claude_home) + state = SleepState.load(cfg.state_path) + self.assertIsNotNone(state.last_harvest_for(proj)) + + +if __name__ == "__main__": + unittest.main()