SkillOpt/skillopt/sleep/experiments/run_experiment.py

"""SkillOpt-Sleep — validation experiment.

Answers the question the user posed: *does nightly offline self-evolution
actually improve the agent?*  Runs deterministically with the MockBackend
(no API key, reproducible) and is the acceptance test for the whole idea.

What it proves:
  1. MONOTONIC LIFT — over N sleep nights, the held-out score rises from a
     baseline (empty skill/memory) toward 1.0 as the gate accepts the
     general rules the persona's tasks require.
  2. GATE SAFETY — an injected harmful edit is REJECTED (held-out score does
     not improve), so a bad nightly proposal can never be adopted.
  3. PLUMBING — harvest->mine->replay->consolidate->stage->adopt all run and
     the adopted artifact, re-scored, retains the lift.

Run:
    python -m skillopt.sleep.experiments.run_experiment
    python -m skillopt.sleep.experiments.run_experiment --persona programmer --nights 3
    python -m skillopt.sleep.experiments.run_experiment --backend anthropic   # real lift
"""
from __future__ import annotations

import argparse
import json
import os
import sys
import tempfile
from typing import List

from skillopt.sleep.backend import get_backend
from skillopt.sleep.consolidate import consolidate
from skillopt.sleep.experiments.personas import (
    PERSONAS,
    harmful_edit_task,
    researcher_persona,
)
from skillopt.sleep.memory import ensure_skill_scaffold
from skillopt.sleep.replay import aggregate_scores, replay_batch
from skillopt.sleep.types import TaskRecord


def _score_holdout(backend, tasks: List[TaskRecord], skill: str, memory: str,
                   metric: str = "mixed", w: float = 0.5) -> float:
    from skillopt.sleep.consolidate import select_gate_score
    holdout = [t for t in tasks if t.split == "holdout"] or tasks
    pairs = replay_batch(backend, holdout, skill, memory)
    h, s = aggregate_scores(pairs)
    return select_gate_score(h, s, metric, w)


def run(persona: str = "researcher", nights: int = 4, backend_name: str = "mock",
        edit_budget: int = 4, seed: int = 42) -> dict:
    from skillopt.sleep.mine import assign_splits

    make = PERSONAS.get(persona, researcher_persona)
    tasks = assign_splits(make(), holdout_fraction=0.34, seed=seed)
    backend = get_backend(backend_name)

    # start from an empty managed skill + empty memory
    skill = ensure_skill_scaffold("", name="skillopt-sleep-learned",
                                  description="Learned preferences.")
    memory = ""

    baseline = _score_holdout(backend, tasks, skill, memory)
    trace = [{"night": 0, "holdout_score": round(baseline, 4), "action": "baseline",
              "n_edits": 0}]

    for night in range(1, nights + 1):
        res = consolidate(
            backend, tasks, skill, memory,
            edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
            evolve_skill=True, evolve_memory=True, night=night,
        )
        if res.accepted:
            skill, memory = res.new_skill, res.new_memory
        trace.append({
            "night": night,
            "holdout_score": round(res.candidate_score, 4),
            "action": res.gate_action,
            "accepted": res.accepted,
            "n_edits": len(res.applied_edits),
            "edits": [e.content for e in res.applied_edits],
            "n_rejected": len(res.rejected_edits),
        })
        # converged: stop early if perfect
        if res.candidate_score >= 0.999:
            break

    after = _score_holdout(backend, tasks, skill, memory)

    # ── gate-safety probe: inject a harmful task whose 'fix' is a bad rule ──
    harmful_tasks = assign_splits([harmful_edit_task()] + make()[:3],
                                  holdout_fraction=0.5, seed=seed)
    h_before = _score_holdout(backend, harmful_tasks, skill, memory)
    res_h = consolidate(backend, harmful_tasks, skill, memory,
                        edit_budget=edit_budget, gate_metric="mixed",
                        evolve_skill=True, evolve_memory=False, night=nights + 1)
    harmful_rule_text = get_backend("mock").RULE_TEXT["__harmful__"]  # type: ignore[attr-defined]
    harmful_rejected = (harmful_rule_text not in res_h.new_skill)

    result = {
        "persona": persona,
        "backend": backend_name,
        "nights_run": len(trace) - 1,
        "baseline_holdout": round(baseline, 4),
        "after_holdout": round(after, 4),
        "lift": round(after - baseline, 4),
        "improved": after > baseline,
        "gate_blocks_harmful": bool(harmful_rejected),
        "final_skill_excerpt": skill[-400:],
        "trace": trace,
    }
    return result


def _assert(cond: bool, msg: str) -> None:
    if not cond:
        print(f"FAIL: {msg}")
        raise SystemExit(1)


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description="SkillOpt-Sleep validation experiment")
    ap.add_argument("--persona", default="researcher", choices=list(PERSONAS.keys()))
    ap.add_argument("--nights", type=int, default=4)
    ap.add_argument("--backend", default="mock", choices=["mock", "anthropic"])
    ap.add_argument("--edit-budget", type=int, default=4)
    ap.add_argument("--json", action="store_true")
    ap.add_argument("--assert-improves", action="store_true",
                    help="exit nonzero unless lift>0 and gate blocks harmful edit")
    args = ap.parse_args(argv)

    res = run(args.persona, nights=args.nights, backend_name=args.backend,
              edit_budget=args.edit_budget)

    if args.json:
        print(json.dumps(res, ensure_ascii=False, indent=2))
    else:
        print(f"=== SkillOpt-Sleep experiment: persona={res['persona']} backend={res['backend']} ===")
        print(f"baseline held-out : {res['baseline_holdout']}")
        print(f"after  held-out   : {res['after_holdout']}   (lift {res['lift']:+.4f})")
        print(f"gate blocks harmful edit: {res['gate_blocks_harmful']}")
        print("trace:")
        for row in res["trace"]:
            edits = "; ".join(row.get("edits", []))[:80]
            print(f"  night {row['night']}: holdout={row['holdout_score']} "
                  f"{row['action']} (+{row['n_edits']} edits) {edits}")

    if args.assert_improves:
        _assert(res["improved"], "held-out score did not improve")
        _assert(res["gate_blocks_harmful"], "gate failed to block harmful edit")
        print("\nPASS: nightly consolidation improves held-out score AND gate blocks regressions.")
    return 0


if __name__ == "__main__":
    sys.exit(main())